feat: possible fix rate limiting and github being blocked in some
countries?
This commit is contained in:
kmmiio99o
parent
47737d1c19
commit
48b5e39f3a
1 changed files with 111 additions and 45 deletions
|
|
@ -16,11 +16,13 @@ import io.ktor.client.request.prepareGet
|
|||
import io.ktor.client.statement.bodyAsChannel
|
||||
import io.ktor.http.HttpHeaders
|
||||
import io.ktor.http.contentLength
|
||||
import io.ktor.http.isSuccess
|
||||
import io.ktor.utils.io.readAvailable
|
||||
import kotlinx.coroutines.CancellationException
|
||||
import java.io.File
|
||||
import java.io.IOException
|
||||
import java.net.SocketTimeoutException
|
||||
import java.net.URI
|
||||
|
||||
/**
|
||||
* Handle downloading remote urls to a path with Ktor.
|
||||
|
|
@ -39,55 +41,108 @@ class KtorDownloadManager(
|
|||
.apply { exists() || createNewFile() }
|
||||
|
||||
try {
|
||||
val httpStmt = http.prepareGet(url) {
|
||||
header(HttpHeaders.CacheControl, "no-cache, no-store")
|
||||
val downloadUrl = rewriteRawGithubUrl(url)
|
||||
|
||||
// Disable compression due to bug on emulators
|
||||
// This header cannot be set with Android's DownloadManager
|
||||
if (IS_PROBABLY_EMULATOR) {
|
||||
header(HttpHeaders.AcceptEncoding, null)
|
||||
// Try the CDN first (jsDelivr), then an alternate raw proxy (raw.githack.com), and finally
|
||||
// fall back to the original URL if all CDN attempts fail.
|
||||
val rawGithack = try {
|
||||
when {
|
||||
// direct raw.githubusercontent.com -> raw.githack.com
|
||||
url.contains("raw.githubusercontent.com", ignoreCase = true) ->
|
||||
url.replace("raw.githubusercontent.com", "raw.githack.com")
|
||||
// github.com/.../raw/... -> convert to raw.githack.com/ORG/REPO/BRANCH/path
|
||||
url.contains("/raw/", ignoreCase = true) && url.contains("github.com", ignoreCase = true) -> {
|
||||
try {
|
||||
val u = URI(url)
|
||||
val segs = u.path.trimStart('/').split('/', limit = 6)
|
||||
if (segs.size >= 5 && segs[2].equals("raw", ignoreCase = true)) {
|
||||
"https://raw.githack.com/${segs[0]}/${segs[1]}/${segs.subList(3, segs.size).joinToString("/")}"
|
||||
} else url
|
||||
} catch (_: Throwable) {
|
||||
url
|
||||
}
|
||||
}
|
||||
else -> url
|
||||
}
|
||||
} catch (_: Throwable) {
|
||||
url
|
||||
}
|
||||
val attempts = listOf(downloadUrl, rawGithack, url).distinct()
|
||||
var lastException: Throwable? = null
|
||||
var downloaded = false
|
||||
|
||||
for (attemptUrl in attempts) {
|
||||
val httpStmt = http.prepareGet(attemptUrl) {
|
||||
header(HttpHeaders.CacheControl, "no-cache, no-store")
|
||||
|
||||
// Disable compression due to bug on emulators
|
||||
// This header cannot be set with Android's DownloadManager
|
||||
if (IS_PROBABLY_EMULATOR) {
|
||||
header(HttpHeaders.AcceptEncoding, null)
|
||||
}
|
||||
}
|
||||
|
||||
try {
|
||||
httpStmt.execute { resp ->
|
||||
if (!resp.status.isSuccess()) {
|
||||
throw IOException("HTTP error ${resp.status.value} for $attemptUrl")
|
||||
}
|
||||
|
||||
val channel = resp.bodyAsChannel()
|
||||
val total = resp.contentLength() ?: 0
|
||||
var retrieved = 0L
|
||||
|
||||
val buf = ByteArray(1024 * 1024 * 1)
|
||||
var bufLen: Int
|
||||
|
||||
tmpOut.outputStream().use { stream ->
|
||||
// Preallocate space for this file
|
||||
if (total > 0 && Build.VERSION.SDK_INT >= 26) {
|
||||
val storageManager = application.getSystemService<StorageManager>()!!
|
||||
|
||||
try {
|
||||
storageManager.allocateBytes(stream.fd, total)
|
||||
} catch (e: IOException) {
|
||||
throw InsufficientStorageException(e.message)
|
||||
}
|
||||
}
|
||||
|
||||
while (!channel.isClosedForRead) {
|
||||
bufLen = channel.readAvailable(buf)
|
||||
if (bufLen <= 0) break
|
||||
|
||||
stream.write(buf, 0, bufLen)
|
||||
stream.flush()
|
||||
|
||||
retrieved += bufLen
|
||||
|
||||
if (total > 0) {
|
||||
if (retrieved > total)
|
||||
throw IOException("Total bytes received exceeds header total!")
|
||||
|
||||
onProgressUpdate?.onUpdate(retrieved / total.toFloat())
|
||||
} else {
|
||||
onProgressUpdate?.onUpdate(null)
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// If we reach here, the attempt succeeded
|
||||
downloaded = true
|
||||
break
|
||||
} catch (e: CancellationException) {
|
||||
// Propagate cancellations immediately
|
||||
throw e
|
||||
} catch (t: Throwable) {
|
||||
// Record the last exception and try the next URL (if any)
|
||||
lastException = t
|
||||
}
|
||||
}
|
||||
|
||||
httpStmt.execute { resp ->
|
||||
val channel = resp.bodyAsChannel()
|
||||
val total = resp.contentLength() ?: 0
|
||||
var retrieved = 0L
|
||||
|
||||
val buf = ByteArray(1024 * 1024 * 1)
|
||||
var bufLen: Int
|
||||
|
||||
tmpOut.outputStream().use { stream ->
|
||||
// Preallocate space for this file
|
||||
if (total > 0 && Build.VERSION.SDK_INT >= 26) {
|
||||
val storageManager = application.getSystemService<StorageManager>()!!
|
||||
|
||||
try {
|
||||
storageManager.allocateBytes(stream.fd, total)
|
||||
} catch (e: IOException) {
|
||||
throw InsufficientStorageException(e.message)
|
||||
}
|
||||
}
|
||||
|
||||
while (!channel.isClosedForRead) {
|
||||
bufLen = channel.readAvailable(buf)
|
||||
if (bufLen <= 0) break
|
||||
|
||||
stream.write(buf, 0, bufLen)
|
||||
stream.flush()
|
||||
|
||||
retrieved += bufLen
|
||||
|
||||
if (total > 0) {
|
||||
if (retrieved > total)
|
||||
throw IOException("Total bytes received exceeds header total!")
|
||||
|
||||
onProgressUpdate?.onUpdate(retrieved / total.toFloat())
|
||||
} else {
|
||||
onProgressUpdate?.onUpdate(null)
|
||||
}
|
||||
}
|
||||
}
|
||||
if (!downloaded) {
|
||||
// No attempt succeeded, throw the last failure to be handled by outer catch blocks
|
||||
throw lastException ?: IOException("Failed to download resource from $url")
|
||||
}
|
||||
} catch (_: CancellationException) {
|
||||
tmpOut.delete()
|
||||
|
|
@ -107,6 +162,17 @@ class KtorDownloadManager(
|
|||
return Result.Success(out)
|
||||
}
|
||||
|
||||
private fun rewriteRawGithubUrl(url: String): String {
|
||||
val regex = Regex("^https?://raw\\.githubusercontent\\.com/([^/]+)/([^/]+)/([^/]+)/(.*)$", RegexOption.IGNORE_CASE)
|
||||
val match = regex.matchEntire(url)
|
||||
return if (match != null) {
|
||||
val (org, repo, branch, path) = match.destructured
|
||||
"https://cdn.jsdelivr.net/gh/$org/$repo@$branch/$path"
|
||||
} else {
|
||||
url
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Wrapper around an exception that occurred from invoking Ktor
|
||||
*/
|
||||
|
|
|
|||
Loading…
Add table
Reference in a new issue