feat: possible fix rate limiting and github being blocked in some

countries?
This commit is contained in:
kmmiio99o 2026-01-10 17:21:14 +01:00
parent 47737d1c19
commit 48b5e39f3a
No known key found for this signature in database
GPG key ID: 163FE39F7772D2EC

View file

@ -16,11 +16,13 @@ import io.ktor.client.request.prepareGet
import io.ktor.client.statement.bodyAsChannel import io.ktor.client.statement.bodyAsChannel
import io.ktor.http.HttpHeaders import io.ktor.http.HttpHeaders
import io.ktor.http.contentLength import io.ktor.http.contentLength
import io.ktor.http.isSuccess
import io.ktor.utils.io.readAvailable import io.ktor.utils.io.readAvailable
import kotlinx.coroutines.CancellationException import kotlinx.coroutines.CancellationException
import java.io.File import java.io.File
import java.io.IOException import java.io.IOException
import java.net.SocketTimeoutException import java.net.SocketTimeoutException
import java.net.URI
/** /**
* Handle downloading remote urls to a path with Ktor. * Handle downloading remote urls to a path with Ktor.
@ -39,55 +41,108 @@ class KtorDownloadManager(
.apply { exists() || createNewFile() } .apply { exists() || createNewFile() }
try { try {
val httpStmt = http.prepareGet(url) { val downloadUrl = rewriteRawGithubUrl(url)
header(HttpHeaders.CacheControl, "no-cache, no-store")
// Disable compression due to bug on emulators // Try the CDN first (jsDelivr), then an alternate raw proxy (raw.githack.com), and finally
// This header cannot be set with Android's DownloadManager // fall back to the original URL if all CDN attempts fail.
if (IS_PROBABLY_EMULATOR) { val rawGithack = try {
header(HttpHeaders.AcceptEncoding, null) when {
// direct raw.githubusercontent.com -> raw.githack.com
url.contains("raw.githubusercontent.com", ignoreCase = true) ->
url.replace("raw.githubusercontent.com", "raw.githack.com")
// github.com/.../raw/... -> convert to raw.githack.com/ORG/REPO/BRANCH/path
url.contains("/raw/", ignoreCase = true) && url.contains("github.com", ignoreCase = true) -> {
try {
val u = URI(url)
val segs = u.path.trimStart('/').split('/', limit = 6)
if (segs.size >= 5 && segs[2].equals("raw", ignoreCase = true)) {
"https://raw.githack.com/${segs[0]}/${segs[1]}/${segs.subList(3, segs.size).joinToString("/")}"
} else url
} catch (_: Throwable) {
url
}
}
else -> url
}
} catch (_: Throwable) {
url
}
val attempts = listOf(downloadUrl, rawGithack, url).distinct()
var lastException: Throwable? = null
var downloaded = false
for (attemptUrl in attempts) {
val httpStmt = http.prepareGet(attemptUrl) {
header(HttpHeaders.CacheControl, "no-cache, no-store")
// Disable compression due to bug on emulators
// This header cannot be set with Android's DownloadManager
if (IS_PROBABLY_EMULATOR) {
header(HttpHeaders.AcceptEncoding, null)
}
}
try {
httpStmt.execute { resp ->
if (!resp.status.isSuccess()) {
throw IOException("HTTP error ${resp.status.value} for $attemptUrl")
}
val channel = resp.bodyAsChannel()
val total = resp.contentLength() ?: 0
var retrieved = 0L
val buf = ByteArray(1024 * 1024 * 1)
var bufLen: Int
tmpOut.outputStream().use { stream ->
// Preallocate space for this file
if (total > 0 && Build.VERSION.SDK_INT >= 26) {
val storageManager = application.getSystemService<StorageManager>()!!
try {
storageManager.allocateBytes(stream.fd, total)
} catch (e: IOException) {
throw InsufficientStorageException(e.message)
}
}
while (!channel.isClosedForRead) {
bufLen = channel.readAvailable(buf)
if (bufLen <= 0) break
stream.write(buf, 0, bufLen)
stream.flush()
retrieved += bufLen
if (total > 0) {
if (retrieved > total)
throw IOException("Total bytes received exceeds header total!")
onProgressUpdate?.onUpdate(retrieved / total.toFloat())
} else {
onProgressUpdate?.onUpdate(null)
}
}
}
}
// If we reach here, the attempt succeeded
downloaded = true
break
} catch (e: CancellationException) {
// Propagate cancellations immediately
throw e
} catch (t: Throwable) {
// Record the last exception and try the next URL (if any)
lastException = t
} }
} }
httpStmt.execute { resp -> if (!downloaded) {
val channel = resp.bodyAsChannel() // No attempt succeeded, throw the last failure to be handled by outer catch blocks
val total = resp.contentLength() ?: 0 throw lastException ?: IOException("Failed to download resource from $url")
var retrieved = 0L
val buf = ByteArray(1024 * 1024 * 1)
var bufLen: Int
tmpOut.outputStream().use { stream ->
// Preallocate space for this file
if (total > 0 && Build.VERSION.SDK_INT >= 26) {
val storageManager = application.getSystemService<StorageManager>()!!
try {
storageManager.allocateBytes(stream.fd, total)
} catch (e: IOException) {
throw InsufficientStorageException(e.message)
}
}
while (!channel.isClosedForRead) {
bufLen = channel.readAvailable(buf)
if (bufLen <= 0) break
stream.write(buf, 0, bufLen)
stream.flush()
retrieved += bufLen
if (total > 0) {
if (retrieved > total)
throw IOException("Total bytes received exceeds header total!")
onProgressUpdate?.onUpdate(retrieved / total.toFloat())
} else {
onProgressUpdate?.onUpdate(null)
}
}
}
} }
} catch (_: CancellationException) { } catch (_: CancellationException) {
tmpOut.delete() tmpOut.delete()
@ -107,6 +162,17 @@ class KtorDownloadManager(
return Result.Success(out) return Result.Success(out)
} }
private fun rewriteRawGithubUrl(url: String): String {
val regex = Regex("^https?://raw\\.githubusercontent\\.com/([^/]+)/([^/]+)/([^/]+)/(.*)$", RegexOption.IGNORE_CASE)
val match = regex.matchEntire(url)
return if (match != null) {
val (org, repo, branch, path) = match.destructured
"https://cdn.jsdelivr.net/gh/$org/$repo@$branch/$path"
} else {
url
}
}
/** /**
* Wrapper around an exception that occurred from invoking Ktor * Wrapper around an exception that occurred from invoking Ktor
*/ */