feat: possible fix rate limiting and github being blocked in some
countries?
This commit is contained in:
parent
47737d1c19
commit
48b5e39f3a
1 changed files with 111 additions and 45 deletions
|
|
@ -16,11 +16,13 @@ import io.ktor.client.request.prepareGet
|
||||||
import io.ktor.client.statement.bodyAsChannel
|
import io.ktor.client.statement.bodyAsChannel
|
||||||
import io.ktor.http.HttpHeaders
|
import io.ktor.http.HttpHeaders
|
||||||
import io.ktor.http.contentLength
|
import io.ktor.http.contentLength
|
||||||
|
import io.ktor.http.isSuccess
|
||||||
import io.ktor.utils.io.readAvailable
|
import io.ktor.utils.io.readAvailable
|
||||||
import kotlinx.coroutines.CancellationException
|
import kotlinx.coroutines.CancellationException
|
||||||
import java.io.File
|
import java.io.File
|
||||||
import java.io.IOException
|
import java.io.IOException
|
||||||
import java.net.SocketTimeoutException
|
import java.net.SocketTimeoutException
|
||||||
|
import java.net.URI
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* Handle downloading remote urls to a path with Ktor.
|
* Handle downloading remote urls to a path with Ktor.
|
||||||
|
|
@ -39,55 +41,108 @@ class KtorDownloadManager(
|
||||||
.apply { exists() || createNewFile() }
|
.apply { exists() || createNewFile() }
|
||||||
|
|
||||||
try {
|
try {
|
||||||
val httpStmt = http.prepareGet(url) {
|
val downloadUrl = rewriteRawGithubUrl(url)
|
||||||
header(HttpHeaders.CacheControl, "no-cache, no-store")
|
|
||||||
|
|
||||||
// Disable compression due to bug on emulators
|
// Try the CDN first (jsDelivr), then an alternate raw proxy (raw.githack.com), and finally
|
||||||
// This header cannot be set with Android's DownloadManager
|
// fall back to the original URL if all CDN attempts fail.
|
||||||
if (IS_PROBABLY_EMULATOR) {
|
val rawGithack = try {
|
||||||
header(HttpHeaders.AcceptEncoding, null)
|
when {
|
||||||
|
// direct raw.githubusercontent.com -> raw.githack.com
|
||||||
|
url.contains("raw.githubusercontent.com", ignoreCase = true) ->
|
||||||
|
url.replace("raw.githubusercontent.com", "raw.githack.com")
|
||||||
|
// github.com/.../raw/... -> convert to raw.githack.com/ORG/REPO/BRANCH/path
|
||||||
|
url.contains("/raw/", ignoreCase = true) && url.contains("github.com", ignoreCase = true) -> {
|
||||||
|
try {
|
||||||
|
val u = URI(url)
|
||||||
|
val segs = u.path.trimStart('/').split('/', limit = 6)
|
||||||
|
if (segs.size >= 5 && segs[2].equals("raw", ignoreCase = true)) {
|
||||||
|
"https://raw.githack.com/${segs[0]}/${segs[1]}/${segs.subList(3, segs.size).joinToString("/")}"
|
||||||
|
} else url
|
||||||
|
} catch (_: Throwable) {
|
||||||
|
url
|
||||||
|
}
|
||||||
|
}
|
||||||
|
else -> url
|
||||||
|
}
|
||||||
|
} catch (_: Throwable) {
|
||||||
|
url
|
||||||
|
}
|
||||||
|
val attempts = listOf(downloadUrl, rawGithack, url).distinct()
|
||||||
|
var lastException: Throwable? = null
|
||||||
|
var downloaded = false
|
||||||
|
|
||||||
|
for (attemptUrl in attempts) {
|
||||||
|
val httpStmt = http.prepareGet(attemptUrl) {
|
||||||
|
header(HttpHeaders.CacheControl, "no-cache, no-store")
|
||||||
|
|
||||||
|
// Disable compression due to bug on emulators
|
||||||
|
// This header cannot be set with Android's DownloadManager
|
||||||
|
if (IS_PROBABLY_EMULATOR) {
|
||||||
|
header(HttpHeaders.AcceptEncoding, null)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
try {
|
||||||
|
httpStmt.execute { resp ->
|
||||||
|
if (!resp.status.isSuccess()) {
|
||||||
|
throw IOException("HTTP error ${resp.status.value} for $attemptUrl")
|
||||||
|
}
|
||||||
|
|
||||||
|
val channel = resp.bodyAsChannel()
|
||||||
|
val total = resp.contentLength() ?: 0
|
||||||
|
var retrieved = 0L
|
||||||
|
|
||||||
|
val buf = ByteArray(1024 * 1024 * 1)
|
||||||
|
var bufLen: Int
|
||||||
|
|
||||||
|
tmpOut.outputStream().use { stream ->
|
||||||
|
// Preallocate space for this file
|
||||||
|
if (total > 0 && Build.VERSION.SDK_INT >= 26) {
|
||||||
|
val storageManager = application.getSystemService<StorageManager>()!!
|
||||||
|
|
||||||
|
try {
|
||||||
|
storageManager.allocateBytes(stream.fd, total)
|
||||||
|
} catch (e: IOException) {
|
||||||
|
throw InsufficientStorageException(e.message)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
while (!channel.isClosedForRead) {
|
||||||
|
bufLen = channel.readAvailable(buf)
|
||||||
|
if (bufLen <= 0) break
|
||||||
|
|
||||||
|
stream.write(buf, 0, bufLen)
|
||||||
|
stream.flush()
|
||||||
|
|
||||||
|
retrieved += bufLen
|
||||||
|
|
||||||
|
if (total > 0) {
|
||||||
|
if (retrieved > total)
|
||||||
|
throw IOException("Total bytes received exceeds header total!")
|
||||||
|
|
||||||
|
onProgressUpdate?.onUpdate(retrieved / total.toFloat())
|
||||||
|
} else {
|
||||||
|
onProgressUpdate?.onUpdate(null)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// If we reach here, the attempt succeeded
|
||||||
|
downloaded = true
|
||||||
|
break
|
||||||
|
} catch (e: CancellationException) {
|
||||||
|
// Propagate cancellations immediately
|
||||||
|
throw e
|
||||||
|
} catch (t: Throwable) {
|
||||||
|
// Record the last exception and try the next URL (if any)
|
||||||
|
lastException = t
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
httpStmt.execute { resp ->
|
if (!downloaded) {
|
||||||
val channel = resp.bodyAsChannel()
|
// No attempt succeeded, throw the last failure to be handled by outer catch blocks
|
||||||
val total = resp.contentLength() ?: 0
|
throw lastException ?: IOException("Failed to download resource from $url")
|
||||||
var retrieved = 0L
|
|
||||||
|
|
||||||
val buf = ByteArray(1024 * 1024 * 1)
|
|
||||||
var bufLen: Int
|
|
||||||
|
|
||||||
tmpOut.outputStream().use { stream ->
|
|
||||||
// Preallocate space for this file
|
|
||||||
if (total > 0 && Build.VERSION.SDK_INT >= 26) {
|
|
||||||
val storageManager = application.getSystemService<StorageManager>()!!
|
|
||||||
|
|
||||||
try {
|
|
||||||
storageManager.allocateBytes(stream.fd, total)
|
|
||||||
} catch (e: IOException) {
|
|
||||||
throw InsufficientStorageException(e.message)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
while (!channel.isClosedForRead) {
|
|
||||||
bufLen = channel.readAvailable(buf)
|
|
||||||
if (bufLen <= 0) break
|
|
||||||
|
|
||||||
stream.write(buf, 0, bufLen)
|
|
||||||
stream.flush()
|
|
||||||
|
|
||||||
retrieved += bufLen
|
|
||||||
|
|
||||||
if (total > 0) {
|
|
||||||
if (retrieved > total)
|
|
||||||
throw IOException("Total bytes received exceeds header total!")
|
|
||||||
|
|
||||||
onProgressUpdate?.onUpdate(retrieved / total.toFloat())
|
|
||||||
} else {
|
|
||||||
onProgressUpdate?.onUpdate(null)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
}
|
||||||
} catch (_: CancellationException) {
|
} catch (_: CancellationException) {
|
||||||
tmpOut.delete()
|
tmpOut.delete()
|
||||||
|
|
@ -107,6 +162,17 @@ class KtorDownloadManager(
|
||||||
return Result.Success(out)
|
return Result.Success(out)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
private fun rewriteRawGithubUrl(url: String): String {
|
||||||
|
val regex = Regex("^https?://raw\\.githubusercontent\\.com/([^/]+)/([^/]+)/([^/]+)/(.*)$", RegexOption.IGNORE_CASE)
|
||||||
|
val match = regex.matchEntire(url)
|
||||||
|
return if (match != null) {
|
||||||
|
val (org, repo, branch, path) = match.destructured
|
||||||
|
"https://cdn.jsdelivr.net/gh/$org/$repo@$branch/$path"
|
||||||
|
} else {
|
||||||
|
url
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* Wrapper around an exception that occurred from invoking Ktor
|
* Wrapper around an exception that occurred from invoking Ktor
|
||||||
*/
|
*/
|
||||||
|
|
|
||||||
Loading…
Add table
Reference in a new issue