WebView based cloudflare interceptor

build.gradle.kts

@@ -18,8 +18,8 @@ allprojects {
     repositories {
         mavenCentral()
         google()
-        maven("https://jitpack.io")
         maven("https://github.com/Suwayomi/Tachidesk-Server/raw/android-jar/")
+        maven("https://jitpack.io")
     }
 }
 

server/build.gradle.kts

@@ -57,7 +57,7 @@ dependencies {
     implementation("com.github.junrar:junrar:7.5.3")
 
     // CloudflareInterceptor
-    implementation("net.sourceforge.htmlunit:htmlunit:2.65.1")
+    implementation("com.microsoft.playwright:playwright:1.28.0")
 
     // AES/CBC/PKCS7Padding Cypher provider for zh.copymanga
     implementation("org.bouncycastle:bcprov-jdk18on:1.72")

server/src/main/kotlin/eu/kanade/tachiyomi/network/NetworkHelper.kt

@@ -39,6 +39,7 @@ class NetworkHelper(context: Context) {
                 .cookieJar(cookieManager)
                 .connectTimeout(30, TimeUnit.SECONDS)
                 .readTimeout(30, TimeUnit.SECONDS)
+                .callTimeout(2, TimeUnit.MINUTES)
                 .addInterceptor(UserAgentInterceptor())
 
             if (serverConfig.debugLogsEnabled) {

server/src/main/kotlin/eu/kanade/tachiyomi/network/OkHttpExtensions.kt

@@ -62,7 +62,7 @@ suspend fun Call.await(): Response {
             object : Callback {
                 override fun onResponse(call: Call, response: Response) {
                     if (!response.isSuccessful) {
-                        continuation.resumeWithException(Exception("HTTP error ${response.code}"))
+                        continuation.resumeWithException(HttpException(response.code))
                         return
                     }
 
@@ -94,7 +94,7 @@ fun Call.asObservableSuccess(): Observable<Response> {
         .doOnNext { response ->
             if (!response.isSuccessful) {
                 response.close()
-                throw Exception("HTTP error ${response.code}")
+                throw HttpException(response.code)
             }
         }
 }
@@ -136,3 +136,5 @@ inline fun <reified T> Response.parseAs(): T {
         return json.decodeFromString(responseBody)
     }
 }
+
+class HttpException(val code: Int) : IllegalStateException("HTTP error $code")

server/src/main/kotlin/eu/kanade/tachiyomi/network/interceptor/CloudflareInterceptor.kt

@@ -1,19 +1,24 @@
 package eu.kanade.tachiyomi.network.interceptor
 
-import com.gargoylesoftware.htmlunit.BrowserVersion
-import com.gargoylesoftware.htmlunit.WebClient
-import com.gargoylesoftware.htmlunit.html.HtmlPage
+import com.microsoft.playwright.Browser
+import com.microsoft.playwright.BrowserType.LaunchOptions
+import com.microsoft.playwright.Page
+import com.microsoft.playwright.Playwright
 import eu.kanade.tachiyomi.network.NetworkHelper
+import eu.kanade.tachiyomi.network.interceptor.CFClearance.resolveWithWebView
 import mu.KotlinLogging
 import okhttp3.Cookie
 import okhttp3.HttpUrl
 import okhttp3.Interceptor
 import okhttp3.Request
 import okhttp3.Response
+import suwayomi.tachidesk.server.ServerConfig
+import suwayomi.tachidesk.server.serverConfig
 import uy.kohesive.injekt.injectLazy
 import java.io.IOException
+import kotlin.time.Duration.Companion.seconds
+import kotlin.time.DurationUnit
 
-// from TachiWeb-Server
 class CloudflareInterceptor : Interceptor {
     private val logger = KotlinLogging.logger {}
 
@@ -25,86 +30,171 @@ class CloudflareInterceptor : Interceptor {
 
         logger.trace { "CloudflareInterceptor is being used." }
 
-        val response = chain.proceed(originalRequest)
+        val originalResponse = chain.proceed(chain.request())
 
         // Check if Cloudflare anti-bot is on
-        if (response.code != 503 || response.header("Server") !in SERVER_CHECK) {
-            return response
+        if (!(originalResponse.code in ERROR_CODES && originalResponse.header("Server") in SERVER_CHECK)) {
+            return originalResponse
         }
 
         logger.debug { "Cloudflare anti-bot is on, CloudflareInterceptor is kicking in..." }
 
         return try {
-            response.close()
+            originalResponse.close()
             network.cookies.remove(originalRequest.url.toUri())
 
-            chain.proceed(resolveChallenge(response))
+            val request = resolveWithWebView(originalRequest)
+
+            chain.proceed(request)
         } catch (e: Exception) {
             // Because OkHttp's enqueue only handles IOExceptions, wrap the exception so that
             // we don't crash the entire app
             throw IOException(e)
         }
     }
 
-    private fun resolveChallenge(response: Response): Request {
-        val browserVersion = BrowserVersion.BrowserVersionBuilder(BrowserVersion.BEST_SUPPORTED)
-            .setUserAgent(response.request.header("User-Agent") ?: BrowserVersion.BEST_SUPPORTED.userAgent)
-            .build()
-        val convertedCookies = WebClient(browserVersion).use { webClient ->
-            webClient.options.isThrowExceptionOnFailingStatusCode = false
-            webClient.options.isThrowExceptionOnScriptError = false
-            webClient.getPage<HtmlPage>(response.request.url.toString())
-            webClient.waitForBackgroundJavaScript(10000)
-            // Challenge solved, process cookies
-            webClient.cookieManager.cookies.filter {
-                // Only include Cloudflare cookies
-                it.name.startsWith("__cf") || it.name.startsWith("cf_")
-            }.map {
-                // Convert cookies -> OkHttp format
-                Cookie.Builder()
-                    .domain(it.domain.removePrefix("."))
-                    .expiresAt(it.expires?.time ?: Long.MAX_VALUE)
-                    .name(it.name)
-                    .path(it.path)
-                    .value(it.value).apply {
-                        if (it.isHttpOnly) httpOnly()
-                        if (it.isSecure) secure()
-                    }.build()
+    companion object {
+        private val ERROR_CODES = listOf(403, 503)
+        private val SERVER_CHECK = arrayOf("cloudflare-nginx", "cloudflare")
+        private val COOKIE_NAMES = listOf("cf_clearance")
+    }
+}
+
+/*
+ * This class is ported from https://github.com/vvanglro/cf-clearance
+ * The original code is licensed under Apache 2.0
+*/
+object CFClearance {
+    private val logger = KotlinLogging.logger {}
+    private val network: NetworkHelper by injectLazy()
+
+    fun resolveWithWebView(originalRequest: Request): Request {
+        val url = originalRequest.url.toString()
+
+        logger.debug { "resolveWithWebView($url)" }
+
+        val cookies = Playwright.create().use { playwright ->
+            playwright.chromium().launch(
+                LaunchOptions()
+                    .setHeadless(false)
+                    .apply {
+                        if (serverConfig.socksProxyEnabled) {
+                            setProxy("socks5://${serverConfig.socksProxyHost}:${serverConfig.socksProxyPort}")
+                        }
+                    }
+            ).use { browser ->
+                val userAgent = originalRequest.header("User-Agent")
+                if (userAgent != null) {
+                    browser.newContext(Browser.NewContextOptions().setUserAgent(userAgent)).use { browserContext ->
+                        browserContext.newPage().use { getCookies(it, url) }
+                    }
+                } else {
+                    browser.newPage().use { getCookies(it, url) }
+                }
             }
         }
 
         // Copy cookies to cookie store
-        convertedCookies.forEach {
+        cookies.groupBy { it.domain }.forEach { (domain, cookies) ->
             network.cookies.addAll(
-                HttpUrl.Builder()
+                url = HttpUrl.Builder()
                     .scheme("http")
-                    .host(it.domain)
+                    .host(domain)
                     .build(),
-                listOf(it)
+                cookies = cookies
             )
         }
         // Merge new and existing cookies for this request
         // Find the cookies that we need to merge into this request
-        val convertedForThisRequest = convertedCookies.filter {
-            it.matches(response.request.url)
+        val convertedForThisRequest = cookies.filter {
+            it.matches(originalRequest.url)
         }
         // Extract cookies from current request
         val existingCookies = Cookie.parseAll(
-            response.request.url,
-            response.request.headers
+            originalRequest.url,
+            originalRequest.headers
         )
         // Filter out existing values of cookies that we are about to merge in
         val filteredExisting = existingCookies.filter { existing ->
             convertedForThisRequest.none { converted -> converted.name == existing.name }
         }
+        logger.trace { "Existing cookies" }
+        logger.trace { existingCookies.joinToString("; ") }
         val newCookies = filteredExisting + convertedForThisRequest
-        return response.request.newBuilder()
-            .header("Cookie", newCookies.map { it.toString() }.joinToString("; "))
+        logger.trace { "New cookies" }
+        logger.trace { newCookies.joinToString("; ") }
+        return originalRequest.newBuilder()
+            .header("Cookie", newCookies.joinToString("; ") { "${it.name}=${it.value}" })
             .build()
     }
 
-    companion object {
-        private val SERVER_CHECK = arrayOf("cloudflare-nginx", "cloudflare")
-        private val COOKIE_NAMES = listOf("cf_clearance")
+    private fun getCookies(page: Page, url: String): List<Cookie> {
+        applyStealthInitScripts(page)
+        page.navigate(url)
+        val challengeResolved = waitForChallengeResolve(page)
+
+        return if (challengeResolved) {
+            val cookies = page.context().cookies()
+
+            logger.debug {
+                val userAgent = page.evaluate("() => {return navigator.userAgent}")
+                "Playwright User-Agent is $userAgent"
+            }
+
+            // Convert PlayWright cookies to OkHttp cookies
+            cookies.map {
+                Cookie.Builder()
+                    .domain(it.domain.removePrefix("."))
+                    .expiresAt(it.expires?.times(1000)?.toLong() ?: Long.MAX_VALUE)
+                    .name(it.name)
+                    .path(it.path)
+                    .value(it.value).apply {
+                        if (it.httpOnly) httpOnly()
+                        if (it.secure) secure()
+                    }.build()
+            }
+        } else {
+            logger.debug { "Cloudflare challenge failed to resolve" }
+            throw CloudflareBypassException()
+        }
     }
+
+    // ref: https://github.com/vvanglro/cf-clearance/blob/44124a8f06d8d0ecf2bf558a027082ff88dab435/cf_clearance/stealth.py#L18
+    private val stealthInitScripts by lazy {
+        arrayOf(
+            ServerConfig::class.java.getResource("/cloudflare-js/canvas.fingerprinting.js")!!.readText(),
+            ServerConfig::class.java.getResource("/cloudflare-js/chrome.global.js")!!.readText(),
+            ServerConfig::class.java.getResource("/cloudflare-js/emulate.touch.js")!!.readText(),
+            ServerConfig::class.java.getResource("/cloudflare-js/navigator.permissions.js")!!.readText(),
+            ServerConfig::class.java.getResource("/cloudflare-js/navigator.webdriver.js")!!.readText(),
+            ServerConfig::class.java.getResource("/cloudflare-js/chrome.runtime.js")!!.readText(),
+            ServerConfig::class.java.getResource("/cloudflare-js/chrome.plugin.js")!!.readText()
+        )
+    }
+
+    // ref: https://github.com/vvanglro/cf-clearance/blob/44124a8f06d8d0ecf2bf558a027082ff88dab435/cf_clearance/stealth.py#L76
+    private fun applyStealthInitScripts(page: Page) {
+        for (script in stealthInitScripts) {
+            page.addInitScript(script)
+        }
+    }
+
+    // ref: https://github.com/vvanglro/cf-clearance/blob/44124a8f06d8d0ecf2bf558a027082ff88dab435/cf_clearance/retry.py#L21
+    private fun waitForChallengeResolve(page: Page): Boolean {
+        // sometimes the user has to solve the captcha challenge manually, potentially wait a long time
+        val timeoutSeconds = 120
+        repeat(timeoutSeconds) {
+            page.waitForTimeout(1.seconds.toDouble(DurationUnit.MILLISECONDS))
+            val success = try {
+                page.querySelector("#challenge-form") == null
+            } catch (e: Exception) {
+                logger.debug(e) { "query Error" }
+                false
+            }
+            if (success) return true
+        }
+        return false
+    }
+
+    private class CloudflareBypassException : Exception()
 }

server/src/main/resources/cloudflare-js/canvas.fingerprinting.js

@@ -0,0 +1,28 @@
+(function () {
+    const ORIGINAL_CANVAS = HTMLCanvasElement.prototype[name];
+    Object.defineProperty(HTMLCanvasElement.prototype, name, {
+        "value": function () {
+            var shift = {
+                'r': Math.floor(Math.random() * 10) - 5,
+                'g': Math.floor(Math.random() * 10) - 5,
+                'b': Math.floor(Math.random() * 10) - 5,
+                'a': Math.floor(Math.random() * 10) - 5
+            };
+            var width = this.width,
+                height = this.height,
+                context = this.getContext("2d");
+            var imageData = context.getImageData(0, 0, width, height);
+            for (var i = 0; i < height; i++) {
+                for (var j = 0; j < width; j++) {
+                    var n = ((i * (width * 4)) + (j * 4));
+                    imageData.data[n + 0] = imageData.data[n + 0] + shift.r;
+                    imageData.data[n + 1] = imageData.data[n + 1] + shift.g;
+                    imageData.data[n + 2] = imageData.data[n + 2] + shift.b;
+                    imageData.data[n + 3] = imageData.data[n + 3] + shift.a;
+                }
+            }
+            context.putImageData(imageData, 0, 0);
+            return ORIGINAL_CANVAS.apply(this, arguments);
+        }
+    });
+})(this);

server/src/main/resources/cloudflare-js/chrome.global.js

@@ -0,0 +1,52 @@
+Object.defineProperty(window, 'chrome', {
+    value: new Proxy(window.chrome, {
+        has: (target, key) => true,
+        get: (target, key) => {
+            return {
+                app: {
+                    isInstalled: false,
+                },
+                webstore: {
+                    onInstallStageChanged: {},
+                    onDownloadProgress: {},
+                },
+                runtime: {
+                    PlatformOs: {
+                        MAC: 'mac',
+                        WIN: 'win',
+                        ANDROID: 'android',
+                        CROS: 'cros',
+                        LINUX: 'linux',
+                        OPENBSD: 'openbsd',
+                    },
+                    PlatformArch: {
+                        ARM: 'arm',
+                        X86_32: 'x86-32',
+                        X86_64: 'x86-64',
+                    },
+                    PlatformNaclArch: {
+                        ARM: 'arm',
+                        X86_32: 'x86-32',
+                        X86_64: 'x86-64',
+                    },
+                    RequestUpdateCheckStatus: {
+                        THROTTLED: 'throttled',
+                        NO_UPDATE: 'no_update',
+                        UPDATE_AVAILABLE: 'update_available',
+                    },
+                    OnInstalledReason: {
+                        INSTALL: 'install',
+                        UPDATE: 'update',
+                        CHROME_UPDATE: 'chrome_update',
+                        SHARED_MODULE_UPDATE: 'shared_module_update',
+                    },
+                    OnRestartRequiredReason: {
+                        APP_UPDATE: 'app_update',
+                        OS_UPDATE: 'os_update',
+                        PERIODIC: 'periodic',
+                    },
+                },
+            }
+        }
+    })
+});