From c53cbc6dd13337fa163e6842e2411f9a49d76fbc Mon Sep 17 00:00:00 2001 From: Aria Moradi Date: Sun, 4 Dec 2022 04:46:21 +0330 Subject: [PATCH 1/5] WebView based cloudflare interceptor ported https://github.com/vvanglro/cf-clearance to kotlin --- build.gradle.kts | 2 +- server/build.gradle.kts | 2 +- .../kanade/tachiyomi/network/NetworkHelper.kt | 1 + .../tachiyomi/network/OkHttpExtensions.kt | 6 +- .../interceptor/CloudflareInterceptor.kt | 140 +++++++++++++----- 5 files changed, 108 insertions(+), 43 deletions(-) diff --git a/build.gradle.kts b/build.gradle.kts index 6693ac343..162af5336 100644 --- a/build.gradle.kts +++ b/build.gradle.kts @@ -18,8 +18,8 @@ allprojects { repositories { mavenCentral() google() - maven("https://jitpack.io") maven("https://github.com/Suwayomi/Tachidesk-Server/raw/android-jar/") + maven("https://jitpack.io") } } diff --git a/server/build.gradle.kts b/server/build.gradle.kts index 0bdbb49d9..3c568de7e 100644 --- a/server/build.gradle.kts +++ b/server/build.gradle.kts @@ -57,7 +57,7 @@ dependencies { implementation("com.github.junrar:junrar:7.5.3") // CloudflareInterceptor - implementation("net.sourceforge.htmlunit:htmlunit:2.65.1") + implementation("com.microsoft.playwright:playwright:1.28.0") // AES/CBC/PKCS7Padding Cypher provider for zh.copymanga implementation("org.bouncycastle:bcprov-jdk18on:1.72") diff --git a/server/src/main/kotlin/eu/kanade/tachiyomi/network/NetworkHelper.kt b/server/src/main/kotlin/eu/kanade/tachiyomi/network/NetworkHelper.kt index 18a2b5e14..a63a25ced 100644 --- a/server/src/main/kotlin/eu/kanade/tachiyomi/network/NetworkHelper.kt +++ b/server/src/main/kotlin/eu/kanade/tachiyomi/network/NetworkHelper.kt @@ -39,6 +39,7 @@ class NetworkHelper(context: Context) { .cookieJar(cookieManager) .connectTimeout(30, TimeUnit.SECONDS) .readTimeout(30, TimeUnit.SECONDS) + .callTimeout(2, TimeUnit.MINUTES) .addInterceptor(UserAgentInterceptor()) if (serverConfig.debugLogsEnabled) { diff --git a/server/src/main/kotlin/eu/kanade/tachiyomi/network/OkHttpExtensions.kt b/server/src/main/kotlin/eu/kanade/tachiyomi/network/OkHttpExtensions.kt index c8f17d309..0e6eb7751 100644 --- a/server/src/main/kotlin/eu/kanade/tachiyomi/network/OkHttpExtensions.kt +++ b/server/src/main/kotlin/eu/kanade/tachiyomi/network/OkHttpExtensions.kt @@ -62,7 +62,7 @@ suspend fun Call.await(): Response { object : Callback { override fun onResponse(call: Call, response: Response) { if (!response.isSuccessful) { - continuation.resumeWithException(Exception("HTTP error ${response.code}")) + continuation.resumeWithException(HttpException(response.code)) return } @@ -94,7 +94,7 @@ fun Call.asObservableSuccess(): Observable { .doOnNext { response -> if (!response.isSuccessful) { response.close() - throw Exception("HTTP error ${response.code}") + throw HttpException(response.code) } } } @@ -136,3 +136,5 @@ inline fun Response.parseAs(): T { return json.decodeFromString(responseBody) } } + +class HttpException(val code: Int) : IllegalStateException("HTTP error $code") diff --git a/server/src/main/kotlin/eu/kanade/tachiyomi/network/interceptor/CloudflareInterceptor.kt b/server/src/main/kotlin/eu/kanade/tachiyomi/network/interceptor/CloudflareInterceptor.kt index 420ae0198..449819b70 100644 --- a/server/src/main/kotlin/eu/kanade/tachiyomi/network/interceptor/CloudflareInterceptor.kt +++ b/server/src/main/kotlin/eu/kanade/tachiyomi/network/interceptor/CloudflareInterceptor.kt @@ -1,8 +1,8 @@ package eu.kanade.tachiyomi.network.interceptor -import com.gargoylesoftware.htmlunit.BrowserVersion -import com.gargoylesoftware.htmlunit.WebClient -import com.gargoylesoftware.htmlunit.html.HtmlPage +import com.microsoft.playwright.BrowserType.LaunchOptions +import com.microsoft.playwright.Page +import com.microsoft.playwright.Playwright import eu.kanade.tachiyomi.network.NetworkHelper import mu.KotlinLogging import okhttp3.Cookie @@ -10,6 +10,8 @@ import okhttp3.HttpUrl import okhttp3.Interceptor import okhttp3.Request import okhttp3.Response +import suwayomi.tachidesk.server.ServerConfig +import suwayomi.tachidesk.server.serverConfig import uy.kohesive.injekt.injectLazy import java.io.IOException @@ -25,20 +27,22 @@ class CloudflareInterceptor : Interceptor { logger.trace { "CloudflareInterceptor is being used." } - val response = chain.proceed(originalRequest) + val originalResponse = chain.proceed(chain.request()) // Check if Cloudflare anti-bot is on - if (response.code != 503 || response.header("Server") !in SERVER_CHECK) { - return response + if (!(originalResponse.code in ERROR_CODES && originalResponse.header("Server") in SERVER_CHECK)) { + return originalResponse } logger.debug { "Cloudflare anti-bot is on, CloudflareInterceptor is kicking in..." } return try { - response.close() + originalResponse.close() network.cookies.remove(originalRequest.url.toUri()) - chain.proceed(resolveChallenge(response)) + val request = resolveWithWebView(originalRequest) + + chain.proceed(request) } catch (e: Exception) { // Because OkHttp's enqueue only handles IOExceptions, wrap the exception so that // we don't crash the entire app @@ -46,35 +50,52 @@ class CloudflareInterceptor : Interceptor { } } - private fun resolveChallenge(response: Response): Request { - val browserVersion = BrowserVersion.BrowserVersionBuilder(BrowserVersion.BEST_SUPPORTED) - .setUserAgent(response.request.header("User-Agent") ?: BrowserVersion.BEST_SUPPORTED.userAgent) - .build() - val convertedCookies = WebClient(browserVersion).use { webClient -> - webClient.options.isThrowExceptionOnFailingStatusCode = false - webClient.options.isThrowExceptionOnScriptError = false - webClient.getPage(response.request.url.toString()) - webClient.waitForBackgroundJavaScript(10000) - // Challenge solved, process cookies - webClient.cookieManager.cookies.filter { - // Only include Cloudflare cookies - it.name.startsWith("__cf") || it.name.startsWith("cf_") - }.map { - // Convert cookies -> OkHttp format - Cookie.Builder() - .domain(it.domain.removePrefix(".")) - .expiresAt(it.expires?.time ?: Long.MAX_VALUE) - .name(it.name) - .path(it.path) - .value(it.value).apply { - if (it.isHttpOnly) httpOnly() - if (it.isSecure) secure() - }.build() + private fun resolveWithWebView(originalRequest: Request): Request { + val url = originalRequest.url.toString() + + logger.debug { "resolveWithWebView($url)" } + + val cookies = Playwright.create().use { playwright -> + val browser = playwright.chromium().launch( + LaunchOptions() + .setHeadless(false) + .apply { + if (serverConfig.socksProxyEnabled) { + setProxy("socks5://${serverConfig.socksProxyHost}:${serverConfig.socksProxyPort}") + } + } + ) + val page = browser.newPage() + stealthInitScripts(page) + page.navigate(url) + + val res = cloudflareRetry(page) +// page.waitForClose(WaitForCloseOptions().setTimeout(0.0), {}) + if (res) { + val cookies = page.context().cookies() + +// val ua = page.evaluate("() => {return navigator.userAgent}") +// println(ua) + cookies.map { + // Convert cookies -> OkHttp format + Cookie.Builder() + .domain(it.domain.removePrefix(".")) + .expiresAt(it.expires?.times(1000)?.toLong() ?: Long.MAX_VALUE) + .name(it.name) + .path(it.path) + .value(it.value).apply { + if (it.httpOnly) httpOnly() + if (it.secure) secure() + }.build() + } + } else { + logger.debug { "cloudflare challenge failed" } + throw CloudflareBypassException() } } // Copy cookies to cookie store - convertedCookies.forEach { + cookies.forEach { network.cookies.addAll( HttpUrl.Builder() .scheme("http") @@ -85,26 +106,67 @@ class CloudflareInterceptor : Interceptor { } // Merge new and existing cookies for this request // Find the cookies that we need to merge into this request - val convertedForThisRequest = convertedCookies.filter { - it.matches(response.request.url) + val convertedForThisRequest = cookies.filter { + it.matches(originalRequest.url) } // Extract cookies from current request val existingCookies = Cookie.parseAll( - response.request.url, - response.request.headers + originalRequest.url, + originalRequest.headers ) // Filter out existing values of cookies that we are about to merge in val filteredExisting = existingCookies.filter { existing -> convertedForThisRequest.none { converted -> converted.name == existing.name } } + println("existing") + println(existingCookies.map { it.toString() }.joinToString("; ")) + println("converted") + println(convertedForThisRequest.map { it.toString() }.joinToString("; ")) val newCookies = filteredExisting + convertedForThisRequest - return response.request.newBuilder() - .header("Cookie", newCookies.map { it.toString() }.joinToString("; ")) + println(newCookies.map { it.toString() }.joinToString("; ")) + return originalRequest.newBuilder() + .header("Cookie", newCookies.map { "${it.name}=${it.value}" }.joinToString("; ")) .build() } + val cfScripts by lazy { + arrayOf( + ServerConfig::class.java.getResource("/cloudflare-js/canvas.fingerprinting.js")!!.readText(), + ServerConfig::class.java.getResource("/cloudflare-js/chrome.global.js")!!.readText(), + ServerConfig::class.java.getResource("/cloudflare-js/emulate.touch.js")!!.readText(), + ServerConfig::class.java.getResource("/cloudflare-js/navigator.permissions.js")!!.readText(), + ServerConfig::class.java.getResource("/cloudflare-js/navigator.webdriver.js")!!.readText(), + ServerConfig::class.java.getResource("/cloudflare-js/chrome.runtime.js")!!.readText(), + ServerConfig::class.java.getResource("/cloudflare-js/chrome.plugin.js")!!.readText() + ) + } + + // ref: https://github.com/vvanglro/cf-clearance/blob/44124a8f06d8d0ecf2bf558a027082ff88dab435/cf_clearance/stealth.py#L76 + fun stealthInitScripts(page: Page) { + for (script in cfScripts) { + page.addInitScript(script) + } + } + + // ref: https://github.com/vvanglro/cf-clearance/blob/44124a8f06d8d0ecf2bf558a027082ff88dab435/cf_clearance/retry.py#L21 + fun cloudflareRetry(page: Page, tries: Int = 30): Boolean { + for (i in 0 until tries) { + page.waitForTimeout(1000.0) + val success = try { + page.querySelector("#challenge-form") == null + } catch (e: Exception) { + false + } + if (success) return true + } + return false + } + companion object { + private val ERROR_CODES = listOf(403, 503) private val SERVER_CHECK = arrayOf("cloudflare-nginx", "cloudflare") private val COOKIE_NAMES = listOf("cf_clearance") } + + private class CloudflareBypassException : Exception() } From 770714b324907d14990d54f06ac5a04099610489 Mon Sep 17 00:00:00 2001 From: Aria Moradi Date: Sun, 4 Dec 2022 04:58:12 +0330 Subject: [PATCH 2/5] code clean up --- .../interceptor/CloudflareInterceptor.kt | 31 +++++++++++++------ 1 file changed, 21 insertions(+), 10 deletions(-) diff --git a/server/src/main/kotlin/eu/kanade/tachiyomi/network/interceptor/CloudflareInterceptor.kt b/server/src/main/kotlin/eu/kanade/tachiyomi/network/interceptor/CloudflareInterceptor.kt index 449819b70..f1ec36432 100644 --- a/server/src/main/kotlin/eu/kanade/tachiyomi/network/interceptor/CloudflareInterceptor.kt +++ b/server/src/main/kotlin/eu/kanade/tachiyomi/network/interceptor/CloudflareInterceptor.kt @@ -4,6 +4,7 @@ import com.microsoft.playwright.BrowserType.LaunchOptions import com.microsoft.playwright.Page import com.microsoft.playwright.Playwright import eu.kanade.tachiyomi.network.NetworkHelper +import eu.kanade.tachiyomi.network.interceptor.CFClearance.resolveWithWebView import mu.KotlinLogging import okhttp3.Cookie import okhttp3.HttpUrl @@ -15,7 +16,6 @@ import suwayomi.tachidesk.server.serverConfig import uy.kohesive.injekt.injectLazy import java.io.IOException -// from TachiWeb-Server class CloudflareInterceptor : Interceptor { private val logger = KotlinLogging.logger {} @@ -50,7 +50,24 @@ class CloudflareInterceptor : Interceptor { } } - private fun resolveWithWebView(originalRequest: Request): Request { + + companion object { + private val ERROR_CODES = listOf(403, 503) + private val SERVER_CHECK = arrayOf("cloudflare-nginx", "cloudflare") + private val COOKIE_NAMES = listOf("cf_clearance") + } +} + +/* + * This class is ported from https://github.com/vvanglro/cf-clearance + * The original code is licensed under Apache 2.0 +*/ +object CFClearance { + private val logger = KotlinLogging.logger {} + private val network: NetworkHelper by injectLazy() + + + fun resolveWithWebView(originalRequest: Request): Request { val url = originalRequest.url.toString() logger.debug { "resolveWithWebView($url)" } @@ -70,7 +87,7 @@ class CloudflareInterceptor : Interceptor { page.navigate(url) val res = cloudflareRetry(page) -// page.waitForClose(WaitForCloseOptions().setTimeout(0.0), {}) + if (res) { val cookies = page.context().cookies() @@ -162,11 +179,5 @@ class CloudflareInterceptor : Interceptor { return false } - companion object { - private val ERROR_CODES = listOf(403, 503) - private val SERVER_CHECK = arrayOf("cloudflare-nginx", "cloudflare") - private val COOKIE_NAMES = listOf("cf_clearance") - } - private class CloudflareBypassException : Exception() -} +} \ No newline at end of file From 13b2fb3af909199eb3ca31cc00421b94b26ed577 Mon Sep 17 00:00:00 2001 From: Aria Moradi Date: Sun, 4 Dec 2022 04:58:28 +0330 Subject: [PATCH 3/5] Forgot to commit these --- .../cloudflare-js/canvas.fingerprinting.js | 28 +++ .../resources/cloudflare-js/chrome.global.js | 52 +++++ .../resources/cloudflare-js/chrome.plugin.js | 203 ++++++++++++++++++ .../resources/cloudflare-js/chrome.runtime.js | 170 +++++++++++++++ .../resources/cloudflare-js/emulate.touch.js | 3 + .../cloudflare-js/navigator.permissions.js | 33 +++ .../cloudflare-js/navigator.webdriver.js | 5 + 7 files changed, 494 insertions(+) create mode 100644 server/src/main/resources/cloudflare-js/canvas.fingerprinting.js create mode 100644 server/src/main/resources/cloudflare-js/chrome.global.js create mode 100644 server/src/main/resources/cloudflare-js/chrome.plugin.js create mode 100644 server/src/main/resources/cloudflare-js/chrome.runtime.js create mode 100644 server/src/main/resources/cloudflare-js/emulate.touch.js create mode 100644 server/src/main/resources/cloudflare-js/navigator.permissions.js create mode 100644 server/src/main/resources/cloudflare-js/navigator.webdriver.js diff --git a/server/src/main/resources/cloudflare-js/canvas.fingerprinting.js b/server/src/main/resources/cloudflare-js/canvas.fingerprinting.js new file mode 100644 index 000000000..836175676 --- /dev/null +++ b/server/src/main/resources/cloudflare-js/canvas.fingerprinting.js @@ -0,0 +1,28 @@ +(function () { + const ORIGINAL_CANVAS = HTMLCanvasElement.prototype[name]; + Object.defineProperty(HTMLCanvasElement.prototype, name, { + "value": function () { + var shift = { + 'r': Math.floor(Math.random() * 10) - 5, + 'g': Math.floor(Math.random() * 10) - 5, + 'b': Math.floor(Math.random() * 10) - 5, + 'a': Math.floor(Math.random() * 10) - 5 + }; + var width = this.width, + height = this.height, + context = this.getContext("2d"); + var imageData = context.getImageData(0, 0, width, height); + for (var i = 0; i < height; i++) { + for (var j = 0; j < width; j++) { + var n = ((i * (width * 4)) + (j * 4)); + imageData.data[n + 0] = imageData.data[n + 0] + shift.r; + imageData.data[n + 1] = imageData.data[n + 1] + shift.g; + imageData.data[n + 2] = imageData.data[n + 2] + shift.b; + imageData.data[n + 3] = imageData.data[n + 3] + shift.a; + } + } + context.putImageData(imageData, 0, 0); + return ORIGINAL_CANVAS.apply(this, arguments); + } + }); +})(this); diff --git a/server/src/main/resources/cloudflare-js/chrome.global.js b/server/src/main/resources/cloudflare-js/chrome.global.js new file mode 100644 index 000000000..243a28991 --- /dev/null +++ b/server/src/main/resources/cloudflare-js/chrome.global.js @@ -0,0 +1,52 @@ +Object.defineProperty(window, 'chrome', { + value: new Proxy(window.chrome, { + has: (target, key) => true, + get: (target, key) => { + return { + app: { + isInstalled: false, + }, + webstore: { + onInstallStageChanged: {}, + onDownloadProgress: {}, + }, + runtime: { + PlatformOs: { + MAC: 'mac', + WIN: 'win', + ANDROID: 'android', + CROS: 'cros', + LINUX: 'linux', + OPENBSD: 'openbsd', + }, + PlatformArch: { + ARM: 'arm', + X86_32: 'x86-32', + X86_64: 'x86-64', + }, + PlatformNaclArch: { + ARM: 'arm', + X86_32: 'x86-32', + X86_64: 'x86-64', + }, + RequestUpdateCheckStatus: { + THROTTLED: 'throttled', + NO_UPDATE: 'no_update', + UPDATE_AVAILABLE: 'update_available', + }, + OnInstalledReason: { + INSTALL: 'install', + UPDATE: 'update', + CHROME_UPDATE: 'chrome_update', + SHARED_MODULE_UPDATE: 'shared_module_update', + }, + OnRestartRequiredReason: { + APP_UPDATE: 'app_update', + OS_UPDATE: 'os_update', + PERIODIC: 'periodic', + }, + }, + } + } + }) +}); diff --git a/server/src/main/resources/cloudflare-js/chrome.plugin.js b/server/src/main/resources/cloudflare-js/chrome.plugin.js new file mode 100644 index 000000000..e996d62c0 --- /dev/null +++ b/server/src/main/resources/cloudflare-js/chrome.plugin.js @@ -0,0 +1,203 @@ +(function () { + const plugin0 = Object.create(Plugin.prototype); + + const mimeType0 = Object.create(MimeType.prototype); + const mimeType1 = Object.create(MimeType.prototype); + Object.defineProperties(mimeType0, { + type: { + get: () => 'application/pdf', + }, + suffixes: { + get: () => 'pdf', + }, + }); + + Object.defineProperties(mimeType1, { + type: { + get: () => 'text/pdf', + }, + suffixes: { + get: () => 'pdf', + }, + }); + + Object.defineProperties(plugin0, { + name: { + get: () => 'Chrome PDF Viewer', + }, + description: { + get: () => 'Portable Document Format', + }, + 0: { + get: () => { + return mimeType0; + }, + }, + 1: { + get: () => { + return mimeType1; + }, + }, + length: { + get: () => 2, + }, + filename: { + get: () => 'internal-pdf-viewer', + }, + }); + + const plugin1 = Object.create(Plugin.prototype); + Object.defineProperties(plugin1, { + name: { + get: () => 'Chromium PDF Viewer', + }, + description: { + get: () => 'Portable Document Format', + }, + 0: { + get: () => { + return mimeType0; + }, + }, + 1: { + get: () => { + return mimeType1; + }, + }, + length: { + get: () => 2, + }, + filename: { + get: () => 'internal-pdf-viewer', + }, + }); + + const plugin2 = Object.create(Plugin.prototype); + Object.defineProperties(plugin2, { + name: { + get: () => 'Microsoft Edge PDF Viewer', + }, + description: { + get: () => 'Portable Document Format', + }, + 0: { + get: () => { + return mimeType0; + }, + }, + 1: { + get: () => { + return mimeType1; + }, + }, + length: { + get: () => 2, + }, + filename: { + get: () => 'internal-pdf-viewer', + }, + }); + + const plugin3 = Object.create(Plugin.prototype); + Object.defineProperties(plugin3, { + name: { + get: () => 'PDF Viewer', + }, + description: { + get: () => 'Portable Document Format', + }, + 0: { + get: () => { + return mimeType0; + }, + }, + 1: { + get: () => { + return mimeType1; + }, + }, + length: { + get: () => 2, + }, + filename: { + get: () => 'internal-pdf-viewer', + }, + }); + + const plugin4 = Object.create(Plugin.prototype); + Object.defineProperties(plugin4, { + name: { + get: () => 'WebKit built-in PDF', + }, + description: { + get: () => 'Portable Document Format', + }, + 0: { + get: () => { + return mimeType0; + }, + }, + 1: { + get: () => { + return mimeType1; + }, + }, + length: { + get: () => 2, + }, + filename: { + get: () => 'internal-pdf-viewer', + }, + }); + + const pluginArray = Object.create(PluginArray.prototype); + + pluginArray['0'] = plugin0; + pluginArray['1'] = plugin1; + pluginArray['2'] = plugin2; + pluginArray['3'] = plugin3; + pluginArray['4'] = plugin4; + + let refreshValue; + + Object.defineProperties(pluginArray, { + length: { + get: () => 5, + }, + item: { + value: (index) => { + if (index > 4294967295) { + index = index % 4294967296; + } + switch (index) { + case 0: + return plugin3; + case 1: + return plugin0; + case 2: + return plugin1; + case 3: + return plugin2; + case 4: + return plugin4; + default: + break; + } + }, + }, + refresh: { + get: () => { + return refreshValue; + }, + set: (value) => { + refreshValue = value; + }, + }, + }); + + Object.defineProperty(Object.getPrototypeOf(navigator), 'plugins', { + get: () => { + return pluginArray; + }, + }); +})(); diff --git a/server/src/main/resources/cloudflare-js/chrome.runtime.js b/server/src/main/resources/cloudflare-js/chrome.runtime.js new file mode 100644 index 000000000..b00befadd --- /dev/null +++ b/server/src/main/resources/cloudflare-js/chrome.runtime.js @@ -0,0 +1,170 @@ +(function () { + window.chrome = {}; + window.chrome.app = { + InstallState: { + DISABLED: 'disabled', + INSTALLED: 'installed', + NOT_INSTALLED: 'not_installed', + }, + RunningState: { + CANNOT_RUN: 'cannot_run', + READY_TO_RUN: 'ready_to_run', + RUNNING: 'running', + }, + getDetails: () => { + '[native code]'; + }, + getIsInstalled: () => { + '[native code]'; + }, + installState: () => { + '[native code]'; + }, + get isInstalled() { + return false; + }, + runningState: () => { + '[native code]'; + }, + }; + + window.chrome.runtime = { + OnInstalledReason: { + CHROME_UPDATE: 'chrome_update', + INSTALL: 'install', + SHARED_MODULE_UPDATE: 'shared_module_update', + UPDATE: 'update', + }, + OnRestartRequiredReason: { + APP_UPDATE: 'app_update', + OS_UPDATE: 'os_update', + PERIODIC: 'periodic', + }, + PlatformArch: { + ARM: 'arm', + ARM64: 'arm64', + MIPS: 'mips', + MIPS64: 'mips64', + X86_32: 'x86-32', + X86_64: 'x86-64', + }, + PlatformNaclArch: { + ARM: 'arm', + MIPS: 'mips', + MIPS64: 'mips64', + X86_32: 'x86-32', + X86_64: 'x86-64', + }, + PlatformOs: { + ANDROID: 'android', + CROS: 'cros', + FUCHSIA: 'fuchsia', + LINUX: 'linux', + MAC: 'mac', + OPENBSD: 'openbsd', + WIN: 'win', + }, + RequestUpdateCheckStatus: { + NO_UPDATE: 'no_update', + THROTTLED: 'throttled', + UPDATE_AVAILABLE: 'update_available', + }, + connect() { + '[native code]'; + }, + sendMessage() { + '[native code]'; + }, + id: undefined, + }; + + let startE = Date.now(); + window.chrome.csi = function () { + '[native code]'; + return { + startE: startE, + onloadT: startE + 281, + pageT: 3947.235, + tran: 15, + }; + }; + + window.chrome.loadTimes = function () { + '[native code]'; + return { + get requestTime() { + return startE / 1000; + }, + get startLoadTime() { + return startE / 1000; + }, + get commitLoadTime() { + return startE / 1000 + 0.324; + }, + get finishDocumentLoadTime() { + return startE / 1000 + 0.498; + }, + get finishLoadTime() { + return startE / 1000 + 0.534; + }, + get firstPaintTime() { + return startE / 1000 + 0.437; + }, + get firstPaintAfterLoadTime() { + return 0; + }, + get navigationType() { + return 'Other'; + }, + get wasFetchedViaSpdy() { + return true; + }, + get wasNpnNegotiated() { + return true; + }, + get npnNegotiatedProtocol() { + return 'h3'; + }, + get wasAlternateProtocolAvailable() { + return false; + }, + get connectionInfo() { + return 'h3'; + }, + }; + }; +})(); + +// Bypass OOPIF test +(function performance_memory() { + const jsHeapSizeLimitInt = 4294705152; + + const total_js_heap_size = 35244183; + const used_js_heap_size = [ + 17632315, 17632315, 17632315, 17634847, 17636091, 17636751, + ]; + + let counter = 0; + + let MemoryInfoProto = Object.getPrototypeOf(performance.memory); + Object.defineProperties(MemoryInfoProto, { + jsHeapSizeLimit: { + get: () => { + return jsHeapSizeLimitInt; + }, + }, + totalJSHeapSize: { + get: () => { + return total_js_heap_size; + }, + }, + usedJSHeapSize: { + get: () => { + if (counter > 5) { + counter = 0; + } + return used_js_heap_size[counter++]; + }, + }, + }); +})(); diff --git a/server/src/main/resources/cloudflare-js/emulate.touch.js b/server/src/main/resources/cloudflare-js/emulate.touch.js new file mode 100644 index 000000000..2e063974e --- /dev/null +++ b/server/src/main/resources/cloudflare-js/emulate.touch.js @@ -0,0 +1,3 @@ +Object.defineProperty(navigator, 'maxTouchPoints', { + get: () => 1 +}); diff --git a/server/src/main/resources/cloudflare-js/navigator.permissions.js b/server/src/main/resources/cloudflare-js/navigator.permissions.js new file mode 100644 index 000000000..c56a7a82b --- /dev/null +++ b/server/src/main/resources/cloudflare-js/navigator.permissions.js @@ -0,0 +1,33 @@ +// https://github.com/microlinkhq/browserless/blob/master/packages/goto/src/evasions/navigator-permissions.js +if (!window.Notification) { + window.Notification = { + permission: 'denied' + } +} +const originalQuery = window.navigator.permissions.query +window.navigator.permissions.__proto__.query = parameters => + parameters.name === 'notifications' + ? Promise.resolve({state: window.Notification.permission}) + : originalQuery(parameters) +const oldCall = Function.prototype.call + +function call() { + return oldCall.apply(this, arguments) +} + +Function.prototype.call = call +const nativeToStringFunctionString = Error.toString().replace(/Error/g, 'toString') +const oldToString = Function.prototype.toString + +function functionToString() { + if (this === window.navigator.permissions.query) { + return 'function query() { [native code] }' + } + if (this === functionToString) { + return nativeToStringFunctionString + } + return oldCall.call(oldToString, this) +} + +// eslint-disable-next-line +Function.prototype.toString = functionToString diff --git a/server/src/main/resources/cloudflare-js/navigator.webdriver.js b/server/src/main/resources/cloudflare-js/navigator.webdriver.js new file mode 100644 index 000000000..749d4e0c1 --- /dev/null +++ b/server/src/main/resources/cloudflare-js/navigator.webdriver.js @@ -0,0 +1,5 @@ +Object.defineProperty(Navigator.prototype, 'webdriver', { + get() { + return false; + }, +}); From c9fe6ed5d758446df424a34db8e4c444a0f7ce1d Mon Sep 17 00:00:00 2001 From: Syer10 Date: Sat, 3 Dec 2022 22:26:52 -0500 Subject: [PATCH 4/5] Get ResolveWithWebView working 1. Make sure to .use all closeable resources 2. Use 10 seconds instead of 1 second for waiting for cloudflare(this was the most probable issue) 3. Use Extension UA when possible 4. Minor cleanup of logging --- .../interceptor/CloudflareInterceptor.kt | 99 ++++++++++--------- .../test/kotlin/masstest/CloudFlareTest.kt | 58 +++++++++++ 2 files changed, 113 insertions(+), 44 deletions(-) create mode 100644 server/src/test/kotlin/masstest/CloudFlareTest.kt diff --git a/server/src/main/kotlin/eu/kanade/tachiyomi/network/interceptor/CloudflareInterceptor.kt b/server/src/main/kotlin/eu/kanade/tachiyomi/network/interceptor/CloudflareInterceptor.kt index f1ec36432..99db3a934 100644 --- a/server/src/main/kotlin/eu/kanade/tachiyomi/network/interceptor/CloudflareInterceptor.kt +++ b/server/src/main/kotlin/eu/kanade/tachiyomi/network/interceptor/CloudflareInterceptor.kt @@ -1,5 +1,6 @@ package eu.kanade.tachiyomi.network.interceptor +import com.microsoft.playwright.Browser import com.microsoft.playwright.BrowserType.LaunchOptions import com.microsoft.playwright.Page import com.microsoft.playwright.Playwright @@ -15,6 +16,8 @@ import suwayomi.tachidesk.server.ServerConfig import suwayomi.tachidesk.server.serverConfig import uy.kohesive.injekt.injectLazy import java.io.IOException +import kotlin.time.Duration.Companion.seconds +import kotlin.time.DurationUnit class CloudflareInterceptor : Interceptor { private val logger = KotlinLogging.logger {} @@ -50,7 +53,6 @@ class CloudflareInterceptor : Interceptor { } } - companion object { private val ERROR_CODES = listOf(403, 503) private val SERVER_CHECK = arrayOf("cloudflare-nginx", "cloudflare") @@ -66,14 +68,13 @@ object CFClearance { private val logger = KotlinLogging.logger {} private val network: NetworkHelper by injectLazy() - fun resolveWithWebView(originalRequest: Request): Request { val url = originalRequest.url.toString() logger.debug { "resolveWithWebView($url)" } val cookies = Playwright.create().use { playwright -> - val browser = playwright.chromium().launch( + playwright.chromium().launch( LaunchOptions() .setHeadless(false) .apply { @@ -81,44 +82,26 @@ object CFClearance { setProxy("socks5://${serverConfig.socksProxyHost}:${serverConfig.socksProxyPort}") } } - ) - val page = browser.newPage() - stealthInitScripts(page) - page.navigate(url) - - val res = cloudflareRetry(page) - - if (res) { - val cookies = page.context().cookies() - -// val ua = page.evaluate("() => {return navigator.userAgent}") -// println(ua) - cookies.map { - // Convert cookies -> OkHttp format - Cookie.Builder() - .domain(it.domain.removePrefix(".")) - .expiresAt(it.expires?.times(1000)?.toLong() ?: Long.MAX_VALUE) - .name(it.name) - .path(it.path) - .value(it.value).apply { - if (it.httpOnly) httpOnly() - if (it.secure) secure() - }.build() + ).use { browser -> + val userAgent = originalRequest.header("User-Agent") + if (userAgent != null) { + browser.newContext(Browser.NewContextOptions().setUserAgent(userAgent)).use { browserContext -> + browserContext.newPage().use { getCookies(it, url) } + } + } else { + browser.newPage().use { getCookies(it, url) } } - } else { - logger.debug { "cloudflare challenge failed" } - throw CloudflareBypassException() } } // Copy cookies to cookie store - cookies.forEach { + cookies.groupBy { it.domain }.forEach { (domain, cookies) -> network.cookies.addAll( - HttpUrl.Builder() + url = HttpUrl.Builder() .scheme("http") - .host(it.domain) + .host(domain) .build(), - listOf(it) + cookies = cookies ) } // Merge new and existing cookies for this request @@ -135,18 +118,45 @@ object CFClearance { val filteredExisting = existingCookies.filter { existing -> convertedForThisRequest.none { converted -> converted.name == existing.name } } - println("existing") - println(existingCookies.map { it.toString() }.joinToString("; ")) - println("converted") - println(convertedForThisRequest.map { it.toString() }.joinToString("; ")) + logger.trace { "Existing cookies" } + logger.trace { existingCookies.joinToString("; ") } val newCookies = filteredExisting + convertedForThisRequest - println(newCookies.map { it.toString() }.joinToString("; ")) + logger.trace { "New cookies" } + logger.trace { newCookies.joinToString("; ") } return originalRequest.newBuilder() - .header("Cookie", newCookies.map { "${it.name}=${it.value}" }.joinToString("; ")) + .header("Cookie", newCookies.joinToString("; ") { "${it.name}=${it.value}" }) .build() } - val cfScripts by lazy { + private fun getCookies(page: Page, url: String): List { + stealthInitScripts(page) + page.navigate(url) + val res = cloudflareRetry(page) + + return if (res) { + val cookies = page.context().cookies() + + /*val ua = page.evaluate("() => {return navigator.userAgent}") + println(ua)*/ + cookies.map { + // Convert cookies -> OkHttp format + Cookie.Builder() + .domain(it.domain.removePrefix(".")) + .expiresAt(it.expires?.times(1000)?.toLong() ?: Long.MAX_VALUE) + .name(it.name) + .path(it.path) + .value(it.value).apply { + if (it.httpOnly) httpOnly() + if (it.secure) secure() + }.build() + } + } else { + logger.debug { "cloudflare challenge failed" } + throw CloudflareBypassException() + } + } + + private val cfScripts by lazy { arrayOf( ServerConfig::class.java.getResource("/cloudflare-js/canvas.fingerprinting.js")!!.readText(), ServerConfig::class.java.getResource("/cloudflare-js/chrome.global.js")!!.readText(), @@ -159,19 +169,20 @@ object CFClearance { } // ref: https://github.com/vvanglro/cf-clearance/blob/44124a8f06d8d0ecf2bf558a027082ff88dab435/cf_clearance/stealth.py#L76 - fun stealthInitScripts(page: Page) { + private fun stealthInitScripts(page: Page) { for (script in cfScripts) { page.addInitScript(script) } } // ref: https://github.com/vvanglro/cf-clearance/blob/44124a8f06d8d0ecf2bf558a027082ff88dab435/cf_clearance/retry.py#L21 - fun cloudflareRetry(page: Page, tries: Int = 30): Boolean { + private fun cloudflareRetry(page: Page, tries: Int = 3): Boolean { for (i in 0 until tries) { - page.waitForTimeout(1000.0) + page.waitForTimeout(10.seconds.toDouble(DurationUnit.MILLISECONDS)) val success = try { page.querySelector("#challenge-form") == null } catch (e: Exception) { + logger.debug(e) { "Error?" } false } if (success) return true @@ -180,4 +191,4 @@ object CFClearance { } private class CloudflareBypassException : Exception() -} \ No newline at end of file +} diff --git a/server/src/test/kotlin/masstest/CloudFlareTest.kt b/server/src/test/kotlin/masstest/CloudFlareTest.kt new file mode 100644 index 000000000..7c7687371 --- /dev/null +++ b/server/src/test/kotlin/masstest/CloudFlareTest.kt @@ -0,0 +1,58 @@ +package masstest + +import eu.kanade.tachiyomi.source.online.HttpSource +import kotlinx.coroutines.runBlocking +import kotlinx.coroutines.test.runTest +import mu.KotlinLogging +import org.junit.jupiter.api.BeforeAll +import org.junit.jupiter.api.Test +import org.junit.jupiter.api.TestInstance +import suwayomi.tachidesk.manga.impl.Source +import suwayomi.tachidesk.manga.impl.extension.Extension +import suwayomi.tachidesk.manga.impl.extension.ExtensionsList +import suwayomi.tachidesk.manga.impl.util.lang.awaitSingle +import suwayomi.tachidesk.manga.impl.util.source.GetCatalogueSource +import suwayomi.tachidesk.server.applicationSetup +import suwayomi.tachidesk.test.BASE_PATH +import suwayomi.tachidesk.test.setLoggingEnabled +import xyz.nulldev.ts.config.CONFIG_PREFIX +import java.io.File + +@TestInstance(TestInstance.Lifecycle.PER_CLASS) +class CloudFlareTest { + lateinit var nhentai: HttpSource + + @BeforeAll + fun setup() { + val dataRoot = File(BASE_PATH).absolutePath + System.setProperty("$CONFIG_PREFIX.server.rootDir", dataRoot) + applicationSetup() + setLoggingEnabled(false) + + runBlocking { + val extensions = ExtensionsList.getExtensionList() + with(extensions.first { it.name == "NHentai" }) { + if (!installed) { + Extension.installExtension(pkgName) + } else if (hasUpdate) { + Extension.updateExtension(pkgName) + } + Unit + } + + nhentai = Source.getSourceList() + .firstNotNullOf { it.id.toLong().takeIf { it == 3122156392225024195L } } + .let(GetCatalogueSource::getCatalogueSourceOrNull) as HttpSource + } + setLoggingEnabled(true) + } + + private val logger = KotlinLogging.logger {} + + @Test + fun `test nhentai browse`() = runTest { + assert(nhentai.fetchPopularManga(1).awaitSingle().mangas.isNotEmpty()) { + "NHentai results were empty" + } + } +} From 4c016bf2edf360e5593a84fd93f58dc9fd5b9a09 Mon Sep 17 00:00:00 2001 From: Aria Moradi Date: Sun, 4 Dec 2022 12:07:18 +0330 Subject: [PATCH 5/5] rewrite and refactor --- .../interceptor/CloudflareInterceptor.kt | 34 +++++++++++-------- 1 file changed, 20 insertions(+), 14 deletions(-) diff --git a/server/src/main/kotlin/eu/kanade/tachiyomi/network/interceptor/CloudflareInterceptor.kt b/server/src/main/kotlin/eu/kanade/tachiyomi/network/interceptor/CloudflareInterceptor.kt index 99db3a934..ddc8cb7dd 100644 --- a/server/src/main/kotlin/eu/kanade/tachiyomi/network/interceptor/CloudflareInterceptor.kt +++ b/server/src/main/kotlin/eu/kanade/tachiyomi/network/interceptor/CloudflareInterceptor.kt @@ -129,17 +129,20 @@ object CFClearance { } private fun getCookies(page: Page, url: String): List { - stealthInitScripts(page) + applyStealthInitScripts(page) page.navigate(url) - val res = cloudflareRetry(page) + val challengeResolved = waitForChallengeResolve(page) - return if (res) { + return if (challengeResolved) { val cookies = page.context().cookies() - /*val ua = page.evaluate("() => {return navigator.userAgent}") - println(ua)*/ + logger.debug { + val userAgent = page.evaluate("() => {return navigator.userAgent}") + "Playwright User-Agent is $userAgent" + } + + // Convert PlayWright cookies to OkHttp cookies cookies.map { - // Convert cookies -> OkHttp format Cookie.Builder() .domain(it.domain.removePrefix(".")) .expiresAt(it.expires?.times(1000)?.toLong() ?: Long.MAX_VALUE) @@ -151,12 +154,13 @@ object CFClearance { }.build() } } else { - logger.debug { "cloudflare challenge failed" } + logger.debug { "Cloudflare challenge failed to resolve" } throw CloudflareBypassException() } } - private val cfScripts by lazy { + // ref: https://github.com/vvanglro/cf-clearance/blob/44124a8f06d8d0ecf2bf558a027082ff88dab435/cf_clearance/stealth.py#L18 + private val stealthInitScripts by lazy { arrayOf( ServerConfig::class.java.getResource("/cloudflare-js/canvas.fingerprinting.js")!!.readText(), ServerConfig::class.java.getResource("/cloudflare-js/chrome.global.js")!!.readText(), @@ -169,20 +173,22 @@ object CFClearance { } // ref: https://github.com/vvanglro/cf-clearance/blob/44124a8f06d8d0ecf2bf558a027082ff88dab435/cf_clearance/stealth.py#L76 - private fun stealthInitScripts(page: Page) { - for (script in cfScripts) { + private fun applyStealthInitScripts(page: Page) { + for (script in stealthInitScripts) { page.addInitScript(script) } } // ref: https://github.com/vvanglro/cf-clearance/blob/44124a8f06d8d0ecf2bf558a027082ff88dab435/cf_clearance/retry.py#L21 - private fun cloudflareRetry(page: Page, tries: Int = 3): Boolean { - for (i in 0 until tries) { - page.waitForTimeout(10.seconds.toDouble(DurationUnit.MILLISECONDS)) + private fun waitForChallengeResolve(page: Page): Boolean { + // sometimes the user has to solve the captcha challenge manually, potentially wait a long time + val timeoutSeconds = 120 + repeat(timeoutSeconds) { + page.waitForTimeout(1.seconds.toDouble(DurationUnit.MILLISECONDS)) val success = try { page.querySelector("#challenge-form") == null } catch (e: Exception) { - logger.debug(e) { "Error?" } + logger.debug(e) { "query Error" } false } if (success) return true