From c038ba78831865c23be964abc019aa2e7162c47c Mon Sep 17 00:00:00 2001 From: Yuri Schimke Date: Sun, 24 Nov 2024 11:25:05 +0000 Subject: [PATCH 01/11] Embed the public suffix database list directly inside a class --- okhttp/build.gradle.kts | 43 ++++++- .../publicsuffix/PublicSuffixDatabase.kt | 112 +---------------- .../internal/publicsuffix/PublicSuffixList.kt | 26 ++++ .../publicsuffix/EmbeddedPublicSuffixList.kt | 35 ++++++ .../resources/META-INF/proguard/okhttp3.pro | 4 - .../publicsuffix/PublicSuffixDatabaseTest.kt | 21 ++-- .../publicsuffix/PublicSuffixListGenerator.kt | 9 +- .../publicsuffix/ResourcePublicSuffixList.kt | 114 ++++++++++++++++++ .../okhttp3/internal/publicsuffix/NOTICE | 0 .../publicsuffix/PublicSuffixDatabase.gz | Bin 10 files changed, 239 insertions(+), 125 deletions(-) create mode 100644 okhttp/src/main/kotlin/okhttp3/internal/publicsuffix/PublicSuffixList.kt create mode 100644 okhttp/src/main/kotlinTemplates/okhttp3/internal/publicsuffix/EmbeddedPublicSuffixList.kt create mode 100644 okhttp/src/test/java/okhttp3/internal/publicsuffix/ResourcePublicSuffixList.kt rename okhttp/src/{main => test}/resources/okhttp3/internal/publicsuffix/NOTICE (100%) rename okhttp/src/{main => test}/resources/okhttp3/internal/publicsuffix/PublicSuffixDatabase.gz (100%) diff --git a/okhttp/build.gradle.kts b/okhttp/build.gradle.kts index 5c3f2fb4e8df..4a973edd6dff 100644 --- a/okhttp/build.gradle.kts +++ b/okhttp/build.gradle.kts @@ -1,5 +1,8 @@ import com.vanniktech.maven.publish.JavadocJar import com.vanniktech.maven.publish.KotlinJvm +import java.io.DataInputStream +import java.io.FileInputStream +import java.util.zip.GZIPInputStream plugins { kotlin("jvm") @@ -9,12 +12,48 @@ plugins { id("binary-compatibility-validator") } -// Build & use okhttp3/internal/-InternalVersion.kt +fun ByteArray.toByteArrayExpression(): String { + return buildString { + append("byteArrayOf(") + this@toByteArrayExpression.forEach { + append(it) + append(", ") + } + append(")") + } +} + val copyKotlinTemplates = tasks.register("copyKotlinTemplates") { from("src/main/kotlinTemplates") into("$buildDir/generated/sources/kotlinTemplates") - expand("projectVersion" to project.version) + filteringCharset = Charsets.UTF_8.toString() + + // TODO replace with KotlinPoet? + val databaseGz = project.file("src/test/resources/okhttp3/internal/publicsuffix/PublicSuffixDatabase.gz") + println("loading $databaseGz") + val (publicSuffixListBytes, publicSuffixListExceptionBytes) = DataInputStream(GZIPInputStream(FileInputStream(databaseGz))).use { + val totalBytes = it.readInt() + val publicSuffixListBytes = it.readNBytes(totalBytes) + println("read $totalBytes") + + val totalExceptionBytes = it.readInt() + val publicSuffixExceptionListBytes = it.readNBytes(totalExceptionBytes) + println("read $totalExceptionBytes") + + Pair(publicSuffixListBytes.toByteArrayExpression(), publicSuffixExceptionListBytes.toByteArrayExpression()) + } + + println(publicSuffixListBytes.substring(0, 10)) + + expand( + // Build & use okhttp3/internal/-InternalVersion.kt + "projectVersion" to project.version, + + // Build okhttp3/internal/publicsuffix/EmbeddedPublicSuffixList.kt + "publicSuffixListBytes" to publicSuffixListBytes, + "publicSuffixListExceptionBytes" to publicSuffixListExceptionBytes + ) } // Build & use okhttp3/internal/idn/IdnaMappingTableInstance.kt diff --git a/okhttp/src/main/kotlin/okhttp3/internal/publicsuffix/PublicSuffixDatabase.kt b/okhttp/src/main/kotlin/okhttp3/internal/publicsuffix/PublicSuffixDatabase.kt index fef4fd01df79..f6e33ac28ae9 100644 --- a/okhttp/src/main/kotlin/okhttp3/internal/publicsuffix/PublicSuffixDatabase.kt +++ b/okhttp/src/main/kotlin/okhttp3/internal/publicsuffix/PublicSuffixDatabase.kt @@ -15,18 +15,8 @@ */ package okhttp3.internal.publicsuffix -import java.io.IOException -import java.io.InterruptedIOException import java.net.IDN -import java.util.concurrent.CountDownLatch -import java.util.concurrent.atomic.AtomicBoolean import okhttp3.internal.and -import okhttp3.internal.platform.Platform -import okio.FileSystem -import okio.GzipSource -import okio.Path -import okio.Path.Companion.toPath -import okio.buffer /** * A database of public suffixes provided by [publicsuffix.org][publicsuffix_org]. @@ -34,21 +24,8 @@ import okio.buffer * [publicsuffix_org]: https://publicsuffix.org/ */ class PublicSuffixDatabase internal constructor( - val path: Path = PUBLIC_SUFFIX_RESOURCE, - val fileSystem: FileSystem = FileSystem.RESOURCES, + private val publicSuffixList: PublicSuffixList ) { - /** True after we've attempted to read the list for the first time. */ - private val listRead = AtomicBoolean(false) - - /** Used for concurrent threads reading the list for the first time. */ - private val readCompleteLatch = CountDownLatch(1) - - // The lists are held as a large array of UTF-8 bytes. This is to avoid allocating lots of strings - // that will likely never be used. Each rule is separated by '\n'. Please see the - // PublicSuffixListGenerator class for how these lists are generated. - // Guarded by this. - private lateinit var publicSuffixListBytes: ByteArray - private lateinit var publicSuffixExceptionListBytes: ByteArray /** * Returns the effective top-level domain plus one (eTLD+1) by referencing the public suffix list. @@ -101,20 +78,7 @@ class PublicSuffixDatabase internal constructor( } private fun findMatchingRule(domainLabels: List): List { - if (!listRead.get() && listRead.compareAndSet(false, true)) { - readTheListUninterruptibly() - } else { - try { - readCompleteLatch.await() - } catch (_: InterruptedException) { - Thread.currentThread().interrupt() // Retain interrupted status. - } - } - - check(::publicSuffixListBytes.isInitialized) { - // May have failed with an IOException - "Unable to load $PUBLIC_SUFFIX_RESOURCE resource from the classpath." - } + publicSuffixList.ensureLoaded() // Break apart the domain into UTF-8 labels, i.e. foo.bar.com turns into [foo, bar, com]. val domainLabelsUtf8Bytes = Array(domainLabels.size) { i -> domainLabels[i].toByteArray() } @@ -123,7 +87,7 @@ class PublicSuffixDatabase internal constructor( // will look like: [foo, bar, com], [bar, com], [com]. The longest matching rule wins. var exactMatch: String? = null for (i in domainLabelsUtf8Bytes.indices) { - val rule = publicSuffixListBytes.binarySearch(domainLabelsUtf8Bytes, i) + val rule = publicSuffixList.bytes.binarySearch(domainLabelsUtf8Bytes, i) if (rule != null) { exactMatch = rule break @@ -140,7 +104,7 @@ class PublicSuffixDatabase internal constructor( val labelsWithWildcard = domainLabelsUtf8Bytes.clone() for (labelIndex in 0 until labelsWithWildcard.size - 1) { labelsWithWildcard[labelIndex] = WILDCARD_LABEL - val rule = publicSuffixListBytes.binarySearch(labelsWithWildcard, labelIndex) + val rule = publicSuffixList.bytes.binarySearch(labelsWithWildcard, labelIndex) if (rule != null) { wildcardMatch = rule break @@ -153,7 +117,7 @@ class PublicSuffixDatabase internal constructor( if (wildcardMatch != null) { for (labelIndex in 0 until domainLabelsUtf8Bytes.size - 1) { val rule = - publicSuffixExceptionListBytes.binarySearch( + publicSuffixList.exceptionBytes.binarySearch( domainLabelsUtf8Bytes, labelIndex, ) @@ -182,77 +146,13 @@ class PublicSuffixDatabase internal constructor( } } - /** - * Reads the public suffix list treating the operation as uninterruptible. We always want to read - * the list otherwise we'll be left in a bad state. If the thread was interrupted prior to this - * operation, it will be re-interrupted after the list is read. - */ - private fun readTheListUninterruptibly() { - var interrupted = false - try { - while (true) { - try { - readTheList() - return - } catch (_: InterruptedIOException) { - Thread.interrupted() // Temporarily clear the interrupted state. - interrupted = true - } catch (e: IOException) { - Platform.get().log("Failed to read public suffix list", Platform.WARN, e) - return - } - } - } finally { - if (interrupted) { - Thread.currentThread().interrupt() // Retain interrupted status. - } - } - } - - @Throws(IOException::class) - private fun readTheList() { - var publicSuffixListBytes: ByteArray? - var publicSuffixExceptionListBytes: ByteArray? - - try { - GzipSource(fileSystem.source(path)).buffer().use { bufferedSource -> - val totalBytes = bufferedSource.readInt() - publicSuffixListBytes = bufferedSource.readByteArray(totalBytes.toLong()) - - val totalExceptionBytes = bufferedSource.readInt() - publicSuffixExceptionListBytes = bufferedSource.readByteArray(totalExceptionBytes.toLong()) - } - - synchronized(this) { - this.publicSuffixListBytes = publicSuffixListBytes!! - this.publicSuffixExceptionListBytes = publicSuffixExceptionListBytes!! - } - } finally { - readCompleteLatch.countDown() - } - } - - /** Visible for testing. */ - fun setListBytes( - publicSuffixListBytes: ByteArray, - publicSuffixExceptionListBytes: ByteArray, - ) { - this.publicSuffixListBytes = publicSuffixListBytes - this.publicSuffixExceptionListBytes = publicSuffixExceptionListBytes - listRead.set(true) - readCompleteLatch.countDown() - } - companion object { - @JvmField - val PUBLIC_SUFFIX_RESOURCE = "/okhttp3/internal/publicsuffix/${PublicSuffixDatabase::class.java.simpleName}.gz".toPath() - private val WILDCARD_LABEL = byteArrayOf('*'.code.toByte()) private val PREVAILING_RULE = listOf("*") private const val EXCEPTION_MARKER = '!' - private val instance = PublicSuffixDatabase() + private val instance = PublicSuffixDatabase(EmbeddedPublicSuffixList) fun get(): PublicSuffixDatabase { return instance diff --git a/okhttp/src/main/kotlin/okhttp3/internal/publicsuffix/PublicSuffixList.kt b/okhttp/src/main/kotlin/okhttp3/internal/publicsuffix/PublicSuffixList.kt new file mode 100644 index 000000000000..97fdc792394c --- /dev/null +++ b/okhttp/src/main/kotlin/okhttp3/internal/publicsuffix/PublicSuffixList.kt @@ -0,0 +1,26 @@ +/* + * Copyright (C) 2024 Block, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package okhttp3.internal.publicsuffix + +/** + * Basic I/O for the PublicSuffixDatabase.gz. + */ +internal interface PublicSuffixList { + fun ensureLoaded() + + val bytes: ByteArray + val exceptionBytes: ByteArray +} diff --git a/okhttp/src/main/kotlinTemplates/okhttp3/internal/publicsuffix/EmbeddedPublicSuffixList.kt b/okhttp/src/main/kotlinTemplates/okhttp3/internal/publicsuffix/EmbeddedPublicSuffixList.kt new file mode 100644 index 000000000000..03caa9994d92 --- /dev/null +++ b/okhttp/src/main/kotlinTemplates/okhttp3/internal/publicsuffix/EmbeddedPublicSuffixList.kt @@ -0,0 +1,35 @@ +/* + * Copyright (C) 2024 Block, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package okhttp3.internal.publicsuffix + +//Note that PublicSuffixDatabase.gz is compiled from The Public Suffix List: +//https://publicsuffix.org/list/public_suffix_list.dat +// +//It is subject to the terms of the Mozilla Public License, v. 2.0: +//https://mozilla.org/MPL/2.0/ + +/** + * A implementation of I/O for PublicSuffixDatabase.gz by directly encoding + * the relevant byte arrays in a class file. + */ +internal object EmbeddedPublicSuffixList: PublicSuffixList { + override fun ensureLoaded() { + } + + override val bytes: ByteArray = $publicSuffixListBytes + + override val exceptionBytes: ByteArray = $publicSuffixListExceptionBytes +} diff --git a/okhttp/src/main/resources/META-INF/proguard/okhttp3.pro b/okhttp/src/main/resources/META-INF/proguard/okhttp3.pro index 74fc3168014a..280a52749be2 100644 --- a/okhttp/src/main/resources/META-INF/proguard/okhttp3.pro +++ b/okhttp/src/main/resources/META-INF/proguard/okhttp3.pro @@ -1,10 +1,6 @@ # JSR 305 annotations are for embedding nullability information. -dontwarn javax.annotation.** -# A resource is loaded with a relative path so the package of this class must be preserved. --keeppackagenames okhttp3.internal.publicsuffix.* --adaptresourcefilenames okhttp3/internal/publicsuffix/PublicSuffixDatabase.gz - # Animal Sniffer compileOnly dependency to ensure APIs are compatible with older versions of Java. -dontwarn org.codehaus.mojo.animal_sniffer.* diff --git a/okhttp/src/test/java/okhttp3/internal/publicsuffix/PublicSuffixDatabaseTest.kt b/okhttp/src/test/java/okhttp3/internal/publicsuffix/PublicSuffixDatabaseTest.kt index e264cb34e36c..7ecb0d543f96 100644 --- a/okhttp/src/test/java/okhttp3/internal/publicsuffix/PublicSuffixDatabaseTest.kt +++ b/okhttp/src/test/java/okhttp3/internal/publicsuffix/PublicSuffixDatabaseTest.kt @@ -31,7 +31,8 @@ import okio.use import org.junit.jupiter.api.Test class PublicSuffixDatabaseTest { - private val publicSuffixDatabase = PublicSuffixDatabase() + private val list = ResourcePublicSuffixList() + private val publicSuffixDatabase = PublicSuffixDatabase(list) @Test fun longestMatchWins() { val buffer = @@ -39,7 +40,7 @@ class PublicSuffixDatabaseTest { .writeUtf8("com\n") .writeUtf8("my.square.com\n") .writeUtf8("square.com\n") - publicSuffixDatabase.setListBytes(buffer.readByteArray(), byteArrayOf()) + list.setListBytes(buffer.readByteArray(), byteArrayOf()) assertThat(publicSuffixDatabase.getEffectiveTldPlusOne("example.com")) .isEqualTo("example.com") assertThat(publicSuffixDatabase.getEffectiveTldPlusOne("foo.example.com")) @@ -56,7 +57,7 @@ class PublicSuffixDatabaseTest { .writeUtf8("*.square.com\n") .writeUtf8("com\n") .writeUtf8("example.com\n") - publicSuffixDatabase.setListBytes(buffer.readByteArray(), byteArrayOf()) + list.setListBytes(buffer.readByteArray(), byteArrayOf()) assertThat(publicSuffixDatabase.getEffectiveTldPlusOne("my.square.com")).isNull() assertThat(publicSuffixDatabase.getEffectiveTldPlusOne("foo.my.square.com")) .isEqualTo("foo.my.square.com") @@ -70,7 +71,7 @@ class PublicSuffixDatabaseTest { .writeUtf8("bbb\n") .writeUtf8("ddd\n") .writeUtf8("fff\n") - publicSuffixDatabase.setListBytes(buffer.readByteArray(), byteArrayOf()) + list.setListBytes(buffer.readByteArray(), byteArrayOf()) assertThat(publicSuffixDatabase.getEffectiveTldPlusOne("aaa")).isNull() assertThat(publicSuffixDatabase.getEffectiveTldPlusOne("ggg")).isNull() assertThat(publicSuffixDatabase.getEffectiveTldPlusOne("ccc")).isNull() @@ -87,7 +88,7 @@ class PublicSuffixDatabaseTest { .writeUtf8("*.square.jp\n") .writeUtf8("example.com\n") .writeUtf8("square.com\n") - publicSuffixDatabase.setListBytes(buffer.readByteArray(), exception.readByteArray()) + list.setListBytes(buffer.readByteArray(), exception.readByteArray()) assertThat(publicSuffixDatabase.getEffectiveTldPlusOne("my.square.jp")) .isEqualTo("my.square.jp") assertThat(publicSuffixDatabase.getEffectiveTldPlusOne("foo.my.square.jp")) @@ -105,14 +106,14 @@ class PublicSuffixDatabaseTest { .writeUtf8("*.square.jp\n") .writeUtf8("example.com\n") .writeUtf8("square.com\n") - publicSuffixDatabase.setListBytes(buffer.readByteArray(), exception.readByteArray()) + list.setListBytes(buffer.readByteArray(), exception.readByteArray()) assertThat(publicSuffixDatabase.getEffectiveTldPlusOne("example.com")).isNull() assertThat(publicSuffixDatabase.getEffectiveTldPlusOne("foo.square.jp")).isNull() } @Test fun allPublicSuffixes() { val buffer = Buffer() - FileSystem.RESOURCES.source(PublicSuffixDatabase.PUBLIC_SUFFIX_RESOURCE).use { resource -> + FileSystem.RESOURCES.source(ResourcePublicSuffixList.PUBLIC_SUFFIX_RESOURCE).use { resource -> GzipSource(resource).buffer().use { source -> val length = source.readInt() buffer.write(source, length.toLong()) @@ -132,7 +133,7 @@ class PublicSuffixDatabaseTest { @Test fun publicSuffixExceptions() { val buffer = Buffer() - FileSystem.RESOURCES.source(PublicSuffixDatabase.PUBLIC_SUFFIX_RESOURCE).use { resource -> + FileSystem.RESOURCES.source(ResourcePublicSuffixList.PUBLIC_SUFFIX_RESOURCE).use { resource -> GzipSource(resource).buffer().use { source -> var length = source.readInt() source.skip(length.toLong()) @@ -163,7 +164,9 @@ class PublicSuffixDatabaseTest { @Test fun secondReadFailsSameAsFirst() { val badPublicSuffixDatabase = PublicSuffixDatabase( - path = "/xxx.gz".toPath(), + ResourcePublicSuffixList( + path = "/xxx.gz".toPath(), + ) ) lateinit var firstFailure: Exception assertFailsWith { diff --git a/okhttp/src/test/java/okhttp3/internal/publicsuffix/PublicSuffixListGenerator.kt b/okhttp/src/test/java/okhttp3/internal/publicsuffix/PublicSuffixListGenerator.kt index 8d77d56d4a63..26a69ac700e3 100644 --- a/okhttp/src/test/java/okhttp3/internal/publicsuffix/PublicSuffixListGenerator.kt +++ b/okhttp/src/test/java/okhttp3/internal/publicsuffix/PublicSuffixListGenerator.kt @@ -13,17 +13,20 @@ * See the License for the specific language governing permissions and * limitations under the License. */ +@file:OptIn(ExperimentalCoroutinesApi::class) + package okhttp3.internal.publicsuffix import java.util.SortedSet import java.util.TreeSet import kotlinx.coroutines.Dispatchers +import kotlinx.coroutines.ExperimentalCoroutinesApi import kotlinx.coroutines.withContext import okhttp3.HttpUrl.Companion.toHttpUrl import okhttp3.OkHttpClient import okhttp3.Request import okhttp3.coroutines.executeAsync -import okhttp3.internal.publicsuffix.PublicSuffixDatabase.Companion.PUBLIC_SUFFIX_RESOURCE +import okhttp3.internal.publicsuffix.ResourcePublicSuffixList.Companion.PUBLIC_SUFFIX_RESOURCE import okio.BufferedSink import okio.ByteString import okio.ByteString.Companion.encodeUtf8 @@ -49,15 +52,13 @@ class PublicSuffixListGenerator( val fileSystem: FileSystem = FileSystem.SYSTEM, val client: OkHttpClient = OkHttpClient(), ) { - private val resources = projectRoot / "okhttp/src/main/resources/okhttp3/internal/publicsuffix" private val testResources = projectRoot / "okhttp/src/test/resources/okhttp3/internal/publicsuffix" private val publicSuffixListDotDat = testResources / "public_suffix_list.dat" - private val outputFile = resources / PUBLIC_SUFFIX_RESOURCE + private val outputFile = testResources / PUBLIC_SUFFIX_RESOURCE val request = Request("https://publicsuffix.org/list/public_suffix_list.dat".toHttpUrl()) suspend fun import() { - check(fileSystem.metadata(resources).isDirectory) check(fileSystem.metadata(testResources).isDirectory) updateLocalFile() diff --git a/okhttp/src/test/java/okhttp3/internal/publicsuffix/ResourcePublicSuffixList.kt b/okhttp/src/test/java/okhttp3/internal/publicsuffix/ResourcePublicSuffixList.kt new file mode 100644 index 000000000000..0e247f90f72b --- /dev/null +++ b/okhttp/src/test/java/okhttp3/internal/publicsuffix/ResourcePublicSuffixList.kt @@ -0,0 +1,114 @@ +package okhttp3.internal.publicsuffix + +import java.io.IOException +import java.io.InterruptedIOException +import java.util.concurrent.CountDownLatch +import java.util.concurrent.atomic.AtomicBoolean +import okhttp3.internal.platform.Platform +import okio.FileSystem +import okio.GzipSource +import okio.Path +import okio.Path.Companion.toPath +import okio.buffer + +internal class ResourcePublicSuffixList( + val path: Path = PUBLIC_SUFFIX_RESOURCE, + val fileSystem: FileSystem = FileSystem.RESOURCES, +) : PublicSuffixList { + /** True after we've attempted to read the list for the first time. */ + private val listRead = AtomicBoolean(false) + + /** Used for concurrent threads reading the list for the first time. */ + private val readCompleteLatch = CountDownLatch(1) + + // The lists are held as a large array of UTF-8 bytes. This is to avoid allocating lots of strings + // that will likely never be used. Each rule is separated by '\n'. Please see the + // PublicSuffixListGenerator class for how these lists are generated. + // Guarded by this. + override lateinit var bytes: ByteArray + override lateinit var exceptionBytes: ByteArray + + @Throws(IOException::class) + private fun readTheList() { + var publicSuffixListBytes: ByteArray? + var publicSuffixExceptionListBytes: ByteArray? + + try { + GzipSource(fileSystem.source(path)).buffer().use { bufferedSource -> + val totalBytes = bufferedSource.readInt() + publicSuffixListBytes = bufferedSource.readByteArray(totalBytes.toLong()) + + val totalExceptionBytes = bufferedSource.readInt() + publicSuffixExceptionListBytes = bufferedSource.readByteArray(totalExceptionBytes.toLong()) + } + + synchronized(this) { + this.bytes = publicSuffixListBytes!! + this.exceptionBytes = publicSuffixExceptionListBytes!! + } + } finally { + readCompleteLatch.countDown() + } + } + + override fun ensureLoaded() { + if (!listRead.get() && listRead.compareAndSet(false, true)) { + readTheListUninterruptibly() + } else { + try { + readCompleteLatch.await() + } catch (_: InterruptedException) { + Thread.currentThread().interrupt() // Retain interrupted status. + } + } + + check(::bytes.isInitialized) { + // May have failed with an IOException + "Unable to load $PUBLIC_SUFFIX_RESOURCE resource from the classpath." + } + } + + /** + * Reads the public suffix list treating the operation as uninterruptible. We always want to read + * the list otherwise we'll be left in a bad state. If the thread was interrupted prior to this + * operation, it will be re-interrupted after the list is read. + */ + private fun readTheListUninterruptibly() { + var interrupted = false + try { + while (true) { + try { + readTheList() + return + } catch (_: InterruptedIOException) { + Thread.interrupted() // Temporarily clear the interrupted state. + interrupted = true + } catch (e: IOException) { + Platform.get().log("Failed to read public suffix list", Platform.WARN, e) + return + } + } + } finally { + if (interrupted) { + Thread.currentThread().interrupt() // Retain interrupted status. + } + } + } + + /** Visible for testing. */ + fun setListBytes( + publicSuffixListBytes: ByteArray, + publicSuffixExceptionListBytes: ByteArray, + ) { + this.bytes = publicSuffixListBytes + this.exceptionBytes = publicSuffixExceptionListBytes + listRead.set(true) + readCompleteLatch.countDown() + } + + companion object { + @JvmField + val PUBLIC_SUFFIX_RESOURCE = + "/okhttp3/internal/publicsuffix/${PublicSuffixDatabase::class.java.simpleName}.gz".toPath() + } +} diff --git a/okhttp/src/main/resources/okhttp3/internal/publicsuffix/NOTICE b/okhttp/src/test/resources/okhttp3/internal/publicsuffix/NOTICE similarity index 100% rename from okhttp/src/main/resources/okhttp3/internal/publicsuffix/NOTICE rename to okhttp/src/test/resources/okhttp3/internal/publicsuffix/NOTICE diff --git a/okhttp/src/main/resources/okhttp3/internal/publicsuffix/PublicSuffixDatabase.gz b/okhttp/src/test/resources/okhttp3/internal/publicsuffix/PublicSuffixDatabase.gz similarity index 100% rename from okhttp/src/main/resources/okhttp3/internal/publicsuffix/PublicSuffixDatabase.gz rename to okhttp/src/test/resources/okhttp3/internal/publicsuffix/PublicSuffixDatabase.gz From 6b776ec9b2be97ce406bfce55c5f81bfec99c65d Mon Sep 17 00:00:00 2001 From: Yuri Schimke Date: Sun, 24 Nov 2024 11:29:45 +0000 Subject: [PATCH 02/11] Add a test of contents --- .../internal/publicsuffix/PublicSuffixDatabaseTest.kt | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/okhttp/src/test/java/okhttp3/internal/publicsuffix/PublicSuffixDatabaseTest.kt b/okhttp/src/test/java/okhttp3/internal/publicsuffix/PublicSuffixDatabaseTest.kt index 7ecb0d543f96..0cab843af13a 100644 --- a/okhttp/src/test/java/okhttp3/internal/publicsuffix/PublicSuffixDatabaseTest.kt +++ b/okhttp/src/test/java/okhttp3/internal/publicsuffix/PublicSuffixDatabaseTest.kt @@ -281,6 +281,11 @@ class PublicSuffixDatabaseTest { checkPublicSuffix("xn--fiqs8s", null) } + @Test fun contentsMatch() { + assertEquals(list.bytes, EmbeddedPublicSuffixList.bytes) + assertEquals(list.exceptionBytes, EmbeddedPublicSuffixList.exceptionBytes) + } + private fun checkPublicSuffix( domain: String, registrablePart: String?, From 7bd9eca543eb6756c03387cbb2562d6d71eb85d9 Mon Sep 17 00:00:00 2001 From: Yuri Schimke Date: Sun, 24 Nov 2024 12:00:46 +0000 Subject: [PATCH 03/11] Use ByteString and base64 encoding. Base64 will added 4/3 overhead, and some decoding, but avoid issues with java class files. --- okhttp/build.gradle.kts | 22 ++++++------------- .../publicsuffix/PublicSuffixDatabase.kt | 5 +++-- .../internal/publicsuffix/PublicSuffixList.kt | 6 +++-- .../publicsuffix/EmbeddedPublicSuffixList.kt | 7 ++++-- .../publicsuffix/PublicSuffixDatabaseTest.kt | 13 ++++++----- .../publicsuffix/ResourcePublicSuffixList.kt | 17 +++++++------- 6 files changed, 36 insertions(+), 34 deletions(-) diff --git a/okhttp/build.gradle.kts b/okhttp/build.gradle.kts index 4a973edd6dff..7d726bc08a3e 100644 --- a/okhttp/build.gradle.kts +++ b/okhttp/build.gradle.kts @@ -2,6 +2,7 @@ import com.vanniktech.maven.publish.JavadocJar import com.vanniktech.maven.publish.KotlinJvm import java.io.DataInputStream import java.io.FileInputStream +import java.util.Base64 import java.util.zip.GZIPInputStream plugins { @@ -12,15 +13,8 @@ plugins { id("binary-compatibility-validator") } -fun ByteArray.toByteArrayExpression(): String { - return buildString { - append("byteArrayOf(") - this@toByteArrayExpression.forEach { - append(it) - append(", ") - } - append(")") - } +fun ByteArray.toByteStringExpression(): String { + return "\"${Base64.getEncoder().encodeToString(this@toByteStringExpression)}\".decodeBase64()!!" } val copyKotlinTemplates = tasks.register("copyKotlinTemplates") { @@ -32,7 +26,7 @@ val copyKotlinTemplates = tasks.register("copyKotlinTemplates") { // TODO replace with KotlinPoet? val databaseGz = project.file("src/test/resources/okhttp3/internal/publicsuffix/PublicSuffixDatabase.gz") println("loading $databaseGz") - val (publicSuffixListBytes, publicSuffixListExceptionBytes) = DataInputStream(GZIPInputStream(FileInputStream(databaseGz))).use { + val (listBytes, exceptionBytes) = DataInputStream(GZIPInputStream(FileInputStream(databaseGz))).use { val totalBytes = it.readInt() val publicSuffixListBytes = it.readNBytes(totalBytes) println("read $totalBytes") @@ -41,18 +35,16 @@ val copyKotlinTemplates = tasks.register("copyKotlinTemplates") { val publicSuffixExceptionListBytes = it.readNBytes(totalExceptionBytes) println("read $totalExceptionBytes") - Pair(publicSuffixListBytes.toByteArrayExpression(), publicSuffixExceptionListBytes.toByteArrayExpression()) + Pair(publicSuffixListBytes.toByteStringExpression(), publicSuffixExceptionListBytes.toByteStringExpression()) } - println(publicSuffixListBytes.substring(0, 10)) - expand( // Build & use okhttp3/internal/-InternalVersion.kt "projectVersion" to project.version, // Build okhttp3/internal/publicsuffix/EmbeddedPublicSuffixList.kt - "publicSuffixListBytes" to publicSuffixListBytes, - "publicSuffixListExceptionBytes" to publicSuffixListExceptionBytes + "publicSuffixListBytes" to listBytes, + "publicSuffixListExceptionBytes" to exceptionBytes ) } diff --git a/okhttp/src/main/kotlin/okhttp3/internal/publicsuffix/PublicSuffixDatabase.kt b/okhttp/src/main/kotlin/okhttp3/internal/publicsuffix/PublicSuffixDatabase.kt index f6e33ac28ae9..a602e2fd0f8e 100644 --- a/okhttp/src/main/kotlin/okhttp3/internal/publicsuffix/PublicSuffixDatabase.kt +++ b/okhttp/src/main/kotlin/okhttp3/internal/publicsuffix/PublicSuffixDatabase.kt @@ -17,6 +17,7 @@ package okhttp3.internal.publicsuffix import java.net.IDN import okhttp3.internal.and +import okio.ByteString /** * A database of public suffixes provided by [publicsuffix.org][publicsuffix_org]. @@ -158,7 +159,7 @@ class PublicSuffixDatabase internal constructor( return instance } - private fun ByteArray.binarySearch( + private fun ByteString.binarySearch( labels: Array, labelIndex: Int, ): String? { @@ -238,7 +239,7 @@ class PublicSuffixDatabase internal constructor( low = mid + end + 1 } else { // Found a match. - match = String(this, mid, publicSuffixLength) + match = this.substring(mid, publicSuffixLength).string(Charsets.UTF_8) break } } diff --git a/okhttp/src/main/kotlin/okhttp3/internal/publicsuffix/PublicSuffixList.kt b/okhttp/src/main/kotlin/okhttp3/internal/publicsuffix/PublicSuffixList.kt index 97fdc792394c..3fdfaa659edc 100644 --- a/okhttp/src/main/kotlin/okhttp3/internal/publicsuffix/PublicSuffixList.kt +++ b/okhttp/src/main/kotlin/okhttp3/internal/publicsuffix/PublicSuffixList.kt @@ -15,12 +15,14 @@ */ package okhttp3.internal.publicsuffix +import okio.ByteString + /** * Basic I/O for the PublicSuffixDatabase.gz. */ internal interface PublicSuffixList { fun ensureLoaded() - val bytes: ByteArray - val exceptionBytes: ByteArray + val bytes: ByteString + val exceptionBytes: ByteString } diff --git a/okhttp/src/main/kotlinTemplates/okhttp3/internal/publicsuffix/EmbeddedPublicSuffixList.kt b/okhttp/src/main/kotlinTemplates/okhttp3/internal/publicsuffix/EmbeddedPublicSuffixList.kt index 03caa9994d92..25cd448e1b24 100644 --- a/okhttp/src/main/kotlinTemplates/okhttp3/internal/publicsuffix/EmbeddedPublicSuffixList.kt +++ b/okhttp/src/main/kotlinTemplates/okhttp3/internal/publicsuffix/EmbeddedPublicSuffixList.kt @@ -21,6 +21,9 @@ package okhttp3.internal.publicsuffix //It is subject to the terms of the Mozilla Public License, v. 2.0: //https://mozilla.org/MPL/2.0/ +import okio.ByteString +import okio.ByteString.Companion.decodeBase64 + /** * A implementation of I/O for PublicSuffixDatabase.gz by directly encoding * the relevant byte arrays in a class file. @@ -29,7 +32,7 @@ internal object EmbeddedPublicSuffixList: PublicSuffixList { override fun ensureLoaded() { } - override val bytes: ByteArray = $publicSuffixListBytes + override val bytes: ByteString = $publicSuffixListBytes - override val exceptionBytes: ByteArray = $publicSuffixListExceptionBytes + override val exceptionBytes: ByteString = $publicSuffixListExceptionBytes } diff --git a/okhttp/src/test/java/okhttp3/internal/publicsuffix/PublicSuffixDatabaseTest.kt b/okhttp/src/test/java/okhttp3/internal/publicsuffix/PublicSuffixDatabaseTest.kt index 0cab843af13a..136a77ff84e6 100644 --- a/okhttp/src/test/java/okhttp3/internal/publicsuffix/PublicSuffixDatabaseTest.kt +++ b/okhttp/src/test/java/okhttp3/internal/publicsuffix/PublicSuffixDatabaseTest.kt @@ -23,6 +23,7 @@ import kotlin.test.assertEquals import kotlin.test.assertFailsWith import okhttp3.internal.toCanonicalHost import okio.Buffer +import okio.ByteString import okio.FileSystem import okio.GzipSource import okio.Path.Companion.toPath @@ -40,7 +41,7 @@ class PublicSuffixDatabaseTest { .writeUtf8("com\n") .writeUtf8("my.square.com\n") .writeUtf8("square.com\n") - list.setListBytes(buffer.readByteArray(), byteArrayOf()) + list.setListBytes(buffer.readByteString(), ByteString.of()) assertThat(publicSuffixDatabase.getEffectiveTldPlusOne("example.com")) .isEqualTo("example.com") assertThat(publicSuffixDatabase.getEffectiveTldPlusOne("foo.example.com")) @@ -57,7 +58,7 @@ class PublicSuffixDatabaseTest { .writeUtf8("*.square.com\n") .writeUtf8("com\n") .writeUtf8("example.com\n") - list.setListBytes(buffer.readByteArray(), byteArrayOf()) + list.setListBytes(buffer.readByteString(), ByteString.of()) assertThat(publicSuffixDatabase.getEffectiveTldPlusOne("my.square.com")).isNull() assertThat(publicSuffixDatabase.getEffectiveTldPlusOne("foo.my.square.com")) .isEqualTo("foo.my.square.com") @@ -71,7 +72,7 @@ class PublicSuffixDatabaseTest { .writeUtf8("bbb\n") .writeUtf8("ddd\n") .writeUtf8("fff\n") - list.setListBytes(buffer.readByteArray(), byteArrayOf()) + list.setListBytes(buffer.readByteString(), ByteString.of()) assertThat(publicSuffixDatabase.getEffectiveTldPlusOne("aaa")).isNull() assertThat(publicSuffixDatabase.getEffectiveTldPlusOne("ggg")).isNull() assertThat(publicSuffixDatabase.getEffectiveTldPlusOne("ccc")).isNull() @@ -88,7 +89,7 @@ class PublicSuffixDatabaseTest { .writeUtf8("*.square.jp\n") .writeUtf8("example.com\n") .writeUtf8("square.com\n") - list.setListBytes(buffer.readByteArray(), exception.readByteArray()) + list.setListBytes(buffer.readByteString(), exception.readByteString()) assertThat(publicSuffixDatabase.getEffectiveTldPlusOne("my.square.jp")) .isEqualTo("my.square.jp") assertThat(publicSuffixDatabase.getEffectiveTldPlusOne("foo.my.square.jp")) @@ -106,7 +107,7 @@ class PublicSuffixDatabaseTest { .writeUtf8("*.square.jp\n") .writeUtf8("example.com\n") .writeUtf8("square.com\n") - list.setListBytes(buffer.readByteArray(), exception.readByteArray()) + list.setListBytes(buffer.readByteString(), exception.readByteString()) assertThat(publicSuffixDatabase.getEffectiveTldPlusOne("example.com")).isNull() assertThat(publicSuffixDatabase.getEffectiveTldPlusOne("foo.square.jp")).isNull() } @@ -282,6 +283,8 @@ class PublicSuffixDatabaseTest { } @Test fun contentsMatch() { + list.ensureLoaded() + assertEquals(list.bytes, EmbeddedPublicSuffixList.bytes) assertEquals(list.exceptionBytes, EmbeddedPublicSuffixList.exceptionBytes) } diff --git a/okhttp/src/test/java/okhttp3/internal/publicsuffix/ResourcePublicSuffixList.kt b/okhttp/src/test/java/okhttp3/internal/publicsuffix/ResourcePublicSuffixList.kt index 0e247f90f72b..377e037361c8 100644 --- a/okhttp/src/test/java/okhttp3/internal/publicsuffix/ResourcePublicSuffixList.kt +++ b/okhttp/src/test/java/okhttp3/internal/publicsuffix/ResourcePublicSuffixList.kt @@ -5,6 +5,7 @@ import java.io.InterruptedIOException import java.util.concurrent.CountDownLatch import java.util.concurrent.atomic.AtomicBoolean import okhttp3.internal.platform.Platform +import okio.ByteString import okio.FileSystem import okio.GzipSource import okio.Path @@ -25,21 +26,21 @@ internal class ResourcePublicSuffixList( // that will likely never be used. Each rule is separated by '\n'. Please see the // PublicSuffixListGenerator class for how these lists are generated. // Guarded by this. - override lateinit var bytes: ByteArray - override lateinit var exceptionBytes: ByteArray + override lateinit var bytes: ByteString + override lateinit var exceptionBytes: ByteString @Throws(IOException::class) private fun readTheList() { - var publicSuffixListBytes: ByteArray? - var publicSuffixExceptionListBytes: ByteArray? + var publicSuffixListBytes: ByteString? + var publicSuffixExceptionListBytes: ByteString? try { GzipSource(fileSystem.source(path)).buffer().use { bufferedSource -> val totalBytes = bufferedSource.readInt() - publicSuffixListBytes = bufferedSource.readByteArray(totalBytes.toLong()) + publicSuffixListBytes = bufferedSource.readByteString(totalBytes.toLong()) val totalExceptionBytes = bufferedSource.readInt() - publicSuffixExceptionListBytes = bufferedSource.readByteArray(totalExceptionBytes.toLong()) + publicSuffixExceptionListBytes = bufferedSource.readByteString(totalExceptionBytes.toLong()) } synchronized(this) { @@ -97,8 +98,8 @@ internal class ResourcePublicSuffixList( /** Visible for testing. */ fun setListBytes( - publicSuffixListBytes: ByteArray, - publicSuffixExceptionListBytes: ByteArray, + publicSuffixListBytes: ByteString, + publicSuffixExceptionListBytes: ByteString, ) { this.bytes = publicSuffixListBytes this.exceptionBytes = publicSuffixExceptionListBytes From c9802684b8d2c390bed760c7a3fe6d371b058395 Mon Sep 17 00:00:00 2001 From: Yuri Schimke Date: Sun, 24 Nov 2024 12:01:42 +0000 Subject: [PATCH 04/11] Cleanup --- okhttp/build.gradle.kts | 3 --- 1 file changed, 3 deletions(-) diff --git a/okhttp/build.gradle.kts b/okhttp/build.gradle.kts index 7d726bc08a3e..3419a3e175ea 100644 --- a/okhttp/build.gradle.kts +++ b/okhttp/build.gradle.kts @@ -25,15 +25,12 @@ val copyKotlinTemplates = tasks.register("copyKotlinTemplates") { // TODO replace with KotlinPoet? val databaseGz = project.file("src/test/resources/okhttp3/internal/publicsuffix/PublicSuffixDatabase.gz") - println("loading $databaseGz") val (listBytes, exceptionBytes) = DataInputStream(GZIPInputStream(FileInputStream(databaseGz))).use { val totalBytes = it.readInt() val publicSuffixListBytes = it.readNBytes(totalBytes) - println("read $totalBytes") val totalExceptionBytes = it.readInt() val publicSuffixExceptionListBytes = it.readNBytes(totalExceptionBytes) - println("read $totalExceptionBytes") Pair(publicSuffixListBytes.toByteStringExpression(), publicSuffixExceptionListBytes.toByteStringExpression()) } From 7c0424f687662fbd484a6352a2b5a010b8f0f7a7 Mon Sep 17 00:00:00 2001 From: Yuri Schimke Date: Sun, 24 Nov 2024 12:10:12 +0000 Subject: [PATCH 05/11] Fix binary search --- .../internal/publicsuffix/PublicSuffixDatabase.kt | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/okhttp/src/main/kotlin/okhttp3/internal/publicsuffix/PublicSuffixDatabase.kt b/okhttp/src/main/kotlin/okhttp3/internal/publicsuffix/PublicSuffixDatabase.kt index a602e2fd0f8e..3d4297298f2d 100644 --- a/okhttp/src/main/kotlin/okhttp3/internal/publicsuffix/PublicSuffixDatabase.kt +++ b/okhttp/src/main/kotlin/okhttp3/internal/publicsuffix/PublicSuffixDatabase.kt @@ -18,6 +18,7 @@ package okhttp3.internal.publicsuffix import java.net.IDN import okhttp3.internal.and import okio.ByteString +import okio.ByteString.Companion.encodeUtf8 /** * A database of public suffixes provided by [publicsuffix.org][publicsuffix_org]. @@ -82,7 +83,7 @@ class PublicSuffixDatabase internal constructor( publicSuffixList.ensureLoaded() // Break apart the domain into UTF-8 labels, i.e. foo.bar.com turns into [foo, bar, com]. - val domainLabelsUtf8Bytes = Array(domainLabels.size) { i -> domainLabels[i].toByteArray() } + val domainLabelsUtf8Bytes = Array(domainLabels.size) { i -> domainLabels[i].encodeUtf8() } // Start by looking for exact matches. We start at the leftmost label. For example, foo.bar.com // will look like: [foo, bar, com], [bar, com], [com]. The longest matching rule wins. @@ -148,7 +149,7 @@ class PublicSuffixDatabase internal constructor( } companion object { - private val WILDCARD_LABEL = byteArrayOf('*'.code.toByte()) + private val WILDCARD_LABEL = ByteString.of('*'.code.toByte()) private val PREVAILING_RULE = listOf("*") private const val EXCEPTION_MARKER = '!' @@ -160,7 +161,7 @@ class PublicSuffixDatabase internal constructor( } private fun ByteString.binarySearch( - labels: Array, + labels: Array, labelIndex: Int, ): String? { var low = 0 @@ -239,7 +240,7 @@ class PublicSuffixDatabase internal constructor( low = mid + end + 1 } else { // Found a match. - match = this.substring(mid, publicSuffixLength).string(Charsets.UTF_8) + match = this.substring(mid, mid + publicSuffixLength).string(Charsets.UTF_8) break } } From b85aecfae3aa5544197a8248912ad95ce1a17024 Mon Sep 17 00:00:00 2001 From: Yuri Schimke Date: Sun, 24 Nov 2024 12:25:46 +0000 Subject: [PATCH 06/11] Exclude template from spotless --- build.gradle.kts | 1 + 1 file changed, 1 insertion(+) diff --git a/build.gradle.kts b/build.gradle.kts index befab1665954..42c6bdee9feb 100644 --- a/build.gradle.kts +++ b/build.gradle.kts @@ -42,6 +42,7 @@ apply(plugin = "com.diffplug.spotless") configure { kotlin { target("**/*.kt") + targetExclude("**/kotlinTemplates/**/*.kt") ktlint() } } From 02805af632d746012d8c1f78db13f86eab54dd26 Mon Sep 17 00:00:00 2001 From: Yuri Schimke Date: Sun, 24 Nov 2024 12:38:21 +0000 Subject: [PATCH 07/11] Fix graal test --- .../src/main/kotlin/okhttp3/PublicSuffixDatabaseTest.kt | 9 --------- 1 file changed, 9 deletions(-) diff --git a/native-image-tests/src/main/kotlin/okhttp3/PublicSuffixDatabaseTest.kt b/native-image-tests/src/main/kotlin/okhttp3/PublicSuffixDatabaseTest.kt index 98a452cd40dc..e92297177024 100644 --- a/native-image-tests/src/main/kotlin/okhttp3/PublicSuffixDatabaseTest.kt +++ b/native-image-tests/src/main/kotlin/okhttp3/PublicSuffixDatabaseTest.kt @@ -19,7 +19,6 @@ import assertk.assertThat import assertk.assertions.isEqualTo import assertk.assertions.isGreaterThan import okhttp3.HttpUrl.Companion.toHttpUrl -import okhttp3.internal.publicsuffix.PublicSuffixDatabase import okhttp3.testing.PlatformRule import okio.FileSystem import org.junit.jupiter.api.Test @@ -36,12 +35,4 @@ class PublicSuffixDatabaseTest { assertThat(url.topPrivateDomain()).isEqualTo("twitter.com") } - - @Test - fun testPublicSuffixes() { - platform.assumeNotGraalVMImage() - - val metadata = FileSystem.RESOURCES.metadata(PublicSuffixDatabase.PUBLIC_SUFFIX_RESOURCE) - assertThat(metadata.size!!).isGreaterThan(30000) - } } From eb9c8a2bf9f4bac66a6951aa8d5be100bab54940 Mon Sep 17 00:00:00 2001 From: Yuri Schimke Date: Sun, 24 Nov 2024 12:52:35 +0000 Subject: [PATCH 08/11] spotless --- .../src/main/kotlin/okhttp3/PublicSuffixDatabaseTest.kt | 2 -- .../okhttp3/internal/publicsuffix/PublicSuffixDatabase.kt | 3 +-- .../okhttp3/internal/publicsuffix/PublicSuffixDatabaseTest.kt | 2 +- 3 files changed, 2 insertions(+), 5 deletions(-) diff --git a/native-image-tests/src/main/kotlin/okhttp3/PublicSuffixDatabaseTest.kt b/native-image-tests/src/main/kotlin/okhttp3/PublicSuffixDatabaseTest.kt index e92297177024..175de8da30fc 100644 --- a/native-image-tests/src/main/kotlin/okhttp3/PublicSuffixDatabaseTest.kt +++ b/native-image-tests/src/main/kotlin/okhttp3/PublicSuffixDatabaseTest.kt @@ -17,10 +17,8 @@ package okhttp3 import assertk.assertThat import assertk.assertions.isEqualTo -import assertk.assertions.isGreaterThan import okhttp3.HttpUrl.Companion.toHttpUrl import okhttp3.testing.PlatformRule -import okio.FileSystem import org.junit.jupiter.api.Test import org.junit.jupiter.api.extension.RegisterExtension diff --git a/okhttp/src/main/kotlin/okhttp3/internal/publicsuffix/PublicSuffixDatabase.kt b/okhttp/src/main/kotlin/okhttp3/internal/publicsuffix/PublicSuffixDatabase.kt index 3d4297298f2d..88094ad1c600 100644 --- a/okhttp/src/main/kotlin/okhttp3/internal/publicsuffix/PublicSuffixDatabase.kt +++ b/okhttp/src/main/kotlin/okhttp3/internal/publicsuffix/PublicSuffixDatabase.kt @@ -26,9 +26,8 @@ import okio.ByteString.Companion.encodeUtf8 * [publicsuffix_org]: https://publicsuffix.org/ */ class PublicSuffixDatabase internal constructor( - private val publicSuffixList: PublicSuffixList + private val publicSuffixList: PublicSuffixList, ) { - /** * Returns the effective top-level domain plus one (eTLD+1) by referencing the public suffix list. * Returns null if the domain is a public suffix or a private address. diff --git a/okhttp/src/test/java/okhttp3/internal/publicsuffix/PublicSuffixDatabaseTest.kt b/okhttp/src/test/java/okhttp3/internal/publicsuffix/PublicSuffixDatabaseTest.kt index 136a77ff84e6..02ef95e947a3 100644 --- a/okhttp/src/test/java/okhttp3/internal/publicsuffix/PublicSuffixDatabaseTest.kt +++ b/okhttp/src/test/java/okhttp3/internal/publicsuffix/PublicSuffixDatabaseTest.kt @@ -167,7 +167,7 @@ class PublicSuffixDatabaseTest { PublicSuffixDatabase( ResourcePublicSuffixList( path = "/xxx.gz".toPath(), - ) + ), ) lateinit var firstFailure: Exception assertFailsWith { From d8d8c87e8dbc93792cf89f7bab80f65211c7e85f Mon Sep 17 00:00:00 2001 From: Yuri Schimke Date: Sun, 24 Nov 2024 15:07:30 +0000 Subject: [PATCH 09/11] decompress in memory --- okhttp/build.gradle.kts | 13 ++--------- .../publicsuffix/EmbeddedPublicSuffixList.kt | 22 +++++++++++++++++-- 2 files changed, 22 insertions(+), 13 deletions(-) diff --git a/okhttp/build.gradle.kts b/okhttp/build.gradle.kts index 3419a3e175ea..9c9ea07c776f 100644 --- a/okhttp/build.gradle.kts +++ b/okhttp/build.gradle.kts @@ -25,23 +25,14 @@ val copyKotlinTemplates = tasks.register("copyKotlinTemplates") { // TODO replace with KotlinPoet? val databaseGz = project.file("src/test/resources/okhttp3/internal/publicsuffix/PublicSuffixDatabase.gz") - val (listBytes, exceptionBytes) = DataInputStream(GZIPInputStream(FileInputStream(databaseGz))).use { - val totalBytes = it.readInt() - val publicSuffixListBytes = it.readNBytes(totalBytes) - - val totalExceptionBytes = it.readInt() - val publicSuffixExceptionListBytes = it.readNBytes(totalExceptionBytes) - - Pair(publicSuffixListBytes.toByteStringExpression(), publicSuffixExceptionListBytes.toByteStringExpression()) - } + val listBytes = databaseGz.readBytes().toByteStringExpression() expand( // Build & use okhttp3/internal/-InternalVersion.kt "projectVersion" to project.version, // Build okhttp3/internal/publicsuffix/EmbeddedPublicSuffixList.kt - "publicSuffixListBytes" to listBytes, - "publicSuffixListExceptionBytes" to exceptionBytes + "publicSuffixListBytes" to listBytes ) } diff --git a/okhttp/src/main/kotlinTemplates/okhttp3/internal/publicsuffix/EmbeddedPublicSuffixList.kt b/okhttp/src/main/kotlinTemplates/okhttp3/internal/publicsuffix/EmbeddedPublicSuffixList.kt index 25cd448e1b24..2e0d055b3083 100644 --- a/okhttp/src/main/kotlinTemplates/okhttp3/internal/publicsuffix/EmbeddedPublicSuffixList.kt +++ b/okhttp/src/main/kotlinTemplates/okhttp3/internal/publicsuffix/EmbeddedPublicSuffixList.kt @@ -21,8 +21,11 @@ package okhttp3.internal.publicsuffix //It is subject to the terms of the Mozilla Public License, v. 2.0: //https://mozilla.org/MPL/2.0/ +import okio.Buffer import okio.ByteString import okio.ByteString.Companion.decodeBase64 +import okio.GzipSource +import okio.buffer /** * A implementation of I/O for PublicSuffixDatabase.gz by directly encoding @@ -32,7 +35,22 @@ internal object EmbeddedPublicSuffixList: PublicSuffixList { override fun ensureLoaded() { } - override val bytes: ByteString = $publicSuffixListBytes + val compressedBytes: ByteString = $publicSuffixListBytes - override val exceptionBytes: ByteString = $publicSuffixListExceptionBytes + override val bytes: ByteString + + override val exceptionBytes: ByteString + + init { + Buffer().use { buffer -> + buffer.write(compressedBytes) + GzipSource(buffer).buffer().use { source -> + val totalBytes = source.readInt() + bytes = source.readByteString(totalBytes.toLong()) + + val totalExceptionBytes = source.readInt() + exceptionBytes = source.readByteString(totalExceptionBytes.toLong()) + } + } + } } From 7e4ba40ea59926452bcd37a257970fb5484cbb12 Mon Sep 17 00:00:00 2001 From: Yuri Schimke Date: Wed, 27 Nov 2024 08:22:53 +0000 Subject: [PATCH 10/11] avoid storing the bytes --- .../okhttp3/internal/publicsuffix/EmbeddedPublicSuffixList.kt | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/okhttp/src/main/kotlinTemplates/okhttp3/internal/publicsuffix/EmbeddedPublicSuffixList.kt b/okhttp/src/main/kotlinTemplates/okhttp3/internal/publicsuffix/EmbeddedPublicSuffixList.kt index 2e0d055b3083..93f14cff3d26 100644 --- a/okhttp/src/main/kotlinTemplates/okhttp3/internal/publicsuffix/EmbeddedPublicSuffixList.kt +++ b/okhttp/src/main/kotlinTemplates/okhttp3/internal/publicsuffix/EmbeddedPublicSuffixList.kt @@ -35,15 +35,13 @@ internal object EmbeddedPublicSuffixList: PublicSuffixList { override fun ensureLoaded() { } - val compressedBytes: ByteString = $publicSuffixListBytes - override val bytes: ByteString override val exceptionBytes: ByteString init { Buffer().use { buffer -> - buffer.write(compressedBytes) + buffer.write($publicSuffixListBytes) GzipSource(buffer).buffer().use { source -> val totalBytes = source.readInt() bytes = source.readByteString(totalBytes.toLong()) From 3daaa99150cbad1b1be112a899c3f4ea7c06b036 Mon Sep 17 00:00:00 2001 From: Yuri Schimke Date: Wed, 27 Nov 2024 08:38:02 +0000 Subject: [PATCH 11/11] update the update process --- .../publicsuffix/PublicSuffixListGenerator.kt | 15 ++++++++++++--- .../publicsuffix/ResourcePublicSuffixList.kt | 2 +- 2 files changed, 13 insertions(+), 4 deletions(-) diff --git a/okhttp/src/test/java/okhttp3/internal/publicsuffix/PublicSuffixListGenerator.kt b/okhttp/src/test/java/okhttp3/internal/publicsuffix/PublicSuffixListGenerator.kt index 26a69ac700e3..174dc49171e8 100644 --- a/okhttp/src/test/java/okhttp3/internal/publicsuffix/PublicSuffixListGenerator.kt +++ b/okhttp/src/test/java/okhttp3/internal/publicsuffix/PublicSuffixListGenerator.kt @@ -52,8 +52,8 @@ class PublicSuffixListGenerator( val fileSystem: FileSystem = FileSystem.SYSTEM, val client: OkHttpClient = OkHttpClient(), ) { - private val testResources = projectRoot / "okhttp/src/test/resources/okhttp3/internal/publicsuffix" - private val publicSuffixListDotDat = testResources / "public_suffix_list.dat" + private val testResources = projectRoot / "okhttp/src/test/resources" + private val publicSuffixListDotDat = testResources / "okhttp3/internal/publicsuffix/public_suffix_list.dat" private val outputFile = testResources / PUBLIC_SUFFIX_RESOURCE val request = Request("https://publicsuffix.org/list/public_suffix_list.dat".toHttpUrl()) @@ -168,5 +168,14 @@ A wildcard rule was added that wildcards the first level! We'll need to change t } suspend fun main() { - PublicSuffixListGenerator().import() + val publicSuffixListGenerator = PublicSuffixListGenerator() + + try { + publicSuffixListGenerator.import() + } finally { + publicSuffixListGenerator.client.run { + connectionPool.evictAll() + dispatcher.executorService.shutdownNow() + } + } } diff --git a/okhttp/src/test/java/okhttp3/internal/publicsuffix/ResourcePublicSuffixList.kt b/okhttp/src/test/java/okhttp3/internal/publicsuffix/ResourcePublicSuffixList.kt index 377e037361c8..3efacb9e66d1 100644 --- a/okhttp/src/test/java/okhttp3/internal/publicsuffix/ResourcePublicSuffixList.kt +++ b/okhttp/src/test/java/okhttp3/internal/publicsuffix/ResourcePublicSuffixList.kt @@ -110,6 +110,6 @@ internal class ResourcePublicSuffixList( companion object { @JvmField val PUBLIC_SUFFIX_RESOURCE = - "/okhttp3/internal/publicsuffix/${PublicSuffixDatabase::class.java.simpleName}.gz".toPath() + "okhttp3/internal/publicsuffix/${PublicSuffixDatabase::class.java.simpleName}.gz".toPath() } }