From f9a85bc1f9f2a54fb6173cb1e5313c090325862a Mon Sep 17 00:00:00 2001 From: Kevin Galligan Date: Fri, 8 Apr 2022 08:38:02 -0400 Subject: [PATCH 1/8] Split driver for new and strict memory models --- drivers/native-driver-strict/build.gradle | 117 +++++ .../native-driver-strict/gradle.properties | 3 + .../driver/native/util/NativeCache.kt | 0 .../sqldelight/driver/native/util/PoolLock.kt | 81 ++++ .../sqldelight/driver/native/util/Stately.kt | 0 .../driver/native/util/NativeCache.kt | 0 .../sqldelight/driver/native/util/Stately.kt | 0 .../sqldelight/driver/native/util/PoolLock.kt | 81 ++++ .../sqldelight/driver/native/util/PoolLock.kt | 81 ++++ .../driver/native/util/NativeCache.kt | 0 .../sqldelight/driver/native/util/PoolLock.kt | 81 ++++ .../cash/sqldelight/driver/native/Borrowed.kt | 6 + .../driver/native/NativeSqlDatabase.kt | 414 ++++++++++++++++++ .../app/cash/sqldelight/driver/native/Pool.kt | 122 ++++++ .../driver/native/SqliterSqlCursor.kt | 37 ++ .../driver/native/SqliterStatement.kt | 42 ++ .../driver/native/util/NativeCache.kt | 10 + .../sqldelight/driver/native/util/PoolLock.kt | 32 ++ .../drivers/native/LazyDriverBaseTest.kt | 108 +++++ .../drivers/native/NativeDriverTest.kt | 14 + .../drivers/native/NativeQueryTest.kt | 14 + .../native/NativeSqliteDriverTest.kt.ignore | 412 +++++++++++++++++ .../drivers/native/NativeTransacterTest.kt | 14 + .../connectionpool/BaseConcurrencyTest.kt | 175 ++++++++ .../NativeSqliteDriverConfigTest.kt | 19 + .../connectionpool/WalConcurrencyTest.kt | 202 +++++++++ drivers/native-driver/build.gradle | 11 +- .../driver/native/NativeSqlDatabase.kt | 4 +- .../app/cash/sqldelight/driver/native/Pool.kt | 7 +- .../driver/native/util/NativeCache.kt | 30 +- .../drivers/native/LazyDriverBaseTest.kt | 5 +- .../connectionpool/WalConcurrencyTest.kt | 7 +- gradle/libs.versions.toml | 2 +- settings.gradle | 1 + 34 files changed, 2102 insertions(+), 30 deletions(-) create mode 100644 drivers/native-driver-strict/build.gradle create mode 100644 drivers/native-driver-strict/gradle.properties rename drivers/{native-driver => native-driver-strict}/src/linuxMain/kotlin/app/cash/sqldelight/driver/native/util/NativeCache.kt (100%) create mode 100644 drivers/native-driver-strict/src/linuxMain/kotlin/app/cash/sqldelight/driver/native/util/PoolLock.kt rename drivers/{native-driver => native-driver-strict}/src/linuxMain/kotlin/app/cash/sqldelight/driver/native/util/Stately.kt (100%) rename drivers/{native-driver => native-driver-strict}/src/mingwMain/kotlin/app/cash/sqldelight/driver/native/util/NativeCache.kt (100%) rename drivers/{native-driver => native-driver-strict}/src/mingwMain/kotlin/app/cash/sqldelight/driver/native/util/Stately.kt (100%) create mode 100644 drivers/native-driver-strict/src/mingwX64Main/kotlin/app/cash/sqldelight/driver/native/util/PoolLock.kt create mode 100644 drivers/native-driver-strict/src/mingwX86Main/kotlin/app/cash/sqldelight/driver/native/util/PoolLock.kt rename drivers/{native-driver => native-driver-strict}/src/nativeDarwinMain/kotlin/app/cash/sqldelight/driver/native/util/NativeCache.kt (100%) create mode 100644 drivers/native-driver-strict/src/nativeDarwinMain/kotlin/app/cash/sqldelight/driver/native/util/PoolLock.kt create mode 100644 drivers/native-driver-strict/src/nativeMain/kotlin/app/cash/sqldelight/driver/native/Borrowed.kt create mode 100644 drivers/native-driver-strict/src/nativeMain/kotlin/app/cash/sqldelight/driver/native/NativeSqlDatabase.kt create mode 100644 drivers/native-driver-strict/src/nativeMain/kotlin/app/cash/sqldelight/driver/native/Pool.kt create mode 100644 drivers/native-driver-strict/src/nativeMain/kotlin/app/cash/sqldelight/driver/native/SqliterSqlCursor.kt create mode 100644 drivers/native-driver-strict/src/nativeMain/kotlin/app/cash/sqldelight/driver/native/SqliterStatement.kt create mode 100644 drivers/native-driver-strict/src/nativeMain/kotlin/app/cash/sqldelight/driver/native/util/NativeCache.kt create mode 100644 drivers/native-driver-strict/src/nativeMain/kotlin/app/cash/sqldelight/driver/native/util/PoolLock.kt create mode 100644 drivers/native-driver-strict/src/nativeTest/kotlin/com/squareup/sqldelight/drivers/native/LazyDriverBaseTest.kt create mode 100644 drivers/native-driver-strict/src/nativeTest/kotlin/com/squareup/sqldelight/drivers/native/NativeDriverTest.kt create mode 100644 drivers/native-driver-strict/src/nativeTest/kotlin/com/squareup/sqldelight/drivers/native/NativeQueryTest.kt create mode 100644 drivers/native-driver-strict/src/nativeTest/kotlin/com/squareup/sqldelight/drivers/native/NativeSqliteDriverTest.kt.ignore create mode 100644 drivers/native-driver-strict/src/nativeTest/kotlin/com/squareup/sqldelight/drivers/native/NativeTransacterTest.kt create mode 100644 drivers/native-driver-strict/src/nativeTest/kotlin/com/squareup/sqldelight/drivers/native/connectionpool/BaseConcurrencyTest.kt create mode 100644 drivers/native-driver-strict/src/nativeTest/kotlin/com/squareup/sqldelight/drivers/native/connectionpool/NativeSqliteDriverConfigTest.kt create mode 100644 drivers/native-driver-strict/src/nativeTest/kotlin/com/squareup/sqldelight/drivers/native/connectionpool/WalConcurrencyTest.kt diff --git a/drivers/native-driver-strict/build.gradle b/drivers/native-driver-strict/build.gradle new file mode 100644 index 00000000000..0eabe491090 --- /dev/null +++ b/drivers/native-driver-strict/build.gradle @@ -0,0 +1,117 @@ +import org.jetbrains.kotlin.konan.target.HostManager + +plugins { + alias(deps.plugins.kotlin.multiplatform) + alias(deps.plugins.publish) + alias(deps.plugins.dokka) +} + +def ideaActive = System.getProperty("idea.active") == "true" + +kotlin { + iosX64() + iosArm32() + iosArm64() + tvosX64() + tvosArm64() + watchosX86() + watchosX64() + watchosArm32() + watchosArm64() + macosX64() + mingwX86() + mingwX64() + linuxX64() + macosArm64() + iosSimulatorArm64() + watchosSimulatorArm64() + tvosSimulatorArm64() + + sourceSets { + commonMain { + dependencies { + api project (':runtime') + } + } + commonTest { + dependencies { + implementation deps.kotlin.test.common + implementation deps.testhelp + } + } + nativeMain { + dependsOn(commonMain) + dependencies { + api deps.sqliter + implementation deps.stately.core + } + } + nativeTest { + dependsOn(commonTest) + dependencies { + implementation project(':drivers:driver-test') + } + } + nativeDarwinMain{ + dependsOn(nativeMain) + } + mingwMain{ + dependsOn(nativeMain) + } + mingwX86Main{ + dependsOn(mingwMain) + } + mingwX64Main{ + dependsOn(mingwMain) + } + linuxMain{ + dependsOn(nativeMain) + } + } + + configure([targets.iosX64, targets.iosArm32, targets.iosArm64, targets.tvosX64, targets.tvosArm64, targets.watchosX86, targets.watchosX64, targets.watchosArm32, targets.watchosArm64, targets.macosX64, targets.macosArm64, targets.iosSimulatorArm64, targets.watchosSimulatorArm64, targets.tvosSimulatorArm64]) { + sourceSets.getByName("${name}Main").dependsOn(sourceSets.nativeDarwinMain) + sourceSets.getByName("${name}Test").dependsOn(sourceSets.nativeTest) + compilations.test { + kotlinOptions.freeCompilerArgs += ["-linker-options", "-lsqlite3"] + } + } + + configure([targets.linuxX64]) { + sourceSets.getByName("${name}Main").dependsOn(sourceSets.linuxMain) + sourceSets.getByName("${name}Test").dependsOn(sourceSets.nativeTest) + compilations.test { + kotlinOptions.freeCompilerArgs += ["-linker-options", "-lsqlite3 -L/usr/lib/x86_64-linux-gnu -L/usr/lib"] + } + } + + configure([targets.mingwX64]) { + sourceSets.getByName("${name}Main").dependsOn(sourceSets.mingwMain) + sourceSets.getByName("${name}Test").dependsOn(sourceSets.nativeTest) + compilations.test { + kotlinOptions.freeCompilerArgs += ["-linker-options", "-Lc:\\msys64\\mingw64\\lib -L$rootDir\\libs -lsqlite3".toString()] + } + } + + configure([targets.mingwX86]) { + sourceSets.getByName("${name}Main").dependsOn(sourceSets.mingwMain) + sourceSets.getByName("${name}Test").dependsOn(sourceSets.nativeTest) + compilations.test { + kotlinOptions.freeCompilerArgs += ["-linker-options", "-Lc:\\msys32\\mingw32\\lib -L$rootDir\\libs -lsqlite3".toString()] + } + } + + //linking fails for the linux test build if not built on a linux host + //ensure the tests and linking for them is only done on linux hosts + if(!HostManager.hostIsLinux) { + tasks.findByName("linuxX64Test")?.enabled = false + tasks.findByName("linkDebugTestLinuxX64")?.enabled = false + } + + if(!HostManager.hostIsMingw) { + tasks.findByName("mingwX64Test")?.enabled = false + tasks.findByName("linkDebugTestMingwX64")?.enabled = false + } +} + +apply from: "$rootDir/gradle/gradle-mvn-push.gradle" diff --git a/drivers/native-driver-strict/gradle.properties b/drivers/native-driver-strict/gradle.properties new file mode 100644 index 00000000000..b749e6d36ed --- /dev/null +++ b/drivers/native-driver-strict/gradle.properties @@ -0,0 +1,3 @@ +POM_ARTIFACT_ID=native-driver +POM_NAME=SQLDelight Native Driver +POM_DESCRIPTION=Native SQLite driver for SQLDelight diff --git a/drivers/native-driver/src/linuxMain/kotlin/app/cash/sqldelight/driver/native/util/NativeCache.kt b/drivers/native-driver-strict/src/linuxMain/kotlin/app/cash/sqldelight/driver/native/util/NativeCache.kt similarity index 100% rename from drivers/native-driver/src/linuxMain/kotlin/app/cash/sqldelight/driver/native/util/NativeCache.kt rename to drivers/native-driver-strict/src/linuxMain/kotlin/app/cash/sqldelight/driver/native/util/NativeCache.kt diff --git a/drivers/native-driver-strict/src/linuxMain/kotlin/app/cash/sqldelight/driver/native/util/PoolLock.kt b/drivers/native-driver-strict/src/linuxMain/kotlin/app/cash/sqldelight/driver/native/util/PoolLock.kt new file mode 100644 index 00000000000..3896457a75e --- /dev/null +++ b/drivers/native-driver-strict/src/linuxMain/kotlin/app/cash/sqldelight/driver/native/util/PoolLock.kt @@ -0,0 +1,81 @@ +package app.cash.sqldelight.driver.native.util + +import co.touchlab.stately.concurrency.AtomicBoolean +import kotlinx.cinterop.alloc +import kotlinx.cinterop.free +import kotlinx.cinterop.nativeHeap +import kotlinx.cinterop.ptr +import platform.posix.pthread_cond_destroy +import platform.posix.pthread_cond_init +import platform.posix.pthread_cond_signal +import platform.posix.pthread_cond_t +import platform.posix.pthread_cond_wait +import platform.posix.pthread_mutex_destroy +import platform.posix.pthread_mutex_init +import platform.posix.pthread_mutex_lock +import platform.posix.pthread_mutex_t +import platform.posix.pthread_mutex_unlock + +internal actual class PoolLock actual constructor() { + private val isActive = AtomicBoolean(true) + private val mutex = nativeHeap.alloc() + .apply { pthread_mutex_init(ptr, null) } + private val cond = nativeHeap.alloc() + .apply { pthread_cond_init(ptr, null) } + + actual fun withLock( + action: CriticalSection.() -> R + ): R { + check(isActive.value) + pthread_mutex_lock(mutex.ptr) + + val result: R + + try { + result = action(CriticalSection()) + } finally { + pthread_mutex_unlock(mutex.ptr) + } + + return result + } + + actual fun notifyConditionChanged() { + pthread_cond_signal(cond.ptr) + } + + actual fun close(): Boolean { + if (isActive.compareAndSet(expected = true, new = false)) { + pthread_cond_destroy(cond.ptr) + pthread_mutex_destroy(mutex.ptr) + nativeHeap.free(cond) + nativeHeap.free(mutex) + return true + } + + return false + } + + actual inner class CriticalSection { + actual fun loopForConditionalResult(block: () -> R?): R { + check(isActive.value) + + var result = block() + + while (result == null) { + pthread_cond_wait(cond.ptr, mutex.ptr) + result = block() + } + + return result + } + + actual fun loopUntilConditionalResult(block: () -> Boolean) { + check(isActive.value) + + while (!block()) { + pthread_cond_wait(cond.ptr, mutex.ptr) + } + } + } +} diff --git a/drivers/native-driver/src/linuxMain/kotlin/app/cash/sqldelight/driver/native/util/Stately.kt b/drivers/native-driver-strict/src/linuxMain/kotlin/app/cash/sqldelight/driver/native/util/Stately.kt similarity index 100% rename from drivers/native-driver/src/linuxMain/kotlin/app/cash/sqldelight/driver/native/util/Stately.kt rename to drivers/native-driver-strict/src/linuxMain/kotlin/app/cash/sqldelight/driver/native/util/Stately.kt diff --git a/drivers/native-driver/src/mingwMain/kotlin/app/cash/sqldelight/driver/native/util/NativeCache.kt b/drivers/native-driver-strict/src/mingwMain/kotlin/app/cash/sqldelight/driver/native/util/NativeCache.kt similarity index 100% rename from drivers/native-driver/src/mingwMain/kotlin/app/cash/sqldelight/driver/native/util/NativeCache.kt rename to drivers/native-driver-strict/src/mingwMain/kotlin/app/cash/sqldelight/driver/native/util/NativeCache.kt diff --git a/drivers/native-driver/src/mingwMain/kotlin/app/cash/sqldelight/driver/native/util/Stately.kt b/drivers/native-driver-strict/src/mingwMain/kotlin/app/cash/sqldelight/driver/native/util/Stately.kt similarity index 100% rename from drivers/native-driver/src/mingwMain/kotlin/app/cash/sqldelight/driver/native/util/Stately.kt rename to drivers/native-driver-strict/src/mingwMain/kotlin/app/cash/sqldelight/driver/native/util/Stately.kt diff --git a/drivers/native-driver-strict/src/mingwX64Main/kotlin/app/cash/sqldelight/driver/native/util/PoolLock.kt b/drivers/native-driver-strict/src/mingwX64Main/kotlin/app/cash/sqldelight/driver/native/util/PoolLock.kt new file mode 100644 index 00000000000..331f846dadb --- /dev/null +++ b/drivers/native-driver-strict/src/mingwX64Main/kotlin/app/cash/sqldelight/driver/native/util/PoolLock.kt @@ -0,0 +1,81 @@ +package app.cash.sqldelight.driver.native.util + +import co.touchlab.stately.concurrency.AtomicBoolean +import kotlinx.cinterop.alloc +import kotlinx.cinterop.free +import kotlinx.cinterop.nativeHeap +import kotlinx.cinterop.ptr +import platform.posix.pthread_cond_destroy +import platform.posix.pthread_cond_init +import platform.posix.pthread_cond_signal +import platform.posix.pthread_cond_tVar +import platform.posix.pthread_cond_wait +import platform.posix.pthread_mutex_destroy +import platform.posix.pthread_mutex_init +import platform.posix.pthread_mutex_lock +import platform.posix.pthread_mutex_tVar +import platform.posix.pthread_mutex_unlock + +internal actual class PoolLock actual constructor() { + private val isActive = AtomicBoolean(true) + private val mutex = nativeHeap.alloc() + .apply { pthread_mutex_init(ptr, null) } + private val cond = nativeHeap.alloc() + .apply { pthread_cond_init(ptr, null) } + + actual fun withLock( + action: CriticalSection.() -> R + ): R { + check(isActive.value) + pthread_mutex_lock(mutex.ptr) + + val result: R + + try { + result = action(CriticalSection()) + } finally { + pthread_mutex_unlock(mutex.ptr) + } + + return result + } + + actual fun notifyConditionChanged() { + pthread_cond_signal(cond.ptr) + } + + actual fun close(): Boolean { + if (isActive.compareAndSet(expected = true, new = false)) { + pthread_cond_destroy(cond.ptr) + pthread_mutex_destroy(mutex.ptr) + nativeHeap.free(cond) + nativeHeap.free(mutex) + return true + } + + return false + } + + actual inner class CriticalSection { + actual fun loopForConditionalResult(block: () -> R?): R { + check(isActive.value) + + var result = block() + + while (result == null) { + pthread_cond_wait(cond.ptr, mutex.ptr) + result = block() + } + + return result + } + + actual fun loopUntilConditionalResult(block: () -> Boolean) { + check(isActive.value) + + while (!block()) { + pthread_cond_wait(cond.ptr, mutex.ptr) + } + } + } +} diff --git a/drivers/native-driver-strict/src/mingwX86Main/kotlin/app/cash/sqldelight/driver/native/util/PoolLock.kt b/drivers/native-driver-strict/src/mingwX86Main/kotlin/app/cash/sqldelight/driver/native/util/PoolLock.kt new file mode 100644 index 00000000000..331f846dadb --- /dev/null +++ b/drivers/native-driver-strict/src/mingwX86Main/kotlin/app/cash/sqldelight/driver/native/util/PoolLock.kt @@ -0,0 +1,81 @@ +package app.cash.sqldelight.driver.native.util + +import co.touchlab.stately.concurrency.AtomicBoolean +import kotlinx.cinterop.alloc +import kotlinx.cinterop.free +import kotlinx.cinterop.nativeHeap +import kotlinx.cinterop.ptr +import platform.posix.pthread_cond_destroy +import platform.posix.pthread_cond_init +import platform.posix.pthread_cond_signal +import platform.posix.pthread_cond_tVar +import platform.posix.pthread_cond_wait +import platform.posix.pthread_mutex_destroy +import platform.posix.pthread_mutex_init +import platform.posix.pthread_mutex_lock +import platform.posix.pthread_mutex_tVar +import platform.posix.pthread_mutex_unlock + +internal actual class PoolLock actual constructor() { + private val isActive = AtomicBoolean(true) + private val mutex = nativeHeap.alloc() + .apply { pthread_mutex_init(ptr, null) } + private val cond = nativeHeap.alloc() + .apply { pthread_cond_init(ptr, null) } + + actual fun withLock( + action: CriticalSection.() -> R + ): R { + check(isActive.value) + pthread_mutex_lock(mutex.ptr) + + val result: R + + try { + result = action(CriticalSection()) + } finally { + pthread_mutex_unlock(mutex.ptr) + } + + return result + } + + actual fun notifyConditionChanged() { + pthread_cond_signal(cond.ptr) + } + + actual fun close(): Boolean { + if (isActive.compareAndSet(expected = true, new = false)) { + pthread_cond_destroy(cond.ptr) + pthread_mutex_destroy(mutex.ptr) + nativeHeap.free(cond) + nativeHeap.free(mutex) + return true + } + + return false + } + + actual inner class CriticalSection { + actual fun loopForConditionalResult(block: () -> R?): R { + check(isActive.value) + + var result = block() + + while (result == null) { + pthread_cond_wait(cond.ptr, mutex.ptr) + result = block() + } + + return result + } + + actual fun loopUntilConditionalResult(block: () -> Boolean) { + check(isActive.value) + + while (!block()) { + pthread_cond_wait(cond.ptr, mutex.ptr) + } + } + } +} diff --git a/drivers/native-driver/src/nativeDarwinMain/kotlin/app/cash/sqldelight/driver/native/util/NativeCache.kt b/drivers/native-driver-strict/src/nativeDarwinMain/kotlin/app/cash/sqldelight/driver/native/util/NativeCache.kt similarity index 100% rename from drivers/native-driver/src/nativeDarwinMain/kotlin/app/cash/sqldelight/driver/native/util/NativeCache.kt rename to drivers/native-driver-strict/src/nativeDarwinMain/kotlin/app/cash/sqldelight/driver/native/util/NativeCache.kt diff --git a/drivers/native-driver-strict/src/nativeDarwinMain/kotlin/app/cash/sqldelight/driver/native/util/PoolLock.kt b/drivers/native-driver-strict/src/nativeDarwinMain/kotlin/app/cash/sqldelight/driver/native/util/PoolLock.kt new file mode 100644 index 00000000000..3896457a75e --- /dev/null +++ b/drivers/native-driver-strict/src/nativeDarwinMain/kotlin/app/cash/sqldelight/driver/native/util/PoolLock.kt @@ -0,0 +1,81 @@ +package app.cash.sqldelight.driver.native.util + +import co.touchlab.stately.concurrency.AtomicBoolean +import kotlinx.cinterop.alloc +import kotlinx.cinterop.free +import kotlinx.cinterop.nativeHeap +import kotlinx.cinterop.ptr +import platform.posix.pthread_cond_destroy +import platform.posix.pthread_cond_init +import platform.posix.pthread_cond_signal +import platform.posix.pthread_cond_t +import platform.posix.pthread_cond_wait +import platform.posix.pthread_mutex_destroy +import platform.posix.pthread_mutex_init +import platform.posix.pthread_mutex_lock +import platform.posix.pthread_mutex_t +import platform.posix.pthread_mutex_unlock + +internal actual class PoolLock actual constructor() { + private val isActive = AtomicBoolean(true) + private val mutex = nativeHeap.alloc() + .apply { pthread_mutex_init(ptr, null) } + private val cond = nativeHeap.alloc() + .apply { pthread_cond_init(ptr, null) } + + actual fun withLock( + action: CriticalSection.() -> R + ): R { + check(isActive.value) + pthread_mutex_lock(mutex.ptr) + + val result: R + + try { + result = action(CriticalSection()) + } finally { + pthread_mutex_unlock(mutex.ptr) + } + + return result + } + + actual fun notifyConditionChanged() { + pthread_cond_signal(cond.ptr) + } + + actual fun close(): Boolean { + if (isActive.compareAndSet(expected = true, new = false)) { + pthread_cond_destroy(cond.ptr) + pthread_mutex_destroy(mutex.ptr) + nativeHeap.free(cond) + nativeHeap.free(mutex) + return true + } + + return false + } + + actual inner class CriticalSection { + actual fun loopForConditionalResult(block: () -> R?): R { + check(isActive.value) + + var result = block() + + while (result == null) { + pthread_cond_wait(cond.ptr, mutex.ptr) + result = block() + } + + return result + } + + actual fun loopUntilConditionalResult(block: () -> Boolean) { + check(isActive.value) + + while (!block()) { + pthread_cond_wait(cond.ptr, mutex.ptr) + } + } + } +} diff --git a/drivers/native-driver-strict/src/nativeMain/kotlin/app/cash/sqldelight/driver/native/Borrowed.kt b/drivers/native-driver-strict/src/nativeMain/kotlin/app/cash/sqldelight/driver/native/Borrowed.kt new file mode 100644 index 00000000000..f3252179e2a --- /dev/null +++ b/drivers/native-driver-strict/src/nativeMain/kotlin/app/cash/sqldelight/driver/native/Borrowed.kt @@ -0,0 +1,6 @@ +package app.cash.sqldelight.driver.native + +internal interface Borrowed { + val value: T + fun release() +} diff --git a/drivers/native-driver-strict/src/nativeMain/kotlin/app/cash/sqldelight/driver/native/NativeSqlDatabase.kt b/drivers/native-driver-strict/src/nativeMain/kotlin/app/cash/sqldelight/driver/native/NativeSqlDatabase.kt new file mode 100644 index 00000000000..79a2f5a393b --- /dev/null +++ b/drivers/native-driver-strict/src/nativeMain/kotlin/app/cash/sqldelight/driver/native/NativeSqlDatabase.kt @@ -0,0 +1,414 @@ +package app.cash.sqldelight.driver.native + +import app.cash.sqldelight.Query +import app.cash.sqldelight.Transacter +import app.cash.sqldelight.db.Closeable +import app.cash.sqldelight.db.SqlCursor +import app.cash.sqldelight.db.SqlDriver +import app.cash.sqldelight.db.SqlPreparedStatement +import app.cash.sqldelight.driver.native.util.nativeCache +import co.touchlab.sqliter.DatabaseConfiguration +import co.touchlab.sqliter.DatabaseConnection +import co.touchlab.sqliter.DatabaseManager +import co.touchlab.sqliter.Statement +import co.touchlab.sqliter.createDatabaseManager +import co.touchlab.sqliter.withStatement +import co.touchlab.stately.collections.SharedHashMap +import co.touchlab.stately.collections.SharedSet +import co.touchlab.stately.concurrency.ThreadLocalRef +import co.touchlab.stately.concurrency.value +import kotlin.native.concurrent.ensureNeverFrozen + +sealed class ConnectionWrapper : SqlDriver { + internal abstract fun accessConnection( + readOnly: Boolean, + block: ThreadConnection.() -> R + ): R + + final override fun execute( + identifier: Int?, + sql: String, + parameters: Int, + binders: (SqlPreparedStatement.() -> Unit)? + ) { + accessConnection(false) { + val statement = useStatement(identifier, sql) + if (binders != null) { + try { + SqliterStatement(statement).binders() + } catch (t: Throwable) { + statement.resetStatement() + clearIfNeeded(identifier, statement) + throw t + } + } + + statement.execute() + statement.resetStatement() + clearIfNeeded(identifier, statement) + } + } + + final override fun executeQuery( + identifier: Int?, + sql: String, + parameters: Int, + binders: (SqlPreparedStatement.() -> Unit)? + ): SqlCursor { + return accessConnection(true) { + val statement = getStatement(identifier, sql) + + if (binders != null) { + try { + SqliterStatement(statement).binders() + } catch (t: Throwable) { + statement.resetStatement() + safePut(identifier, statement) + throw t + } + } + + val cursor = statement.query() + + SqliterSqlCursor(cursor) { + statement.resetStatement() + if (closed) + statement.finalizeStatement() + safePut(identifier, statement) + } + } + } +} + +/** + * Native driver implementation. + * + * The driver creates two connection pools, which default to 1 connection maximum. There is a reader pool, which + * handles all query requests outside of a transaction. The other pool is the transaction pool, which handles + * all transactions and write requests outside of a transaction. + * + * When a transaction is started, that thread is aligned with a transaction pool connection. Attempting a write or + * starting another transaction, if no connections are available, will cause the caller to wait. + * + * You can have multiple connections in the transaction pool, but this would only be useful for read transactions. Writing + * from multiple connections in an overlapping manner can be problematic. + * + * Aligning a transaction to a thread means you cannot operate on a single transaction from multiple threads. + * However, it would be difficult to find a use case where this would be desirable or safe. Currently, the native + * implementation of kotlinx.coroutines does not use thread pooling. When that changes, we'll need a way to handle + * transaction/connection alignment similar to what the Android/JVM driver implemented. + * + * https://medium.com/androiddevelopers/threading-models-in-coroutines-and-android-sqlite-api-6cab11f7eb90 + * + * To use SqlDelight during create/upgrade processes, you can alternatively wrap a real connection + * with wrapConnection. + * + * SqlPreparedStatement instances also do not point to real resources until either execute or + * executeQuery is called. The SqlPreparedStatement structure also maintains a thread-aligned + * instance which accumulates bind calls. Those are replayed on a real SQLite statement instance + * when execute or executeQuery is called. This avoids race conditions with bind calls. + */ +class NativeSqliteDriver( + private val databaseManager: DatabaseManager, + maxReaderConnections: Int = 1, +) : ConnectionWrapper(), SqlDriver { + constructor( + configuration: DatabaseConfiguration, + maxReaderConnections: Int = 1 + ) : this( + databaseManager = createDatabaseManager(configuration), + maxReaderConnections = maxReaderConnections + ) + + constructor( + schema: SqlDriver.Schema, + name: String, + maxReaderConnections: Int = 1 + ) : this( + configuration = DatabaseConfiguration( + name = name, + version = schema.version, + create = { connection -> + wrapConnection(connection) { schema.create(it) } + }, + upgrade = { connection, oldVersion, newVersion -> + wrapConnection(connection) { schema.migrate(it, oldVersion, newVersion) } + } + ), + maxReaderConnections = maxReaderConnections + ) + + // A pool of reader connections used by all operations not in a transaction + internal val transactionPool: Pool + internal val readerPool: Pool + + // Once a transaction is started and connection borrowed, it will be here, but only for that + // thread + private val borrowedConnectionThread = ThreadLocalRef>() + private val listeners = SharedHashMap>() + + init { + // Single connection for transactions + transactionPool = Pool(1) { + ThreadConnection(databaseManager.createMultiThreadedConnection()) { _ -> + borrowedConnectionThread.let { + it.get()?.release() + it.value = null + } + } + } + + val maxReaderConnectionsForConfig: Int = when { + databaseManager.configuration.inMemory -> 1 // Memory db's are single connection, generally. You can use named connections, but there are other issues that need to be designed for + else -> maxReaderConnections + } + readerPool = Pool(maxReaderConnectionsForConfig) { + val connection = databaseManager.createMultiThreadedConnection() + connection.withStatement("PRAGMA query_only = 1") { execute() } // Ensure read only + ThreadConnection(connection) { + throw UnsupportedOperationException("Should never be in a transaction") + } + } + } + + override fun addListener(listener: Query.Listener, queryKeys: Array) { + queryKeys.forEach { + listeners.getOrPut(it, { SharedSet() }).add(listener) + } + } + + override fun removeListener(listener: Query.Listener, queryKeys: Array) { + queryKeys.forEach { + listeners[it]?.remove(listener) + } + } + + override fun notifyListeners(queryKeys: Array) { + val listenersToNotify = SharedSet() + queryKeys.forEach { listeners[it]?.let(listenersToNotify::addAll) } + listenersToNotify.forEach(Query.Listener::queryResultsChanged) + } + + override fun currentTransaction(): Transacter.Transaction? { + return borrowedConnectionThread.get()?.value?.transaction?.value + } + + override fun newTransaction(): Transacter.Transaction { + val alreadyBorrowed = borrowedConnectionThread.get() + return if (alreadyBorrowed == null) { + val borrowed = transactionPool.borrowEntry() + + try { + val trans = borrowed.value.newTransaction() + + borrowedConnectionThread.value = borrowed + trans + } catch (e: Throwable) { + // Unlock on failure. + borrowed.release() + throw e + } + } else { + alreadyBorrowed.value.newTransaction() + } + } + + /** + * If we're in a transaction, then I have a connection. Otherwise use shared. + */ + override fun accessConnection( + readOnly: Boolean, + block: ThreadConnection.() -> R + ): R { + val mine = borrowedConnectionThread.get() + + return if (readOnly) { + // Code intends to read, which doesn't need to block + if (mine != null) { + mine.value.block() + } else { + readerPool.access(block) + } + } else { + // Code intends to write, for which we're managing locks in code + if (mine != null) { + mine.value.block() + } else { + transactionPool.access(block) + } + } + } + + override fun close() { + transactionPool.close() + readerPool.close() + } +} + +/** + * Sqliter's DatabaseConfiguration takes lambda arguments for it's create and upgrade operations, + * which each take a DatabaseConnection argument. Use wrapConnection to have SqlDelight access this + * passed connection and avoid the pooling that the full SqlDriver instance performs. + * + * Note that queries created during this operation will be cleaned up. If holding onto a cursor from + * a wrap call, it will no longer be viable. + */ +fun wrapConnection( + connection: DatabaseConnection, + block: (SqlDriver) -> Unit +) { + val conn = SqliterWrappedConnection(ThreadConnection(connection) {}) + try { + block(conn) + } finally { + conn.close() + } +} + +/** + * SqlDriverConnection that wraps a Sqliter connection. Useful for migration tasks, or if you + * don't want the polling. + */ +internal class SqliterWrappedConnection( + private val threadConnection: ThreadConnection +) : ConnectionWrapper(), + SqlDriver { + override fun currentTransaction(): Transacter.Transaction? = threadConnection.transaction.value + + override fun newTransaction(): Transacter.Transaction = threadConnection.newTransaction() + + override fun accessConnection( + readOnly: Boolean, + block: ThreadConnection.() -> R + ): R = threadConnection.block() + + override fun addListener(listener: Query.Listener, queryKeys: Array) { + // No-op + } + + override fun removeListener(listener: Query.Listener, queryKeys: Array) { + // No-op + } + + override fun notifyListeners(queryKeys: Array) { + // No-op + } + + override fun close() { + threadConnection.cleanUp() + } +} + +/** + * Wraps and manages a "real" database connection. + * + * SQLite statements are specific to connections, and must be finalized explicitly. Cursors are + * backed by a statement resource, so we keep links to open cursors to allow us to close them out + * properly in cases where the user does not. + */ +internal class ThreadConnection( + private val connection: DatabaseConnection, + private val onEndTransaction: (ThreadConnection) -> Unit +) : Closeable { + internal val transaction = ThreadLocalRef() + internal val closed: Boolean + get() = connection.closed + + internal val statementCache = nativeCache() + + fun safePut(identifier: Int?, statement: Statement) { + val removed = if (identifier == null) { + statement + } else { + statementCache.put(identifier.toString(), statement) + } + removed?.finalizeStatement() + } + + fun getStatement(identifier: Int?, sql: String): Statement { + val statement = removeCreateStatement(identifier, sql) + return statement + } + + fun useStatement(identifier: Int?, sql: String): Statement { + return if (identifier != null) { + statementCache.getOrCreate(identifier.toString()) { + connection.createStatement(sql) + } + } else { + connection.createStatement(sql) + } + } + + fun clearIfNeeded(identifier: Int?, statement: Statement) { + if (identifier == null) { + statement.finalizeStatement() + } + } + + /** + * For cursors. Cursors are actually backed by SQLite statement instances, so they need to be + * removed from the cache when in use. We're giving out a SQLite resource here, so extra care. + */ + private fun removeCreateStatement(identifier: Int?, sql: String): Statement { + if (identifier != null) { + val cached = statementCache.remove(identifier.toString()) + if (cached != null) + return cached + } + + return connection.createStatement(sql) + } + + fun newTransaction(): Transacter.Transaction { + val enclosing = transaction.value + + // Create here, in case we bomb... + if (enclosing == null) { + connection.beginTransaction() + } + + val trans = Transaction(enclosing) + transaction.value = trans + + return trans + } + + /** + * This should only be called directly from wrapConnection. Clean resources without actually closing + * the underlying connection. + */ + internal fun cleanUp() { + statementCache.cleanUp { + it.finalizeStatement() + } + } + + override fun close() { + cleanUp() + connection.close() + } + + private inner class Transaction( + override val enclosingTransaction: Transacter.Transaction? + ) : Transacter.Transaction() { + init { ensureNeverFrozen() } + + override fun endTransaction(successful: Boolean) { + transaction.value = enclosingTransaction + + if (enclosingTransaction == null) { + try { + if (successful) { + connection.setTransactionSuccessful() + } + + connection.endTransaction() + } finally { + // Release if we have + onEndTransaction(this@ThreadConnection) + } + } + } + } +} diff --git a/drivers/native-driver-strict/src/nativeMain/kotlin/app/cash/sqldelight/driver/native/Pool.kt b/drivers/native-driver-strict/src/nativeMain/kotlin/app/cash/sqldelight/driver/native/Pool.kt new file mode 100644 index 00000000000..9f213039fa4 --- /dev/null +++ b/drivers/native-driver-strict/src/nativeMain/kotlin/app/cash/sqldelight/driver/native/Pool.kt @@ -0,0 +1,122 @@ +package app.cash.sqldelight.driver.native + +import app.cash.sqldelight.db.Closeable +import app.cash.sqldelight.driver.native.util.PoolLock +import co.touchlab.stately.concurrency.AtomicBoolean +import kotlin.native.concurrent.AtomicReference +import kotlin.native.concurrent.freeze + +/** + * A shared pool of connections. Borrowing is blocking when all connections are in use, and the pool has reached its + * designated capacity. + */ +internal class Pool(internal val capacity: Int, private val producer: () -> T) { + /** + * Hold a list of active connections. If it is null, it means the MultiPool has been closed. + */ + private val entriesRef = AtomicReference?>(listOf().freeze()) + private val poolLock = PoolLock() + + /** + * For test purposes only + */ + internal fun entryCount(): Int = poolLock.withLock { + entriesRef.value?.size ?: 0 + } + + fun borrowEntry(): Borrowed { + val snapshot = entriesRef.value ?: throw ClosedMultiPoolException + + // Fastpath: Borrow the first available entry. + val firstAvailable = snapshot.firstOrNull { it.tryToAcquire() } + + if (firstAvailable != null) { + return firstAvailable.asBorrowed(poolLock) + } + + // Slowpath: Create a new entry if capacity limit has not been reached, or wait for the next available entry. + val nextAvailable = poolLock.withLock { + // Reload the list since it could've been updated by other threads concurrently. + val entries = entriesRef.value ?: throw ClosedMultiPoolException + + if (entries.count() < capacity) { + // Capacity hasn't been reached — create a new entry to serve this call. + val newEntry = Entry(producer()) + val done = newEntry.tryToAcquire() + check(done) + + entriesRef.value = (entries + listOf(newEntry)).freeze() + return@withLock newEntry + } else { + // Capacity is reached — wait for the next available entry. + return@withLock loopForConditionalResult { + // Reload the list, since the thread can be suspended here while the list of entries has been modified. + val innerEntries = entriesRef.value ?: throw ClosedMultiPoolException + innerEntries.firstOrNull { it.tryToAcquire() } + } + } + } + + return nextAvailable.asBorrowed(poolLock) + } + + fun access(action: (T) -> R): R { + val borrowed = borrowEntry() + return try { + action(borrowed.value) + } finally { + borrowed.release() + } + } + + fun close() { + if (!poolLock.close()) + return + + val entries = entriesRef.value + val done = entriesRef.compareAndSet(entries, null) + check(done) + + entries?.forEach { it.value.close() } + } + + inner class Entry(val value: T) { + val isAvailable = AtomicBoolean(true) + + init { freeze() } + + fun tryToAcquire(): Boolean = isAvailable.compareAndSet(expected = true, new = false) + + fun asBorrowed(poolLock: PoolLock): Borrowed = object : Borrowed { + override val value: T + get() = this@Entry.value + + override fun release() { + /** + * Mark-as-available should be done before signalling blocked threads via [PoolLock.notifyConditionChanged], + * since the happens-before relationship guarantees the woken thread to see the + * available entry (if not having been taken by other threads during the wake-up lead time). + */ + + val done = isAvailable.compareAndSet(expected = false, new = true) + check(done) + + // While signalling blocked threads does not require locking, doing so avoids a subtle race + // condition in which: + // + // 1. a [loopForConditionalResult] iteration in [borrowEntry] slow path is happening concurrently; + // 2. the iteration fails to see the atomic `isAvailable = true` above; + // 3. we signal availability here but it is a no-op due to no waiting blocker; and finally + // 4. the iteration entered an indefinite blocking wait, not being aware of us having signalled availability here. + // + // By acquiring the pool lock first, signalling cannot happen concurrently with the loop + // iterations in [borrowEntry], thus eliminating the race condition. + poolLock.withLock { + poolLock.notifyConditionChanged() + } + } + } + } +} + +private val ClosedMultiPoolException get() = IllegalStateException("Attempt to access a closed MultiPool.") diff --git a/drivers/native-driver-strict/src/nativeMain/kotlin/app/cash/sqldelight/driver/native/SqliterSqlCursor.kt b/drivers/native-driver-strict/src/nativeMain/kotlin/app/cash/sqldelight/driver/native/SqliterSqlCursor.kt new file mode 100644 index 00000000000..67dad49ac39 --- /dev/null +++ b/drivers/native-driver-strict/src/nativeMain/kotlin/app/cash/sqldelight/driver/native/SqliterSqlCursor.kt @@ -0,0 +1,37 @@ +package app.cash.sqldelight.driver.native + +import app.cash.sqldelight.db.SqlCursor +import co.touchlab.sqliter.Cursor +import co.touchlab.sqliter.getBytesOrNull +import co.touchlab.sqliter.getDoubleOrNull +import co.touchlab.sqliter.getLongOrNull +import co.touchlab.sqliter.getStringOrNull + +/** + * Wrapper for cursor calls. Cursors point to real SQLite statements, so we need to be careful with + * them. If dev closes the outer structure, this will get closed as well, which means it could start + * throwing errors if you're trying to access it. + */ +internal class SqliterSqlCursor( + private val cursor: Cursor, + private val recycler: () -> Unit +) : SqlCursor { + + override fun close() { + recycler() + } + + override fun getBytes(index: Int): ByteArray? = cursor.getBytesOrNull(index) + + override fun getDouble(index: Int): Double? = cursor.getDoubleOrNull(index) + + override fun getLong(index: Int): Long? = cursor.getLongOrNull(index) + + override fun getString(index: Int): String? = cursor.getStringOrNull(index) + + override fun getBoolean(index: Int): Boolean? { + return (cursor.getLongOrNull(index) ?: return null) == 1L + } + + override fun next(): Boolean = cursor.next() +} diff --git a/drivers/native-driver-strict/src/nativeMain/kotlin/app/cash/sqldelight/driver/native/SqliterStatement.kt b/drivers/native-driver-strict/src/nativeMain/kotlin/app/cash/sqldelight/driver/native/SqliterStatement.kt new file mode 100644 index 00000000000..87e7ceec2ee --- /dev/null +++ b/drivers/native-driver-strict/src/nativeMain/kotlin/app/cash/sqldelight/driver/native/SqliterStatement.kt @@ -0,0 +1,42 @@ +package app.cash.sqldelight.driver.native + +import app.cash.sqldelight.db.SqlPreparedStatement +import co.touchlab.sqliter.Statement +import co.touchlab.sqliter.bindBlob +import co.touchlab.sqliter.bindDouble +import co.touchlab.sqliter.bindLong +import co.touchlab.sqliter.bindString + +/** + * @param [recycle] A function which recycles any resources this statement is backed by. + */ +internal class SqliterStatement( + private val statement: Statement +) : SqlPreparedStatement { + override fun bindBytes(index: Int, bytes: ByteArray?) { + statement.bindBlob(index, bytes) + } + + override fun bindLong(index: Int, long: Long?) { + statement.bindLong(index, long) + } + + override fun bindDouble(index: Int, double: Double?) { + statement.bindDouble(index, double) + } + + override fun bindString(index: Int, string: String?) { + statement.bindString(index, string) + } + + override fun bindBoolean(index: Int, boolean: Boolean?) { + statement.bindLong( + index, + when (boolean) { + null -> null + true -> 1L + false -> 0L + } + ) + } +} diff --git a/drivers/native-driver-strict/src/nativeMain/kotlin/app/cash/sqldelight/driver/native/util/NativeCache.kt b/drivers/native-driver-strict/src/nativeMain/kotlin/app/cash/sqldelight/driver/native/util/NativeCache.kt new file mode 100644 index 00000000000..1fa67375ce3 --- /dev/null +++ b/drivers/native-driver-strict/src/nativeMain/kotlin/app/cash/sqldelight/driver/native/util/NativeCache.kt @@ -0,0 +1,10 @@ +package app.cash.sqldelight.driver.native.util + +internal interface NativeCache { + fun put(key: String, value: T?): T? + fun getOrCreate(key: String, block: () -> T): T + fun remove(key: String): T? + fun cleanUp(block: (T) -> Unit) +} + +internal expect fun nativeCache(): NativeCache diff --git a/drivers/native-driver-strict/src/nativeMain/kotlin/app/cash/sqldelight/driver/native/util/PoolLock.kt b/drivers/native-driver-strict/src/nativeMain/kotlin/app/cash/sqldelight/driver/native/util/PoolLock.kt new file mode 100644 index 00000000000..ac852d0f82e --- /dev/null +++ b/drivers/native-driver-strict/src/nativeMain/kotlin/app/cash/sqldelight/driver/native/util/PoolLock.kt @@ -0,0 +1,32 @@ +package app.cash.sqldelight.driver.native.util + +@Suppress("NO_ACTUAL_FOR_EXPECT") +internal expect class PoolLock() { + fun withLock( + action: CriticalSection.() -> R + ): R + + /** + * Select one blocked thread in [CriticalSection.loopForConditionalResult] to be woken up for + * re-evaluation, if any. + */ + fun notifyConditionChanged() + + fun close(): Boolean + + inner class CriticalSection { + /** + * Evaluate the given lambda of a conditional result in an infinite loop, until the result is + * available. + * + * If null is produced, the current thread enters suspension, and are only woken up for + * re-evaluation by a subsequent [PoolLock.notifyConditionChanged] call. Note that the lock + * would not be held by the current thread during its suspension. This allows resources + * protected by the same lock to remain accessible by other threads, provided that they do not + * depend on the same conditional result. + */ + fun loopForConditionalResult(block: () -> R?): R + + fun loopUntilConditionalResult(block: () -> Boolean) + } +} diff --git a/drivers/native-driver-strict/src/nativeTest/kotlin/com/squareup/sqldelight/drivers/native/LazyDriverBaseTest.kt b/drivers/native-driver-strict/src/nativeTest/kotlin/com/squareup/sqldelight/drivers/native/LazyDriverBaseTest.kt new file mode 100644 index 00000000000..b008d712700 --- /dev/null +++ b/drivers/native-driver-strict/src/nativeTest/kotlin/com/squareup/sqldelight/drivers/native/LazyDriverBaseTest.kt @@ -0,0 +1,108 @@ +package com.squareup.sqldelight.drivers.native + +import app.cash.sqldelight.TransacterImpl +import app.cash.sqldelight.db.SqlDriver +import app.cash.sqldelight.driver.native.NativeSqliteDriver +import app.cash.sqldelight.driver.native.wrapConnection +import co.touchlab.sqliter.DatabaseConfiguration +import co.touchlab.sqliter.DatabaseFileContext.deleteDatabase +import co.touchlab.sqliter.DatabaseManager +import co.touchlab.sqliter.createDatabaseManager +import co.touchlab.stately.freeze +import kotlin.test.AfterTest +import kotlin.test.BeforeTest + +abstract class LazyDriverBaseTest { + protected lateinit var driver: NativeSqliteDriver + private var manager: DatabaseManager? = null + + protected abstract val memory: Boolean + + private val transacterInternal: TransacterImpl by lazy { + object : TransacterImpl(driver) {} + } + + protected val transacter: TransacterImpl + get() { + val t = transacterInternal + t.freeze() + return t + } + + @BeforeTest fun setup() { + driver = setupDatabase(schema = defaultSchema()) + } + + @AfterTest fun tearDown() { + driver.close() + } + + protected fun defaultSchema(): SqlDriver.Schema { + return object : SqlDriver.Schema { + override val version: Int = 1 + + override fun create(driver: SqlDriver) { + driver.execute( + 20, + """ + |CREATE TABLE test ( + | id INTEGER PRIMARY KEY, + | value TEXT + |); + """.trimMargin(), + 0 + ) + driver.execute( + 30, + """ + |CREATE TABLE nullability_test ( + | id INTEGER PRIMARY KEY, + | integer_value INTEGER, + | text_value TEXT, + | blob_value BLOB, + | real_value REAL + |); + """.trimMargin(), + 0 + ) + } + + override fun migrate( + driver: SqlDriver, + oldVersion: Int, + newVersion: Int + ) { + // No-op. + } + } + } + + protected fun altInit(config: DatabaseConfiguration) { + driver.close() + driver = setupDatabase(defaultSchema(), config) + } + + private fun setupDatabase( + schema: SqlDriver.Schema, + config: DatabaseConfiguration = defaultConfiguration(schema) + ): NativeSqliteDriver { + deleteDatabase(config.name!!) + // This isn't pretty, but just for test + manager = createDatabaseManager(config) + return NativeSqliteDriver(manager!!) + } + + protected fun defaultConfiguration(schema: SqlDriver.Schema): DatabaseConfiguration { + return DatabaseConfiguration( + name = "testdb", + version = 1, + create = { connection -> + wrapConnection(connection) { + schema.create(it) + } + }, + extendedConfig = DatabaseConfiguration.Extended(busyTimeout = 20_000), + inMemory = true + ) + } +} diff --git a/drivers/native-driver-strict/src/nativeTest/kotlin/com/squareup/sqldelight/drivers/native/NativeDriverTest.kt b/drivers/native-driver-strict/src/nativeTest/kotlin/com/squareup/sqldelight/drivers/native/NativeDriverTest.kt new file mode 100644 index 00000000000..9b1d04141d7 --- /dev/null +++ b/drivers/native-driver-strict/src/nativeTest/kotlin/com/squareup/sqldelight/drivers/native/NativeDriverTest.kt @@ -0,0 +1,14 @@ +package com.squareup.sqldelight.drivers.native + +import app.cash.sqldelight.db.SqlDriver +import app.cash.sqldelight.driver.native.NativeSqliteDriver +import co.touchlab.sqliter.DatabaseFileContext.deleteDatabase +import com.squareup.sqldelight.driver.test.DriverTest + +class NativeDriverTest : DriverTest() { + override fun setupDatabase(schema: SqlDriver.Schema): SqlDriver { + val name = "testdb" + deleteDatabase(name) + return NativeSqliteDriver(schema, name) + } +} diff --git a/drivers/native-driver-strict/src/nativeTest/kotlin/com/squareup/sqldelight/drivers/native/NativeQueryTest.kt b/drivers/native-driver-strict/src/nativeTest/kotlin/com/squareup/sqldelight/drivers/native/NativeQueryTest.kt new file mode 100644 index 00000000000..62669adc566 --- /dev/null +++ b/drivers/native-driver-strict/src/nativeTest/kotlin/com/squareup/sqldelight/drivers/native/NativeQueryTest.kt @@ -0,0 +1,14 @@ +package com.squareup.sqldelight.drivers.native + +import app.cash.sqldelight.db.SqlDriver +import app.cash.sqldelight.driver.native.NativeSqliteDriver +import co.touchlab.sqliter.DatabaseFileContext +import com.squareup.sqldelight.driver.test.QueryTest + +class NativeQueryTest : QueryTest() { + override fun setupDatabase(schema: SqlDriver.Schema): SqlDriver { + val name = "testdb" + DatabaseFileContext.deleteDatabase(name) + return NativeSqliteDriver(schema, name) + } +} diff --git a/drivers/native-driver-strict/src/nativeTest/kotlin/com/squareup/sqldelight/drivers/native/NativeSqliteDriverTest.kt.ignore b/drivers/native-driver-strict/src/nativeTest/kotlin/com/squareup/sqldelight/drivers/native/NativeSqliteDriverTest.kt.ignore new file mode 100644 index 00000000000..64eaaa3d620 --- /dev/null +++ b/drivers/native-driver-strict/src/nativeTest/kotlin/com/squareup/sqldelight/drivers/native/NativeSqliteDriverTest.kt.ignore @@ -0,0 +1,412 @@ +package com.squareup.sqldelight.drivers.native + +import co.touchlab.sqliter.DatabaseConfiguration +import co.touchlab.sqliter.createDatabaseManager +import co.touchlab.stately.collections.frozenLinkedList +import co.touchlab.stately.concurrency.AtomicBoolean +import co.touchlab.stately.concurrency.AtomicInt +import co.touchlab.testhelp.concurrency.ThreadOperations +import co.touchlab.testhelp.concurrency.sleep +import app.cash.sqldelight.Transacter +import app.cash.sqldelight.db.SqlCursor +import kotlin.test.Test +import kotlin.test.assertEquals +import kotlin.test.assertFails +import kotlin.test.assertFalse +import kotlin.test.assertSame +import kotlin.test.assertFails + +//Run tests with WAL db +class NativeSqliteDriverTestWAL : NativeSqliteDriverTest() { + override val memory: Boolean = false +} + +//Run tests with memory db +class NativeSqliteDriverTestMemory : NativeSqliteDriverTest() { + override val memory: Boolean = true + + @Test + fun `wrapConnection does not close connection`() { + val closed = AtomicBoolean(true) + val config = DatabaseConfiguration( + name = "memorydb", + version = 1, + inMemory = true, + create = { connection -> + wrapConnection(connection) { + defaultSchema().create(it) + } + + closed.value = connection.closed + }) + val dbm = createDatabaseManager(config) + dbm.createMultiThreadedConnection().close() + + assertFalse(closed.value) + } +} + +abstract class NativeSqliteDriverTest : LazyDriverBaseTest() { + + /*@Test + fun `close with open transaction fails`(){ + transacter.transaction { + assertFails { driver.close() } + } + + //Still working? There's probably a better general test for this. + driver.queryPool.access { + val stmt = it.getStatement(null, "select * from test") + stmt.finalizeStatement() + } + }*/ + + //Kind of a sanity check + @Test + fun `threads share statement main connection multithreaded`() { + altInit(defaultConfiguration(defaultSchema()).copy(inMemory = true)) + val ops = ThreadOperations { } + val INSERTS = 10_000 + for (i in 0 until INSERTS) { + ops.exe { + driver.execute(1, "insert into test(id, value)values(?, ?)", 2) { + bindLong(1, i.toLong()) + bindString(2, "Hey $i") + } + } + } + + ops.run(10) + + assertEquals(INSERTS.toLong(), countTestRows(driver)) + val strSet = mutableSetOf() + val query = driver.executeQuery(2, "select id, value from test", 0) + var sum = 0L + while (query.next()) { + strSet.add(query.getString(1)!!) + sum += query.getLong(0)!! + } + + assertEquals(sum, (0 until INSERTS).fold(0L) { a, b -> a + b }) + assertEquals(INSERTS, strSet.size) + } + + @Test + fun `failing transaction clears lock`() { + assertFails { + transacter.transaction { + driver.execute(1, "insert into test(id, value)values(?, ?)", 2) { + bindLong(1, 1) + bindString(2, "asdf") + } + + throw IllegalStateException("Fail") + } + } + + transacter.transaction { + try { + driver.execute(1, "insert into test(id, value)values(?, ?)", 2) { + bindLong(1, 1) + bindString(2, "asdf") + } + + } catch (e: Exception) { + e.printStackTrace() + throw e + } + } + + assertEquals(1, countTestRows(driver)) + } + + @Test + fun `bad bind doens't taint future binding`() { + transacter.transaction { + assertFails { + driver.execute(1, "insert into test(id, value)values(?, ?)", 2) { + bindLong(1, 1) + bindString(3, "asdf") + } + } + } + + transacter.transaction { + driver.execute(1, "insert into test(id, value)values(?, ?)", 2) { + bindLong(1, 1) + bindString(2, "asdf") + } + } + + assertEquals(1, countTestRows(driver)) + } + + @Test + fun `failed bind dumps sqlite statement`() { + assertFails { + driver.execute(1, "insert into test(id, value) values(?, ?)", 2) { + assertEquals(0, driver.queryPool.entry.statementCache.size) + bindLong(1, 1L) + throw assertFails { bindLong(3, 1L) } + } + } + assertEquals(1, driver.queryPool.entry.statementCache.size) + } + + @Test + fun `failures don't leak resources`() { + val transacter = transacter + + val ops = ThreadOperations { } + val threads = 3 + for (i in 1..10) { + ops.exe { + exeQuiet { + transacter.transaction { + + for (i in 0 until 10) { + driver.execute(1, "insert into test(id, value)values(?, ?)", 2) { + bindLong(1, i.toLong()) + bindString(2, "Hey $i") + } + } + + throw IllegalStateException("Nah") + } + } + + exeQuiet { + driver.execute(1, "insert into test(id, value)values(?, ?)", 2) { + bindLong(1, i.toLong()) + bindString(3, "Hey $i") + } + } + + driver.executeQuery(2, "select id, value from test", 0).next() + + exeQuiet { + driver.executeQuery(3, "select id, value from toast", 0).next() + } + } + } + + ops.run(threads) + + assertEquals(10, driver.queryPool.entry.cursorCollection.size) + assertEquals(0, countTestRows(driver)) + + //If we've leaked anything the test cleanup will fail... + } + + fun exeQuiet(proc: () -> Unit) { + try { + proc() + } catch (e: Exception) { + } + } + + @Test + fun `multiple thread transactions wait and complete successfully`() { + val THREADS = 25 + val LOOPS = 50 + + val GLOBALLOOPS = 100 + + for (i in 0 until GLOBALLOOPS) { + insertThreadLoop(THREADS * LOOPS * i, THREADS, transacter, LOOPS) + } + + assertEquals((THREADS * LOOPS * GLOBALLOOPS).toLong(), countTestRows(driver)) + } + + private fun countTestRows(conn: NativeSqliteDriver): Long { + val query = conn.executeQuery(10, "select count(*) from test", 0) + query.next() + val count = query.getLong(0) + query.close() + return count!! + } + + private fun insertThreadLoop( + start: Int, + THREADS: Int, + transacter: Transacter, + LOOPS: Int + ) { + val ops = ThreadOperations { driver } + + for (i in 0 until THREADS) { + ops.exe { conn -> + + transacter.transaction { + try { + for (j in 0 until LOOPS) { + conn.execute(1, "insert into test(id, value)values(?, ?)", 2) { + val idInt = i * LOOPS + j + start + bindLong(1, idInt.toLong()) + bindString(2, "row $idInt") + } + } + } catch (e: Throwable) { + e.printStackTrace() + throw e + } + } + } + } + + ops.run(THREADS) + } + + @Test + fun `query statements cached but only 1`() { + val stmt = { driver.executeQuery(1, "select * from test", 0) } + + assertEquals(0, driver.queryPool.entry.statementCache.size) + assertEquals(0, driver.queryPool.entry.cursorCollection.size) + + val query = stmt() + + assertEquals(0, driver.queryPool.entry.statementCache.size) + assertEquals(1, driver.queryPool.entry.cursorCollection.size) + query.close() + + assertEquals(1, driver.queryPool.entry.statementCache.size) + assertEquals(0, driver.queryPool.entry.cursorCollection.size) + + val queryA = stmt() + val queryB = stmt() + + assertEquals(0, driver.queryPool.entry.statementCache.size) + assertEquals(2, driver.queryPool.entry.cursorCollection.size) + + queryA.close() + queryB.close() + + assertEquals(1, driver.queryPool.entry.statementCache.size) + assertEquals(0, driver.queryPool.entry.cursorCollection.size) + + val ops = ThreadOperations { stmt } + val THREAD = 4 + val collectCursors = frozenLinkedList() + for (i in 0 until THREAD) { + ops.exe { + collectCursors.add(stmt()) + } + } + + ops.run(THREAD) + + assertEquals(0, driver.queryPool.entry.statementCache.size) + assertEquals(THREAD, driver.queryPool.entry.cursorCollection.size) + collectCursors.forEach { it.close() } + assertEquals(1, driver.queryPool.entry.statementCache.size) + assertEquals(0, driver.queryPool.entry.cursorCollection.size) + } + + @Test + fun `query exception clears statement`() { + assertFails { + driver.executeQuery(1, "select * from test", 0) { + throw assertFails { bindLong(1, 2L) } + } + } + + assertEquals(1, driver.queryPool.entry.statementCache.size) + } + + @Test + fun `SinglePool access locked`() { + val ops = ThreadOperations { SinglePool { AtomicInt(0) } } + val failed = AtomicBoolean(false) + for (i in 0 until 150) { + ops.exe { + it.access { + val atStart = it.incrementAndGet() + if (atStart != 1) + failed.value = true + + sleep(10) + + val atEnd = it.decrementAndGet() + if (atEnd != 0) + failed.value = true + } + } + } + + ops.run(5) + assertFalse(failed.value) + } + + @Test + fun `SinglePool re-borrow fails`() { + val pool = SinglePool {} + val borrowed = pool.borrowEntry() + assertFails { pool.borrowEntry() } + borrowed.release() + } + + @Test + fun `caching by index works as expected`() { + val transacter = transacter + driver.execute(1, "insert into test(id, value)values(?, ?)", 2) { + bindLong(1, 22L) + bindString(2, "Hey 22") + } + + assertEquals(1, driver.queryPool.entry.statementCache.size) + assertEquals(0, driver.transactionPool.entry.statementCache.size) + + transacter.transaction { + driver.execute(1, "insert into test(id, value)values(?, ?)", 2) { + bindLong(1, 33L) + bindString(2, "Hey 33") + } + } + + assertEquals(1, driver.queryPool.entry.statementCache.size) + assertEquals(1, driver.transactionPool.entry.statementCache.size) + + val statement = + driver.transactionPool.entry.statementCache.entries.iterator().next().value + + transacter.transaction { + driver.execute(1, "insert into test(id, value)values(?, ?)", 2) { + bindLong(1, 34L) + bindString(2, "Hey 34") + } + } + + assertEquals(1, driver.queryPool.entry.statementCache.size) + assertEquals(1, driver.transactionPool.entry.statementCache.size) + + assertSame( + driver.transactionPool.entry.statementCache.entries.iterator().next().value, + statement) + } + + @Test + fun `null identifier doesn't cache`() { + val transacter = transacter + driver.execute(null, "insert into test(id, value)values(?, ?)", 2) { + bindLong(1, 22L) + bindString(2, "Hey 22") + } + + + assertEquals(0, driver.queryPool.entry.statementCache.size) + assertEquals(0, driver.transactionPool.entry.statementCache.size) + + transacter.transaction { + driver.execute(null, "insert into test(id, value)values(?, ?)", 2) { + bindLong(1, 23L) + bindString(2, "Hey 23") + } + } + + assertEquals(0, driver.queryPool.entry.statementCache.size) + assertEquals(0, driver.transactionPool.entry.statementCache.size) + } + +} \ No newline at end of file diff --git a/drivers/native-driver-strict/src/nativeTest/kotlin/com/squareup/sqldelight/drivers/native/NativeTransacterTest.kt b/drivers/native-driver-strict/src/nativeTest/kotlin/com/squareup/sqldelight/drivers/native/NativeTransacterTest.kt new file mode 100644 index 00000000000..820046d2721 --- /dev/null +++ b/drivers/native-driver-strict/src/nativeTest/kotlin/com/squareup/sqldelight/drivers/native/NativeTransacterTest.kt @@ -0,0 +1,14 @@ +package com.squareup.sqldelight.drivers.native + +import app.cash.sqldelight.db.SqlDriver +import app.cash.sqldelight.driver.native.NativeSqliteDriver +import co.touchlab.sqliter.DatabaseFileContext.deleteDatabase +import com.squareup.sqldelight.driver.test.TransacterTest + +class NativeTransacterTest : TransacterTest() { + override fun setupDatabase(schema: SqlDriver.Schema): SqlDriver { + val name = "testdb" + deleteDatabase(name) + return NativeSqliteDriver(schema, name) + } +} diff --git a/drivers/native-driver-strict/src/nativeTest/kotlin/com/squareup/sqldelight/drivers/native/connectionpool/BaseConcurrencyTest.kt b/drivers/native-driver-strict/src/nativeTest/kotlin/com/squareup/sqldelight/drivers/native/connectionpool/BaseConcurrencyTest.kt new file mode 100644 index 00000000000..6790d461f89 --- /dev/null +++ b/drivers/native-driver-strict/src/nativeTest/kotlin/com/squareup/sqldelight/drivers/native/connectionpool/BaseConcurrencyTest.kt @@ -0,0 +1,175 @@ +package com.squareup.sqldelight.drivers.native.connectionpool + +import app.cash.sqldelight.db.SqlDriver +import app.cash.sqldelight.driver.native.NativeSqliteDriver +import app.cash.sqldelight.driver.native.wrapConnection +import co.touchlab.sqliter.DatabaseConfiguration +import co.touchlab.sqliter.DatabaseFileContext +import co.touchlab.sqliter.JournalMode +import co.touchlab.testhelp.concurrency.currentTimeMillis +import co.touchlab.testhelp.concurrency.sleep +import kotlin.native.concurrent.AtomicInt +import kotlin.native.concurrent.Worker +import kotlin.test.AfterTest + +abstract class BaseConcurrencyTest { + fun countRows(myDriver: SqlDriver = driver): Long { + val cur = myDriver.executeQuery(0, "SELECT count(*) FROM test", 0) + try { + cur.next() + val count = cur.getLong(0) + return count!! + } finally { + cur.close() + } + } + + private var _driver: SqlDriver? = null + private var dbName: String? = null + internal val driver: SqlDriver + get() = _driver!! + + internal inner class ConcurrentContext { + private val myWorkers = arrayListOf() + internal fun createWorker(): Worker { + val w = Worker.start() + myWorkers.add(w) + return w + } + + internal fun stopWorkers() { + myWorkers.forEach { it.requestTermination() } + } + } + + internal fun runConcurrent(block: ConcurrentContext.() -> Unit) { + val context = ConcurrentContext() + try { + context.block() + } finally { + context.stopWorkers() + } + } + + fun setupDatabase( + schema: SqlDriver.Schema, + dbType: DbType, + configBase: DatabaseConfiguration, + maxReaderConnections: Int = 4 + ): SqlDriver { + // Some failing tests can leave the db in a weird state, so on each run we have a different db per test + val name = "testdb_${globalDbCount.addAndGet(1)}" + dbName = name + DatabaseFileContext.deleteDatabase(name) + val configCommon = configBase.copy( + name = name, + version = 1, + create = { conn -> + wrapConnection(conn) { driver -> + schema.create(driver) + } + } + ) + return when (dbType) { + DbType.RegularWal -> { + NativeSqliteDriver( + configCommon, + maxReaderConnections = maxReaderConnections + ) + } + DbType.RegularDelete -> { + val config = configCommon.copy(journalMode = JournalMode.DELETE) + NativeSqliteDriver( + config, + maxReaderConnections = maxReaderConnections + ) + } + DbType.InMemoryShared -> { + val config = configCommon.copy(inMemory = true) + NativeSqliteDriver( + config, + maxReaderConnections = maxReaderConnections + ) + } + DbType.InMemorySingle -> { + val config = configCommon.copy(name = null, inMemory = true) + NativeSqliteDriver( + config, + maxReaderConnections = maxReaderConnections + ) + } + } + } + + enum class DbType { + RegularWal, RegularDelete, InMemoryShared, InMemorySingle + } + + fun createDriver( + dbType: DbType, + configBase: DatabaseConfiguration = DatabaseConfiguration(name = null, version = 1, create = {}), + ): SqlDriver { + return setupDatabase( + schema = object : SqlDriver.Schema { + override val version: Int = 1 + + override fun create(driver: SqlDriver) { + driver.execute( + null, + """ + CREATE TABLE test ( + id INTEGER NOT NULL PRIMARY KEY, + value TEXT NOT NULL + ); + """.trimIndent(), + 0 + ) + } + + override fun migrate( + driver: SqlDriver, + oldVersion: Int, + newVersion: Int + ) { + // No-op. + } + }, + dbType, + configBase + ) + } + + internal fun waitFor(timeout: Long = 10_000, block: () -> Boolean) { + val start = currentTimeMillis() + var wasTimeout = false + + while (!block() && !wasTimeout) { + sleep(200) + wasTimeout = (currentTimeMillis() - start) > timeout + } + + if (wasTimeout) + throw IllegalStateException("Timeout $timeout exceeded") + } + + fun initDriver(dbType: DbType) { + _driver = createDriver(dbType) + } + + @AfterTest + fun tearDown() { + _driver?.close() + dbName?.let { DatabaseFileContext.deleteDatabase(it) } + } + + internal fun insertTestData(testData: TestData, driver: SqlDriver = this.driver) { + driver.execute(1, "INSERT INTO test VALUES (?, ?)", 2) { + bindLong(1, testData.id) + bindString(2, testData.value) + } + } + + internal data class TestData(val id: Long, val value: String) +} + +private val globalDbCount = AtomicInt(0) diff --git a/drivers/native-driver-strict/src/nativeTest/kotlin/com/squareup/sqldelight/drivers/native/connectionpool/NativeSqliteDriverConfigTest.kt b/drivers/native-driver-strict/src/nativeTest/kotlin/com/squareup/sqldelight/drivers/native/connectionpool/NativeSqliteDriverConfigTest.kt new file mode 100644 index 00000000000..c92e938c468 --- /dev/null +++ b/drivers/native-driver-strict/src/nativeTest/kotlin/com/squareup/sqldelight/drivers/native/connectionpool/NativeSqliteDriverConfigTest.kt @@ -0,0 +1,19 @@ +package com.squareup.sqldelight.drivers.native.connectionpool + +import app.cash.sqldelight.driver.native.NativeSqliteDriver +import kotlin.test.Test +import kotlin.test.assertEquals + +/** + * Testing driver config impact on internals. This is basically verifying that connection max config is set properly + * internally. + */ +class NativeSqliteDriverConfigTest : BaseConcurrencyTest() { + @Test + fun limitReaderConnectionsForMemoryDb() { + assertEquals(1, (createDriver(DbType.InMemorySingle) as NativeSqliteDriver).readerPool.capacity) + assertEquals(1, (createDriver(DbType.InMemoryShared) as NativeSqliteDriver).readerPool.capacity) + assertEquals(4, (createDriver(DbType.RegularWal) as NativeSqliteDriver).readerPool.capacity) + assertEquals(4, (createDriver(DbType.RegularDelete) as NativeSqliteDriver).readerPool.capacity) + } +} diff --git a/drivers/native-driver-strict/src/nativeTest/kotlin/com/squareup/sqldelight/drivers/native/connectionpool/WalConcurrencyTest.kt b/drivers/native-driver-strict/src/nativeTest/kotlin/com/squareup/sqldelight/drivers/native/connectionpool/WalConcurrencyTest.kt new file mode 100644 index 00000000000..fc7d9c8fec0 --- /dev/null +++ b/drivers/native-driver-strict/src/nativeTest/kotlin/com/squareup/sqldelight/drivers/native/connectionpool/WalConcurrencyTest.kt @@ -0,0 +1,202 @@ +package com.squareup.sqldelight.drivers.native.connectionpool + +import app.cash.sqldelight.Query +import app.cash.sqldelight.TransacterImpl +import app.cash.sqldelight.db.SqlCursor +import app.cash.sqldelight.driver.native.NativeSqliteDriver +import co.touchlab.testhelp.concurrency.ThreadOperations +import co.touchlab.testhelp.concurrency.sleep +import kotlin.native.concurrent.AtomicInt +import kotlin.native.concurrent.TransferMode +import kotlin.native.concurrent.Worker +import kotlin.native.concurrent.freeze +import kotlin.test.BeforeTest +import kotlin.test.Ignore +import kotlin.test.Test +import kotlin.test.assertEquals +import kotlin.test.assertTrue + +/** + * Testing multiple read and transaction pool connections. These were + * written when it a was a single pool, so this will need some refactor. + * The reader pool is much more likely to be multiple with a single transaction pool + * connection, which removes a lot of the potential concurrency issues, but introduces new things + * we should probably test. + */ +class WalConcurrencyTest : BaseConcurrencyTest() { + @BeforeTest + fun setup() { + initDriver(DbType.RegularWal) + } + + /** + * This is less important now that we have a separate reader pool again, but will revisit. + */ + @Test + fun writeNotBlockRead() { + assertEquals(countRows(), 0) + + val transacter: TransacterImpl = object : TransacterImpl(driver) {} + val worker = Worker.start() + val counter = AtomicInt(0) + val transactionStarted = AtomicInt(0) + + val block = { + transacter.transaction { + insertTestData(TestData(1L, "arst 1")) + transactionStarted.increment() + sleep(1500) + counter.increment() + } + } + + val future = worker.execute(TransferMode.SAFE, { block.freeze() }) { it() } + + // When ready, transaction started but sleeping + waitFor { transactionStarted.value > 0 } + + // These three should run before transaction is done (not blocking) + assertEquals(counter.value, 0) + assertEquals(0L, countRows()) + assertEquals(counter.value, 0) + + future.result + + worker.requestTermination() + } + + /** + * Reader pool stress test + */ + @Test @Ignore + fun manyReads() = runConcurrent { + val transacter: TransacterImpl = object : TransacterImpl(driver) {} + val dataSize = 2_000 + transacter.transaction { + repeat(dataSize) { + insertTestData(TestData(it.toLong(), "Data $it")) + } + } + + val ops = ThreadOperations {} + val totalCount = AtomicInt(0) + val queryRuns = 100 + repeat(queryRuns) { + ops.exe { + totalCount.addAndGet(testDataQuery().executeAsList().size) + } + } + ops.run(6) + assertEquals(totalCount.value, dataSize * queryRuns) + val readerPool = (driver as NativeSqliteDriver).readerPool + // Make sure we actually created all of the connections + assertTrue(readerPool.entryCount() > 1, "Reader pool size ${readerPool.entryCount()}") + } + + private val mapper = { cursor: SqlCursor -> + TestData( + cursor.getLong(0)!!, cursor.getString(1)!! + ) + } + + private fun testDataQuery(): Query { + return object : Query(mapper) { + override fun execute(): SqlCursor { + return driver.executeQuery(0, "SELECT * FROM test", 0) + } + + override fun addListener(listener: Listener) { + driver.addListener(listener, arrayOf("test")) + } + + override fun removeListener(listener: Listener) { + driver.removeListener(listener, arrayOf("test")) + } + } + } + + @Test + fun writeBlocksWrite() { + val transacter: TransacterImpl = object : TransacterImpl(driver) {} + val worker = Worker.start() + val counter = AtomicInt(0) + val transactionStarted = AtomicInt(0) + + val block = { + transacter.transaction { + insertTestData(TestData(1L, "arst 1")) + transactionStarted.increment() + sleep(1500) + counter.increment() + } + } + + val future = worker.execute(TransferMode.SAFE, { block.freeze() }) { it() } + + // Transaction with write started but sleeping + waitFor { transactionStarted.value > 0 } + + assertEquals(counter.value, 0) + insertTestData(TestData(2L, "arst 2")) // This waits on transaction to wrap up + assertEquals(counter.value, 1) // Counter would be zero if write didn't block (see above) + + future.result + + worker.requestTermination() + } + + @Test + fun multipleWritesDontTimeOut() { + val transacter: TransacterImpl = object : TransacterImpl(driver) {} + val worker = Worker.start() + val transactionStarted = AtomicInt(0) + + val block = { + transacter.transaction { + insertTestData(TestData(1L, "arst 1"), driver) + transactionStarted.increment() + sleep(1500) + insertTestData(TestData(5L, "arst 1"), driver) + } + } + + val future = worker.execute(TransferMode.SAFE, { block.freeze() }) { it() } + + // When we get here, first transaction has run a write command, and is sleeping + waitFor { transactionStarted.value > 0 } + transacter.transaction { + insertTestData(TestData(2L, "arst 2"), driver) + } + + future.result + worker.requestTermination() + } + + /** + * Just a bunch of inserts on multiple threads. More of a stress test. + */ + @Test + fun multiWrite() { + val ops = ThreadOperations {} + val times = 10_000 + val transacter: TransacterImpl = object : TransacterImpl(driver) {} + + repeat(times) { index -> + ops.exe { + transacter.transaction { + insertTestData(TestData(index.toLong(), "arst $index")) + + val id2 = index.toLong() + times + insertTestData(TestData(id2, "arst $id2")) + + val id3 = index.toLong() + times + times + insertTestData(TestData(id3, "arst $id3")) + } + } + } + + ops.run(10) + + assertEquals(countRows(), times.toLong() * 3) + } +} diff --git a/drivers/native-driver/build.gradle b/drivers/native-driver/build.gradle index 0eabe491090..2341ca6ffa4 100644 --- a/drivers/native-driver/build.gradle +++ b/drivers/native-driver/build.gradle @@ -55,14 +55,11 @@ kotlin { nativeDarwinMain{ dependsOn(nativeMain) } - mingwMain{ - dependsOn(nativeMain) - } mingwX86Main{ - dependsOn(mingwMain) + dependsOn(nativeMain) } mingwX64Main{ - dependsOn(mingwMain) + dependsOn(nativeMain) } linuxMain{ dependsOn(nativeMain) @@ -86,7 +83,7 @@ kotlin { } configure([targets.mingwX64]) { - sourceSets.getByName("${name}Main").dependsOn(sourceSets.mingwMain) + sourceSets.getByName("${name}Main").dependsOn(sourceSets.nativeMain) sourceSets.getByName("${name}Test").dependsOn(sourceSets.nativeTest) compilations.test { kotlinOptions.freeCompilerArgs += ["-linker-options", "-Lc:\\msys64\\mingw64\\lib -L$rootDir\\libs -lsqlite3".toString()] @@ -94,7 +91,7 @@ kotlin { } configure([targets.mingwX86]) { - sourceSets.getByName("${name}Main").dependsOn(sourceSets.mingwMain) + sourceSets.getByName("${name}Main").dependsOn(sourceSets.nativeMain) sourceSets.getByName("${name}Test").dependsOn(sourceSets.nativeTest) compilations.test { kotlinOptions.freeCompilerArgs += ["-linker-options", "-Lc:\\msys32\\mingw32\\lib -L$rootDir\\libs -lsqlite3".toString()] diff --git a/drivers/native-driver/src/nativeMain/kotlin/app/cash/sqldelight/driver/native/NativeSqlDatabase.kt b/drivers/native-driver/src/nativeMain/kotlin/app/cash/sqldelight/driver/native/NativeSqlDatabase.kt index 79a2f5a393b..ea10dfb16bc 100644 --- a/drivers/native-driver/src/nativeMain/kotlin/app/cash/sqldelight/driver/native/NativeSqlDatabase.kt +++ b/drivers/native-driver/src/nativeMain/kotlin/app/cash/sqldelight/driver/native/NativeSqlDatabase.kt @@ -6,7 +6,7 @@ import app.cash.sqldelight.db.Closeable import app.cash.sqldelight.db.SqlCursor import app.cash.sqldelight.db.SqlDriver import app.cash.sqldelight.db.SqlPreparedStatement -import app.cash.sqldelight.driver.native.util.nativeCache +import app.cash.sqldelight.driver.native.util.MutableCache import co.touchlab.sqliter.DatabaseConfiguration import co.touchlab.sqliter.DatabaseConnection import co.touchlab.sqliter.DatabaseManager @@ -314,7 +314,7 @@ internal class ThreadConnection( internal val closed: Boolean get() = connection.closed - internal val statementCache = nativeCache() + internal val statementCache = MutableCache() fun safePut(identifier: Int?, statement: Statement) { val removed = if (identifier == null) { diff --git a/drivers/native-driver/src/nativeMain/kotlin/app/cash/sqldelight/driver/native/Pool.kt b/drivers/native-driver/src/nativeMain/kotlin/app/cash/sqldelight/driver/native/Pool.kt index 9f213039fa4..ee22dcd4c57 100644 --- a/drivers/native-driver/src/nativeMain/kotlin/app/cash/sqldelight/driver/native/Pool.kt +++ b/drivers/native-driver/src/nativeMain/kotlin/app/cash/sqldelight/driver/native/Pool.kt @@ -4,7 +4,6 @@ import app.cash.sqldelight.db.Closeable import app.cash.sqldelight.driver.native.util.PoolLock import co.touchlab.stately.concurrency.AtomicBoolean import kotlin.native.concurrent.AtomicReference -import kotlin.native.concurrent.freeze /** * A shared pool of connections. Borrowing is blocking when all connections are in use, and the pool has reached its @@ -14,7 +13,7 @@ internal class Pool(internal val capacity: Int, private val produ /** * Hold a list of active connections. If it is null, it means the MultiPool has been closed. */ - private val entriesRef = AtomicReference?>(listOf().freeze()) + private val entriesRef = AtomicReference?>(listOf()) private val poolLock = PoolLock() /** @@ -45,7 +44,7 @@ internal class Pool(internal val capacity: Int, private val produ val done = newEntry.tryToAcquire() check(done) - entriesRef.value = (entries + listOf(newEntry)).freeze() + entriesRef.value = (entries + listOf(newEntry)) return@withLock newEntry } else { // Capacity is reached — wait for the next available entry. @@ -83,8 +82,6 @@ internal class Pool(internal val capacity: Int, private val produ inner class Entry(val value: T) { val isAvailable = AtomicBoolean(true) - init { freeze() } - fun tryToAcquire(): Boolean = isAvailable.compareAndSet(expected = true, new = false) fun asBorrowed(poolLock: PoolLock): Borrowed = object : Borrowed { diff --git a/drivers/native-driver/src/nativeMain/kotlin/app/cash/sqldelight/driver/native/util/NativeCache.kt b/drivers/native-driver/src/nativeMain/kotlin/app/cash/sqldelight/driver/native/util/NativeCache.kt index 1fa67375ce3..85e2a95afb2 100644 --- a/drivers/native-driver/src/nativeMain/kotlin/app/cash/sqldelight/driver/native/util/NativeCache.kt +++ b/drivers/native-driver/src/nativeMain/kotlin/app/cash/sqldelight/driver/native/util/NativeCache.kt @@ -1,10 +1,26 @@ package app.cash.sqldelight.driver.native.util -internal interface NativeCache { - fun put(key: String, value: T?): T? - fun getOrCreate(key: String, block: () -> T): T - fun remove(key: String): T? - fun cleanUp(block: (T) -> Unit) -} +internal class MutableCache { + private val dictionary = mutableMapOf() + private val lock = PoolLock() + + fun put(key: String, value: T?): T? = lock.withLock { + if (value == null) { + dictionary.remove(key) + } else { + dictionary.put(key, value) + } + } + + fun getOrCreate(key: String, block: () -> T): T = lock.withLock { + dictionary.getOrPut(key, block) + } -internal expect fun nativeCache(): NativeCache + fun remove(key: String): T? = lock.withLock { + dictionary.remove(key) + } + + fun cleanUp(block: (T) -> Unit) = lock.withLock { + dictionary.values.forEach(block) + } +} diff --git a/drivers/native-driver/src/nativeTest/kotlin/com/squareup/sqldelight/drivers/native/LazyDriverBaseTest.kt b/drivers/native-driver/src/nativeTest/kotlin/com/squareup/sqldelight/drivers/native/LazyDriverBaseTest.kt index b008d712700..65fc8b7b1a4 100644 --- a/drivers/native-driver/src/nativeTest/kotlin/com/squareup/sqldelight/drivers/native/LazyDriverBaseTest.kt +++ b/drivers/native-driver/src/nativeTest/kotlin/com/squareup/sqldelight/drivers/native/LazyDriverBaseTest.kt @@ -8,7 +8,6 @@ import co.touchlab.sqliter.DatabaseConfiguration import co.touchlab.sqliter.DatabaseFileContext.deleteDatabase import co.touchlab.sqliter.DatabaseManager import co.touchlab.sqliter.createDatabaseManager -import co.touchlab.stately.freeze import kotlin.test.AfterTest import kotlin.test.BeforeTest @@ -24,9 +23,7 @@ abstract class LazyDriverBaseTest { protected val transacter: TransacterImpl get() { - val t = transacterInternal - t.freeze() - return t + return transacterInternal } @BeforeTest fun setup() { diff --git a/drivers/native-driver/src/nativeTest/kotlin/com/squareup/sqldelight/drivers/native/connectionpool/WalConcurrencyTest.kt b/drivers/native-driver/src/nativeTest/kotlin/com/squareup/sqldelight/drivers/native/connectionpool/WalConcurrencyTest.kt index fc7d9c8fec0..d38391fda9f 100644 --- a/drivers/native-driver/src/nativeTest/kotlin/com/squareup/sqldelight/drivers/native/connectionpool/WalConcurrencyTest.kt +++ b/drivers/native-driver/src/nativeTest/kotlin/com/squareup/sqldelight/drivers/native/connectionpool/WalConcurrencyTest.kt @@ -9,7 +9,6 @@ import co.touchlab.testhelp.concurrency.sleep import kotlin.native.concurrent.AtomicInt import kotlin.native.concurrent.TransferMode import kotlin.native.concurrent.Worker -import kotlin.native.concurrent.freeze import kotlin.test.BeforeTest import kotlin.test.Ignore import kotlin.test.Test @@ -50,7 +49,7 @@ class WalConcurrencyTest : BaseConcurrencyTest() { } } - val future = worker.execute(TransferMode.SAFE, { block.freeze() }) { it() } + val future = worker.execute(TransferMode.SAFE, { block }) { it() } // When ready, transaction started but sleeping waitFor { transactionStarted.value > 0 } @@ -131,7 +130,7 @@ class WalConcurrencyTest : BaseConcurrencyTest() { } } - val future = worker.execute(TransferMode.SAFE, { block.freeze() }) { it() } + val future = worker.execute(TransferMode.SAFE, { block }) { it() } // Transaction with write started but sleeping waitFor { transactionStarted.value > 0 } @@ -160,7 +159,7 @@ class WalConcurrencyTest : BaseConcurrencyTest() { } } - val future = worker.execute(TransferMode.SAFE, { block.freeze() }) { it() } + val future = worker.execute(TransferMode.SAFE, { block }) { it() } // When we get here, first transaction has run a write command, and is sleeping waitFor { transactionStarted.value > 0 } diff --git a/gradle/libs.versions.toml b/gradle/libs.versions.toml index 69840e1f815..2dee2fbc18a 100644 --- a/gradle/libs.versions.toml +++ b/gradle/libs.versions.toml @@ -6,7 +6,7 @@ idea = "211.7628.21" # Android Studio Bumblebee (see https://plugins.jetbrains.c androidxSqlite = "2.2.0" schemaCrawler = "16.16.14" stately = "1.2.1" -sqliter = "1.1.1" +sqliter = "1.1.2" testhelp = "0.6.1" sqljs = "1.6.2" paging3 = "3.1.1" diff --git a/settings.gradle b/settings.gradle index 1234c1cb331..041ef8607c2 100644 --- a/settings.gradle +++ b/settings.gradle @@ -24,6 +24,7 @@ include ':dialects:sqlite-3-35' include ':drivers:android-driver' include ':drivers:jdbc-driver' include ':drivers:native-driver' +include ':drivers:native-driver-strict' include ':drivers:sqlite-driver' include ':drivers:sqljs-driver' include ':drivers:driver-test' From 86dca785d5db0c1a39e58088bdd41acc05056830 Mon Sep 17 00:00:00 2001 From: Kevin Galligan Date: Mon, 2 May 2022 21:00:38 -0400 Subject: [PATCH 2/8] Update pool lock and memory model --- .github/workflows/PR.yml | 4 + drivers/native-driver/build.gradle | 17 ++-- .../sqldelight/driver/native/util/PoolLock.kt | 0 .../sqldelight/driver/native/util/PoolLock.kt | 81 ------------------- .../sqldelight/driver/native/util/PoolLock.kt | 81 ------------------- .../sqldelight/driver/native/util/PoolLock.kt | 0 gradle.properties | 9 ++- 7 files changed, 23 insertions(+), 169 deletions(-) rename drivers/native-driver/src/{mingwX64Main => mingwMain}/kotlin/app/cash/sqldelight/driver/native/util/PoolLock.kt (100%) delete mode 100644 drivers/native-driver/src/mingwX86Main/kotlin/app/cash/sqldelight/driver/native/util/PoolLock.kt delete mode 100644 drivers/native-driver/src/nativeDarwinMain/kotlin/app/cash/sqldelight/driver/native/util/PoolLock.kt rename drivers/native-driver/src/{linuxMain => nativeLinuxLikeMain}/kotlin/app/cash/sqldelight/driver/native/util/PoolLock.kt (100%) diff --git a/.github/workflows/PR.yml b/.github/workflows/PR.yml index 3827e058651..0b7bd4761f6 100644 --- a/.github/workflows/PR.yml +++ b/.github/workflows/PR.yml @@ -73,6 +73,10 @@ jobs: if: matrix.os == 'macOS-latest' && matrix.job == 'test' run: ./gradlew iosX64Test --stacktrace + - name: Run ios tests with strict memory model + if: matrix.os == 'macOS-latest' && matrix.job == 'test' + run: ./gradlew :drivers:native-driver-strict:iosX64Test -Pkotlin.native.binary.memoryModel=strict --stacktrace + # Build the sample - name: Publish the sqlite dialect if: matrix.os == 'macOS-latest' && matrix.job == 'gradle-plugin-tests' diff --git a/drivers/native-driver/build.gradle b/drivers/native-driver/build.gradle index 2341ca6ffa4..c432fd9216d 100644 --- a/drivers/native-driver/build.gradle +++ b/drivers/native-driver/build.gradle @@ -43,7 +43,6 @@ kotlin { dependsOn(commonMain) dependencies { api deps.sqliter - implementation deps.stately.core } } nativeTest { @@ -52,17 +51,23 @@ kotlin { implementation project(':drivers:driver-test') } } + nativeLinuxLikeMain { + dependsOn(nativeMain) + } nativeDarwinMain{ + dependsOn(nativeLinuxLikeMain) + } + mingwMain{ dependsOn(nativeMain) } mingwX86Main{ - dependsOn(nativeMain) + dependsOn(mingwMain) } mingwX64Main{ - dependsOn(nativeMain) + dependsOn(mingwMain) } linuxMain{ - dependsOn(nativeMain) + dependsOn(nativeLinuxLikeMain) } } @@ -83,7 +88,7 @@ kotlin { } configure([targets.mingwX64]) { - sourceSets.getByName("${name}Main").dependsOn(sourceSets.nativeMain) + sourceSets.getByName("${name}Main").dependsOn(sourceSets.mingwMain) sourceSets.getByName("${name}Test").dependsOn(sourceSets.nativeTest) compilations.test { kotlinOptions.freeCompilerArgs += ["-linker-options", "-Lc:\\msys64\\mingw64\\lib -L$rootDir\\libs -lsqlite3".toString()] @@ -91,7 +96,7 @@ kotlin { } configure([targets.mingwX86]) { - sourceSets.getByName("${name}Main").dependsOn(sourceSets.nativeMain) + sourceSets.getByName("${name}Main").dependsOn(sourceSets.mingwMain) sourceSets.getByName("${name}Test").dependsOn(sourceSets.nativeTest) compilations.test { kotlinOptions.freeCompilerArgs += ["-linker-options", "-Lc:\\msys32\\mingw32\\lib -L$rootDir\\libs -lsqlite3".toString()] diff --git a/drivers/native-driver/src/mingwX64Main/kotlin/app/cash/sqldelight/driver/native/util/PoolLock.kt b/drivers/native-driver/src/mingwMain/kotlin/app/cash/sqldelight/driver/native/util/PoolLock.kt similarity index 100% rename from drivers/native-driver/src/mingwX64Main/kotlin/app/cash/sqldelight/driver/native/util/PoolLock.kt rename to drivers/native-driver/src/mingwMain/kotlin/app/cash/sqldelight/driver/native/util/PoolLock.kt diff --git a/drivers/native-driver/src/mingwX86Main/kotlin/app/cash/sqldelight/driver/native/util/PoolLock.kt b/drivers/native-driver/src/mingwX86Main/kotlin/app/cash/sqldelight/driver/native/util/PoolLock.kt deleted file mode 100644 index 331f846dadb..00000000000 --- a/drivers/native-driver/src/mingwX86Main/kotlin/app/cash/sqldelight/driver/native/util/PoolLock.kt +++ /dev/null @@ -1,81 +0,0 @@ -package app.cash.sqldelight.driver.native.util - -import co.touchlab.stately.concurrency.AtomicBoolean -import kotlinx.cinterop.alloc -import kotlinx.cinterop.free -import kotlinx.cinterop.nativeHeap -import kotlinx.cinterop.ptr -import platform.posix.pthread_cond_destroy -import platform.posix.pthread_cond_init -import platform.posix.pthread_cond_signal -import platform.posix.pthread_cond_tVar -import platform.posix.pthread_cond_wait -import platform.posix.pthread_mutex_destroy -import platform.posix.pthread_mutex_init -import platform.posix.pthread_mutex_lock -import platform.posix.pthread_mutex_tVar -import platform.posix.pthread_mutex_unlock - -internal actual class PoolLock actual constructor() { - private val isActive = AtomicBoolean(true) - private val mutex = nativeHeap.alloc() - .apply { pthread_mutex_init(ptr, null) } - private val cond = nativeHeap.alloc() - .apply { pthread_cond_init(ptr, null) } - - actual fun withLock( - action: CriticalSection.() -> R - ): R { - check(isActive.value) - pthread_mutex_lock(mutex.ptr) - - val result: R - - try { - result = action(CriticalSection()) - } finally { - pthread_mutex_unlock(mutex.ptr) - } - - return result - } - - actual fun notifyConditionChanged() { - pthread_cond_signal(cond.ptr) - } - - actual fun close(): Boolean { - if (isActive.compareAndSet(expected = true, new = false)) { - pthread_cond_destroy(cond.ptr) - pthread_mutex_destroy(mutex.ptr) - nativeHeap.free(cond) - nativeHeap.free(mutex) - return true - } - - return false - } - - actual inner class CriticalSection { - actual fun loopForConditionalResult(block: () -> R?): R { - check(isActive.value) - - var result = block() - - while (result == null) { - pthread_cond_wait(cond.ptr, mutex.ptr) - result = block() - } - - return result - } - - actual fun loopUntilConditionalResult(block: () -> Boolean) { - check(isActive.value) - - while (!block()) { - pthread_cond_wait(cond.ptr, mutex.ptr) - } - } - } -} diff --git a/drivers/native-driver/src/nativeDarwinMain/kotlin/app/cash/sqldelight/driver/native/util/PoolLock.kt b/drivers/native-driver/src/nativeDarwinMain/kotlin/app/cash/sqldelight/driver/native/util/PoolLock.kt deleted file mode 100644 index 3896457a75e..00000000000 --- a/drivers/native-driver/src/nativeDarwinMain/kotlin/app/cash/sqldelight/driver/native/util/PoolLock.kt +++ /dev/null @@ -1,81 +0,0 @@ -package app.cash.sqldelight.driver.native.util - -import co.touchlab.stately.concurrency.AtomicBoolean -import kotlinx.cinterop.alloc -import kotlinx.cinterop.free -import kotlinx.cinterop.nativeHeap -import kotlinx.cinterop.ptr -import platform.posix.pthread_cond_destroy -import platform.posix.pthread_cond_init -import platform.posix.pthread_cond_signal -import platform.posix.pthread_cond_t -import platform.posix.pthread_cond_wait -import platform.posix.pthread_mutex_destroy -import platform.posix.pthread_mutex_init -import platform.posix.pthread_mutex_lock -import platform.posix.pthread_mutex_t -import platform.posix.pthread_mutex_unlock - -internal actual class PoolLock actual constructor() { - private val isActive = AtomicBoolean(true) - private val mutex = nativeHeap.alloc() - .apply { pthread_mutex_init(ptr, null) } - private val cond = nativeHeap.alloc() - .apply { pthread_cond_init(ptr, null) } - - actual fun withLock( - action: CriticalSection.() -> R - ): R { - check(isActive.value) - pthread_mutex_lock(mutex.ptr) - - val result: R - - try { - result = action(CriticalSection()) - } finally { - pthread_mutex_unlock(mutex.ptr) - } - - return result - } - - actual fun notifyConditionChanged() { - pthread_cond_signal(cond.ptr) - } - - actual fun close(): Boolean { - if (isActive.compareAndSet(expected = true, new = false)) { - pthread_cond_destroy(cond.ptr) - pthread_mutex_destroy(mutex.ptr) - nativeHeap.free(cond) - nativeHeap.free(mutex) - return true - } - - return false - } - - actual inner class CriticalSection { - actual fun loopForConditionalResult(block: () -> R?): R { - check(isActive.value) - - var result = block() - - while (result == null) { - pthread_cond_wait(cond.ptr, mutex.ptr) - result = block() - } - - return result - } - - actual fun loopUntilConditionalResult(block: () -> Boolean) { - check(isActive.value) - - while (!block()) { - pthread_cond_wait(cond.ptr, mutex.ptr) - } - } - } -} diff --git a/drivers/native-driver/src/linuxMain/kotlin/app/cash/sqldelight/driver/native/util/PoolLock.kt b/drivers/native-driver/src/nativeLinuxLikeMain/kotlin/app/cash/sqldelight/driver/native/util/PoolLock.kt similarity index 100% rename from drivers/native-driver/src/linuxMain/kotlin/app/cash/sqldelight/driver/native/util/PoolLock.kt rename to drivers/native-driver/src/nativeLinuxLikeMain/kotlin/app/cash/sqldelight/driver/native/util/PoolLock.kt diff --git a/gradle.properties b/gradle.properties index 3f4f2a2574a..5d1fa3e57d2 100644 --- a/gradle.properties +++ b/gradle.properties @@ -21,4 +21,11 @@ android.useAndroidX=true kotlin.js.compiler=both kotlin.mpp.stability.nowarn=true -kotlin.native.ignoreDisabledTargets=true \ No newline at end of file +kotlin.native.ignoreDisabledTargets=true + +kotlin.mpp.enableGranularSourceSetsMetadata=true +kotlin.native.enableDependencyPropagation=false +kotlin.mpp.enableCInteropCommonization=true +kotlin.mpp.commonizerLogLevel=info + +kotlin.native.binary.memoryModel=experimental \ No newline at end of file From 8ef851f30c82816dc56c3ec773b8bcab06ffb6c5 Mon Sep 17 00:00:00 2001 From: Kevin Galligan Date: Mon, 2 May 2022 22:00:28 -0400 Subject: [PATCH 3/8] Update driver docs --- docs/native_sqlite/index.md | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) diff --git a/docs/native_sqlite/index.md b/docs/native_sqlite/index.md index 6a2c540eb54..65c59d4cf47 100644 --- a/docs/native_sqlite/index.md +++ b/docs/native_sqlite/index.md @@ -26,6 +26,22 @@ kotlin { val driver: SqlDriver = NativeSqliteDriver(Database.Schema, "test.db") ``` +### Native Driver Modules + +There are two native driver modules. One, `native-driver`, is for the newer memory model. `native-driver-strict` +provides support for the original strict memory model. + +Projects that continue to use the strict memory model should switch to `native-driver-strict`. The Kotlin/Native platform +and libraries from Jetbrains are generally focused on the new memory model going forward. Moving to the new memory will be +generally recommended in the near future, as libraries like kotlinx.coroutines are likely to drop support for the strict +memory model around the release of Kotlin 1.7. + +For strict memory support, replace the dependency above with: + +```kotlin +implementation ("com.squareup.sqldelight:native-driver-strict:{{ versions.sqldelight }}") +``` + {% include 'common/index_queries.md' %} ## Reader Connection Pools From d72c96f368aa97890baba6c56d70b742de727796 Mon Sep 17 00:00:00 2001 From: Kevin Galligan Date: Tue, 3 May 2022 08:54:23 -0400 Subject: [PATCH 4/8] Rename file for lint check --- .../driver/native/util/{NativeCache.kt => MutableCache.kt} | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename drivers/native-driver/src/nativeMain/kotlin/app/cash/sqldelight/driver/native/util/{NativeCache.kt => MutableCache.kt} (100%) diff --git a/drivers/native-driver/src/nativeMain/kotlin/app/cash/sqldelight/driver/native/util/NativeCache.kt b/drivers/native-driver/src/nativeMain/kotlin/app/cash/sqldelight/driver/native/util/MutableCache.kt similarity index 100% rename from drivers/native-driver/src/nativeMain/kotlin/app/cash/sqldelight/driver/native/util/NativeCache.kt rename to drivers/native-driver/src/nativeMain/kotlin/app/cash/sqldelight/driver/native/util/MutableCache.kt From f32d70377187667cdc7d0b0fff6cb15f14f2efa6 Mon Sep 17 00:00:00 2001 From: Kevin Galligan Date: Tue, 3 May 2022 21:16:37 -0400 Subject: [PATCH 5/8] Simplified driver with support for both memory models --- .github/workflows/PR.yml | 2 +- docs/native_sqlite/index.md | 19 +- drivers/native-driver-strict/build.gradle | 117 ----- .../native-driver-strict/gradle.properties | 3 - .../driver/native/util/NativeCache.kt | 18 - .../sqldelight/driver/native/util/PoolLock.kt | 81 ---- .../sqldelight/driver/native/util/Stately.kt | 21 - .../driver/native/util/NativeCache.kt | 18 - .../sqldelight/driver/native/util/Stately.kt | 21 - .../sqldelight/driver/native/util/PoolLock.kt | 81 ---- .../sqldelight/driver/native/util/PoolLock.kt | 81 ---- .../driver/native/util/NativeCache.kt | 49 --- .../sqldelight/driver/native/util/PoolLock.kt | 81 ---- .../cash/sqldelight/driver/native/Borrowed.kt | 6 - .../driver/native/NativeSqlDatabase.kt | 414 ------------------ .../app/cash/sqldelight/driver/native/Pool.kt | 122 ------ .../driver/native/SqliterSqlCursor.kt | 37 -- .../driver/native/SqliterStatement.kt | 42 -- .../driver/native/util/NativeCache.kt | 10 - .../sqldelight/driver/native/util/PoolLock.kt | 32 -- .../drivers/native/LazyDriverBaseTest.kt | 108 ----- .../drivers/native/NativeDriverTest.kt | 14 - .../drivers/native/NativeQueryTest.kt | 14 - .../native/NativeSqliteDriverTest.kt.ignore | 412 ----------------- .../drivers/native/NativeTransacterTest.kt | 14 - .../connectionpool/BaseConcurrencyTest.kt | 175 -------- .../NativeSqliteDriverConfigTest.kt | 19 - .../connectionpool/WalConcurrencyTest.kt | 202 --------- .../driver/native/NativeSqlDatabase.kt | 114 ++--- .../app/cash/sqldelight/driver/native/Pool.kt | 7 +- .../driver/native/util/MemoryModel.kt | 9 + .../driver/native/util/MutableCache.kt | 103 ++++- .../drivers/native/LazyDriverBaseTest.kt | 5 +- .../connectionpool/WalConcurrencyTest.kt | 7 +- runtime/build.gradle | 6 +- settings.gradle | 1 - 36 files changed, 159 insertions(+), 2306 deletions(-) delete mode 100644 drivers/native-driver-strict/build.gradle delete mode 100644 drivers/native-driver-strict/gradle.properties delete mode 100644 drivers/native-driver-strict/src/linuxMain/kotlin/app/cash/sqldelight/driver/native/util/NativeCache.kt delete mode 100644 drivers/native-driver-strict/src/linuxMain/kotlin/app/cash/sqldelight/driver/native/util/PoolLock.kt delete mode 100644 drivers/native-driver-strict/src/linuxMain/kotlin/app/cash/sqldelight/driver/native/util/Stately.kt delete mode 100644 drivers/native-driver-strict/src/mingwMain/kotlin/app/cash/sqldelight/driver/native/util/NativeCache.kt delete mode 100644 drivers/native-driver-strict/src/mingwMain/kotlin/app/cash/sqldelight/driver/native/util/Stately.kt delete mode 100644 drivers/native-driver-strict/src/mingwX64Main/kotlin/app/cash/sqldelight/driver/native/util/PoolLock.kt delete mode 100644 drivers/native-driver-strict/src/mingwX86Main/kotlin/app/cash/sqldelight/driver/native/util/PoolLock.kt delete mode 100644 drivers/native-driver-strict/src/nativeDarwinMain/kotlin/app/cash/sqldelight/driver/native/util/NativeCache.kt delete mode 100644 drivers/native-driver-strict/src/nativeDarwinMain/kotlin/app/cash/sqldelight/driver/native/util/PoolLock.kt delete mode 100644 drivers/native-driver-strict/src/nativeMain/kotlin/app/cash/sqldelight/driver/native/Borrowed.kt delete mode 100644 drivers/native-driver-strict/src/nativeMain/kotlin/app/cash/sqldelight/driver/native/NativeSqlDatabase.kt delete mode 100644 drivers/native-driver-strict/src/nativeMain/kotlin/app/cash/sqldelight/driver/native/Pool.kt delete mode 100644 drivers/native-driver-strict/src/nativeMain/kotlin/app/cash/sqldelight/driver/native/SqliterSqlCursor.kt delete mode 100644 drivers/native-driver-strict/src/nativeMain/kotlin/app/cash/sqldelight/driver/native/SqliterStatement.kt delete mode 100644 drivers/native-driver-strict/src/nativeMain/kotlin/app/cash/sqldelight/driver/native/util/NativeCache.kt delete mode 100644 drivers/native-driver-strict/src/nativeMain/kotlin/app/cash/sqldelight/driver/native/util/PoolLock.kt delete mode 100644 drivers/native-driver-strict/src/nativeTest/kotlin/com/squareup/sqldelight/drivers/native/LazyDriverBaseTest.kt delete mode 100644 drivers/native-driver-strict/src/nativeTest/kotlin/com/squareup/sqldelight/drivers/native/NativeDriverTest.kt delete mode 100644 drivers/native-driver-strict/src/nativeTest/kotlin/com/squareup/sqldelight/drivers/native/NativeQueryTest.kt delete mode 100644 drivers/native-driver-strict/src/nativeTest/kotlin/com/squareup/sqldelight/drivers/native/NativeSqliteDriverTest.kt.ignore delete mode 100644 drivers/native-driver-strict/src/nativeTest/kotlin/com/squareup/sqldelight/drivers/native/NativeTransacterTest.kt delete mode 100644 drivers/native-driver-strict/src/nativeTest/kotlin/com/squareup/sqldelight/drivers/native/connectionpool/BaseConcurrencyTest.kt delete mode 100644 drivers/native-driver-strict/src/nativeTest/kotlin/com/squareup/sqldelight/drivers/native/connectionpool/NativeSqliteDriverConfigTest.kt delete mode 100644 drivers/native-driver-strict/src/nativeTest/kotlin/com/squareup/sqldelight/drivers/native/connectionpool/WalConcurrencyTest.kt create mode 100644 drivers/native-driver/src/nativeMain/kotlin/app/cash/sqldelight/driver/native/util/MemoryModel.kt diff --git a/.github/workflows/PR.yml b/.github/workflows/PR.yml index 486453c0d97..fd533735861 100644 --- a/.github/workflows/PR.yml +++ b/.github/workflows/PR.yml @@ -77,7 +77,7 @@ jobs: - name: Run ios tests with strict memory model if: matrix.os == 'macOS-latest' && matrix.job == 'test' - run: ./gradlew :drivers:native-driver-strict:iosX64Test -Pkotlin.native.binary.memoryModel=strict --stacktrace + run: ./gradlew iosX64Test -Pkotlin.native.binary.memoryModel=strict --stacktrace # Build the sample - name: Build the sample diff --git a/docs/native_sqlite/index.md b/docs/native_sqlite/index.md index 65c59d4cf47..f8526f05557 100644 --- a/docs/native_sqlite/index.md +++ b/docs/native_sqlite/index.md @@ -26,21 +26,12 @@ kotlin { val driver: SqlDriver = NativeSqliteDriver(Database.Schema, "test.db") ``` -### Native Driver Modules +### Kotlin/Native Memory Models -There are two native driver modules. One, `native-driver`, is for the newer memory model. `native-driver-strict` -provides support for the original strict memory model. - -Projects that continue to use the strict memory model should switch to `native-driver-strict`. The Kotlin/Native platform -and libraries from Jetbrains are generally focused on the new memory model going forward. Moving to the new memory will be -generally recommended in the near future, as libraries like kotlinx.coroutines are likely to drop support for the strict -memory model around the release of Kotlin 1.7. - -For strict memory support, replace the dependency above with: - -```kotlin -implementation ("com.squareup.sqldelight:native-driver-strict:{{ versions.sqldelight }}") -``` +The SQLDelight native driver is compatible with both the original strict memory model and the updated +memory model. However, it is optimized for the new memory model, and as most of the official Jetbrains +libraries will be gradually dropping support for the strict memory model, support for the strict +memory model may be deprecated or removed in future releases. {% include 'common/index_queries.md' %} diff --git a/drivers/native-driver-strict/build.gradle b/drivers/native-driver-strict/build.gradle deleted file mode 100644 index 0eabe491090..00000000000 --- a/drivers/native-driver-strict/build.gradle +++ /dev/null @@ -1,117 +0,0 @@ -import org.jetbrains.kotlin.konan.target.HostManager - -plugins { - alias(deps.plugins.kotlin.multiplatform) - alias(deps.plugins.publish) - alias(deps.plugins.dokka) -} - -def ideaActive = System.getProperty("idea.active") == "true" - -kotlin { - iosX64() - iosArm32() - iosArm64() - tvosX64() - tvosArm64() - watchosX86() - watchosX64() - watchosArm32() - watchosArm64() - macosX64() - mingwX86() - mingwX64() - linuxX64() - macosArm64() - iosSimulatorArm64() - watchosSimulatorArm64() - tvosSimulatorArm64() - - sourceSets { - commonMain { - dependencies { - api project (':runtime') - } - } - commonTest { - dependencies { - implementation deps.kotlin.test.common - implementation deps.testhelp - } - } - nativeMain { - dependsOn(commonMain) - dependencies { - api deps.sqliter - implementation deps.stately.core - } - } - nativeTest { - dependsOn(commonTest) - dependencies { - implementation project(':drivers:driver-test') - } - } - nativeDarwinMain{ - dependsOn(nativeMain) - } - mingwMain{ - dependsOn(nativeMain) - } - mingwX86Main{ - dependsOn(mingwMain) - } - mingwX64Main{ - dependsOn(mingwMain) - } - linuxMain{ - dependsOn(nativeMain) - } - } - - configure([targets.iosX64, targets.iosArm32, targets.iosArm64, targets.tvosX64, targets.tvosArm64, targets.watchosX86, targets.watchosX64, targets.watchosArm32, targets.watchosArm64, targets.macosX64, targets.macosArm64, targets.iosSimulatorArm64, targets.watchosSimulatorArm64, targets.tvosSimulatorArm64]) { - sourceSets.getByName("${name}Main").dependsOn(sourceSets.nativeDarwinMain) - sourceSets.getByName("${name}Test").dependsOn(sourceSets.nativeTest) - compilations.test { - kotlinOptions.freeCompilerArgs += ["-linker-options", "-lsqlite3"] - } - } - - configure([targets.linuxX64]) { - sourceSets.getByName("${name}Main").dependsOn(sourceSets.linuxMain) - sourceSets.getByName("${name}Test").dependsOn(sourceSets.nativeTest) - compilations.test { - kotlinOptions.freeCompilerArgs += ["-linker-options", "-lsqlite3 -L/usr/lib/x86_64-linux-gnu -L/usr/lib"] - } - } - - configure([targets.mingwX64]) { - sourceSets.getByName("${name}Main").dependsOn(sourceSets.mingwMain) - sourceSets.getByName("${name}Test").dependsOn(sourceSets.nativeTest) - compilations.test { - kotlinOptions.freeCompilerArgs += ["-linker-options", "-Lc:\\msys64\\mingw64\\lib -L$rootDir\\libs -lsqlite3".toString()] - } - } - - configure([targets.mingwX86]) { - sourceSets.getByName("${name}Main").dependsOn(sourceSets.mingwMain) - sourceSets.getByName("${name}Test").dependsOn(sourceSets.nativeTest) - compilations.test { - kotlinOptions.freeCompilerArgs += ["-linker-options", "-Lc:\\msys32\\mingw32\\lib -L$rootDir\\libs -lsqlite3".toString()] - } - } - - //linking fails for the linux test build if not built on a linux host - //ensure the tests and linking for them is only done on linux hosts - if(!HostManager.hostIsLinux) { - tasks.findByName("linuxX64Test")?.enabled = false - tasks.findByName("linkDebugTestLinuxX64")?.enabled = false - } - - if(!HostManager.hostIsMingw) { - tasks.findByName("mingwX64Test")?.enabled = false - tasks.findByName("linkDebugTestMingwX64")?.enabled = false - } -} - -apply from: "$rootDir/gradle/gradle-mvn-push.gradle" diff --git a/drivers/native-driver-strict/gradle.properties b/drivers/native-driver-strict/gradle.properties deleted file mode 100644 index b749e6d36ed..00000000000 --- a/drivers/native-driver-strict/gradle.properties +++ /dev/null @@ -1,3 +0,0 @@ -POM_ARTIFACT_ID=native-driver -POM_NAME=SQLDelight Native Driver -POM_DESCRIPTION=Native SQLite driver for SQLDelight diff --git a/drivers/native-driver-strict/src/linuxMain/kotlin/app/cash/sqldelight/driver/native/util/NativeCache.kt b/drivers/native-driver-strict/src/linuxMain/kotlin/app/cash/sqldelight/driver/native/util/NativeCache.kt deleted file mode 100644 index 2da7e85546e..00000000000 --- a/drivers/native-driver-strict/src/linuxMain/kotlin/app/cash/sqldelight/driver/native/util/NativeCache.kt +++ /dev/null @@ -1,18 +0,0 @@ -package app.cash.sqldelight.driver.native.util - -import co.touchlab.stately.collections.SharedHashMap -import co.touchlab.stately.collections.frozenHashMap - -internal actual fun nativeCache(): NativeCache = - NativeCacheImpl() - -private class NativeCacheImpl : NativeCache { - private val dictionary = frozenHashMap() as SharedHashMap - - override fun put(key: String, value: T?): T? = dictionary.put(key, value) - override fun getOrCreate(key: String, block: () -> T): T = dictionary.getOrPut(key, block)!! - override fun remove(key: String): T? = dictionary.remove(key) - override fun cleanUp(block: (T) -> Unit) { - dictionary.values.filterNotNull().forEach(block) - } -} diff --git a/drivers/native-driver-strict/src/linuxMain/kotlin/app/cash/sqldelight/driver/native/util/PoolLock.kt b/drivers/native-driver-strict/src/linuxMain/kotlin/app/cash/sqldelight/driver/native/util/PoolLock.kt deleted file mode 100644 index 3896457a75e..00000000000 --- a/drivers/native-driver-strict/src/linuxMain/kotlin/app/cash/sqldelight/driver/native/util/PoolLock.kt +++ /dev/null @@ -1,81 +0,0 @@ -package app.cash.sqldelight.driver.native.util - -import co.touchlab.stately.concurrency.AtomicBoolean -import kotlinx.cinterop.alloc -import kotlinx.cinterop.free -import kotlinx.cinterop.nativeHeap -import kotlinx.cinterop.ptr -import platform.posix.pthread_cond_destroy -import platform.posix.pthread_cond_init -import platform.posix.pthread_cond_signal -import platform.posix.pthread_cond_t -import platform.posix.pthread_cond_wait -import platform.posix.pthread_mutex_destroy -import platform.posix.pthread_mutex_init -import platform.posix.pthread_mutex_lock -import platform.posix.pthread_mutex_t -import platform.posix.pthread_mutex_unlock - -internal actual class PoolLock actual constructor() { - private val isActive = AtomicBoolean(true) - private val mutex = nativeHeap.alloc() - .apply { pthread_mutex_init(ptr, null) } - private val cond = nativeHeap.alloc() - .apply { pthread_cond_init(ptr, null) } - - actual fun withLock( - action: CriticalSection.() -> R - ): R { - check(isActive.value) - pthread_mutex_lock(mutex.ptr) - - val result: R - - try { - result = action(CriticalSection()) - } finally { - pthread_mutex_unlock(mutex.ptr) - } - - return result - } - - actual fun notifyConditionChanged() { - pthread_cond_signal(cond.ptr) - } - - actual fun close(): Boolean { - if (isActive.compareAndSet(expected = true, new = false)) { - pthread_cond_destroy(cond.ptr) - pthread_mutex_destroy(mutex.ptr) - nativeHeap.free(cond) - nativeHeap.free(mutex) - return true - } - - return false - } - - actual inner class CriticalSection { - actual fun loopForConditionalResult(block: () -> R?): R { - check(isActive.value) - - var result = block() - - while (result == null) { - pthread_cond_wait(cond.ptr, mutex.ptr) - result = block() - } - - return result - } - - actual fun loopUntilConditionalResult(block: () -> Boolean) { - check(isActive.value) - - while (!block()) { - pthread_cond_wait(cond.ptr, mutex.ptr) - } - } - } -} diff --git a/drivers/native-driver-strict/src/linuxMain/kotlin/app/cash/sqldelight/driver/native/util/Stately.kt b/drivers/native-driver-strict/src/linuxMain/kotlin/app/cash/sqldelight/driver/native/util/Stately.kt deleted file mode 100644 index 71d38c0621a..00000000000 --- a/drivers/native-driver-strict/src/linuxMain/kotlin/app/cash/sqldelight/driver/native/util/Stately.kt +++ /dev/null @@ -1,21 +0,0 @@ -package app.cash.sqldelight.driver.native.util - -import co.touchlab.stately.collections.SharedHashMap -import co.touchlab.stately.collections.SharedLinkedList - -/** - * This should probably be in Stately - */ -internal fun SharedLinkedList.cleanUp(block: (T) -> Unit) { - val extractList = ArrayList(size) - extractList.addAll(this) - this.clear() - extractList.forEach(block) -} - -internal fun SharedHashMap.cleanUp(block: (Map.Entry) -> Unit) { - val extractMap = HashMap(this.size) - extractMap.putAll(this) - this.clear() - extractMap.forEach(block) -} diff --git a/drivers/native-driver-strict/src/mingwMain/kotlin/app/cash/sqldelight/driver/native/util/NativeCache.kt b/drivers/native-driver-strict/src/mingwMain/kotlin/app/cash/sqldelight/driver/native/util/NativeCache.kt deleted file mode 100644 index 2da7e85546e..00000000000 --- a/drivers/native-driver-strict/src/mingwMain/kotlin/app/cash/sqldelight/driver/native/util/NativeCache.kt +++ /dev/null @@ -1,18 +0,0 @@ -package app.cash.sqldelight.driver.native.util - -import co.touchlab.stately.collections.SharedHashMap -import co.touchlab.stately.collections.frozenHashMap - -internal actual fun nativeCache(): NativeCache = - NativeCacheImpl() - -private class NativeCacheImpl : NativeCache { - private val dictionary = frozenHashMap() as SharedHashMap - - override fun put(key: String, value: T?): T? = dictionary.put(key, value) - override fun getOrCreate(key: String, block: () -> T): T = dictionary.getOrPut(key, block)!! - override fun remove(key: String): T? = dictionary.remove(key) - override fun cleanUp(block: (T) -> Unit) { - dictionary.values.filterNotNull().forEach(block) - } -} diff --git a/drivers/native-driver-strict/src/mingwMain/kotlin/app/cash/sqldelight/driver/native/util/Stately.kt b/drivers/native-driver-strict/src/mingwMain/kotlin/app/cash/sqldelight/driver/native/util/Stately.kt deleted file mode 100644 index 71d38c0621a..00000000000 --- a/drivers/native-driver-strict/src/mingwMain/kotlin/app/cash/sqldelight/driver/native/util/Stately.kt +++ /dev/null @@ -1,21 +0,0 @@ -package app.cash.sqldelight.driver.native.util - -import co.touchlab.stately.collections.SharedHashMap -import co.touchlab.stately.collections.SharedLinkedList - -/** - * This should probably be in Stately - */ -internal fun SharedLinkedList.cleanUp(block: (T) -> Unit) { - val extractList = ArrayList(size) - extractList.addAll(this) - this.clear() - extractList.forEach(block) -} - -internal fun SharedHashMap.cleanUp(block: (Map.Entry) -> Unit) { - val extractMap = HashMap(this.size) - extractMap.putAll(this) - this.clear() - extractMap.forEach(block) -} diff --git a/drivers/native-driver-strict/src/mingwX64Main/kotlin/app/cash/sqldelight/driver/native/util/PoolLock.kt b/drivers/native-driver-strict/src/mingwX64Main/kotlin/app/cash/sqldelight/driver/native/util/PoolLock.kt deleted file mode 100644 index 331f846dadb..00000000000 --- a/drivers/native-driver-strict/src/mingwX64Main/kotlin/app/cash/sqldelight/driver/native/util/PoolLock.kt +++ /dev/null @@ -1,81 +0,0 @@ -package app.cash.sqldelight.driver.native.util - -import co.touchlab.stately.concurrency.AtomicBoolean -import kotlinx.cinterop.alloc -import kotlinx.cinterop.free -import kotlinx.cinterop.nativeHeap -import kotlinx.cinterop.ptr -import platform.posix.pthread_cond_destroy -import platform.posix.pthread_cond_init -import platform.posix.pthread_cond_signal -import platform.posix.pthread_cond_tVar -import platform.posix.pthread_cond_wait -import platform.posix.pthread_mutex_destroy -import platform.posix.pthread_mutex_init -import platform.posix.pthread_mutex_lock -import platform.posix.pthread_mutex_tVar -import platform.posix.pthread_mutex_unlock - -internal actual class PoolLock actual constructor() { - private val isActive = AtomicBoolean(true) - private val mutex = nativeHeap.alloc() - .apply { pthread_mutex_init(ptr, null) } - private val cond = nativeHeap.alloc() - .apply { pthread_cond_init(ptr, null) } - - actual fun withLock( - action: CriticalSection.() -> R - ): R { - check(isActive.value) - pthread_mutex_lock(mutex.ptr) - - val result: R - - try { - result = action(CriticalSection()) - } finally { - pthread_mutex_unlock(mutex.ptr) - } - - return result - } - - actual fun notifyConditionChanged() { - pthread_cond_signal(cond.ptr) - } - - actual fun close(): Boolean { - if (isActive.compareAndSet(expected = true, new = false)) { - pthread_cond_destroy(cond.ptr) - pthread_mutex_destroy(mutex.ptr) - nativeHeap.free(cond) - nativeHeap.free(mutex) - return true - } - - return false - } - - actual inner class CriticalSection { - actual fun loopForConditionalResult(block: () -> R?): R { - check(isActive.value) - - var result = block() - - while (result == null) { - pthread_cond_wait(cond.ptr, mutex.ptr) - result = block() - } - - return result - } - - actual fun loopUntilConditionalResult(block: () -> Boolean) { - check(isActive.value) - - while (!block()) { - pthread_cond_wait(cond.ptr, mutex.ptr) - } - } - } -} diff --git a/drivers/native-driver-strict/src/mingwX86Main/kotlin/app/cash/sqldelight/driver/native/util/PoolLock.kt b/drivers/native-driver-strict/src/mingwX86Main/kotlin/app/cash/sqldelight/driver/native/util/PoolLock.kt deleted file mode 100644 index 331f846dadb..00000000000 --- a/drivers/native-driver-strict/src/mingwX86Main/kotlin/app/cash/sqldelight/driver/native/util/PoolLock.kt +++ /dev/null @@ -1,81 +0,0 @@ -package app.cash.sqldelight.driver.native.util - -import co.touchlab.stately.concurrency.AtomicBoolean -import kotlinx.cinterop.alloc -import kotlinx.cinterop.free -import kotlinx.cinterop.nativeHeap -import kotlinx.cinterop.ptr -import platform.posix.pthread_cond_destroy -import platform.posix.pthread_cond_init -import platform.posix.pthread_cond_signal -import platform.posix.pthread_cond_tVar -import platform.posix.pthread_cond_wait -import platform.posix.pthread_mutex_destroy -import platform.posix.pthread_mutex_init -import platform.posix.pthread_mutex_lock -import platform.posix.pthread_mutex_tVar -import platform.posix.pthread_mutex_unlock - -internal actual class PoolLock actual constructor() { - private val isActive = AtomicBoolean(true) - private val mutex = nativeHeap.alloc() - .apply { pthread_mutex_init(ptr, null) } - private val cond = nativeHeap.alloc() - .apply { pthread_cond_init(ptr, null) } - - actual fun withLock( - action: CriticalSection.() -> R - ): R { - check(isActive.value) - pthread_mutex_lock(mutex.ptr) - - val result: R - - try { - result = action(CriticalSection()) - } finally { - pthread_mutex_unlock(mutex.ptr) - } - - return result - } - - actual fun notifyConditionChanged() { - pthread_cond_signal(cond.ptr) - } - - actual fun close(): Boolean { - if (isActive.compareAndSet(expected = true, new = false)) { - pthread_cond_destroy(cond.ptr) - pthread_mutex_destroy(mutex.ptr) - nativeHeap.free(cond) - nativeHeap.free(mutex) - return true - } - - return false - } - - actual inner class CriticalSection { - actual fun loopForConditionalResult(block: () -> R?): R { - check(isActive.value) - - var result = block() - - while (result == null) { - pthread_cond_wait(cond.ptr, mutex.ptr) - result = block() - } - - return result - } - - actual fun loopUntilConditionalResult(block: () -> Boolean) { - check(isActive.value) - - while (!block()) { - pthread_cond_wait(cond.ptr, mutex.ptr) - } - } - } -} diff --git a/drivers/native-driver-strict/src/nativeDarwinMain/kotlin/app/cash/sqldelight/driver/native/util/NativeCache.kt b/drivers/native-driver-strict/src/nativeDarwinMain/kotlin/app/cash/sqldelight/driver/native/util/NativeCache.kt deleted file mode 100644 index 1d125394702..00000000000 --- a/drivers/native-driver-strict/src/nativeDarwinMain/kotlin/app/cash/sqldelight/driver/native/util/NativeCache.kt +++ /dev/null @@ -1,49 +0,0 @@ -package app.cash.sqldelight.driver.native.util - -import platform.Foundation.NSMutableDictionary -import platform.Foundation.allValues -import platform.Foundation.setValue -import platform.Foundation.valueForKey -import kotlin.native.concurrent.freeze - -internal actual fun nativeCache(): NativeCache = - NativeCacheImpl() - -private class NativeCacheImpl : NativeCache { - private val dictionary = NSMutableDictionary() - private val lock = PoolLock() - - @Suppress("UNCHECKED_CAST") - override fun put(key: String, value: T?): T? { - value.freeze() - return lock.withLock { - val r = dictionary.valueForKey(key) - dictionary.setValue(value, key) - r as? T - } - } - - @Suppress("UNCHECKED_CAST") - override fun getOrCreate(key: String, block: () -> T): T = lock.withLock { - val r = dictionary.valueForKey(key) as? T - r ?: block().let { newValue -> - newValue.freeze() - dictionary.setValue(newValue, key) - newValue - } - } - - @Suppress("UNCHECKED_CAST") - override fun remove(key: String): T? = lock.withLock { - val r = dictionary.valueForKey(key) - dictionary.removeObjectForKey(key) - r as? T - } - - @Suppress("UNCHECKED_CAST") - override fun cleanUp(block: (T) -> Unit) = lock.withLock { - dictionary.allValues.forEach { entry -> - entry?.let { block(entry as T) } - } - } -} diff --git a/drivers/native-driver-strict/src/nativeDarwinMain/kotlin/app/cash/sqldelight/driver/native/util/PoolLock.kt b/drivers/native-driver-strict/src/nativeDarwinMain/kotlin/app/cash/sqldelight/driver/native/util/PoolLock.kt deleted file mode 100644 index 3896457a75e..00000000000 --- a/drivers/native-driver-strict/src/nativeDarwinMain/kotlin/app/cash/sqldelight/driver/native/util/PoolLock.kt +++ /dev/null @@ -1,81 +0,0 @@ -package app.cash.sqldelight.driver.native.util - -import co.touchlab.stately.concurrency.AtomicBoolean -import kotlinx.cinterop.alloc -import kotlinx.cinterop.free -import kotlinx.cinterop.nativeHeap -import kotlinx.cinterop.ptr -import platform.posix.pthread_cond_destroy -import platform.posix.pthread_cond_init -import platform.posix.pthread_cond_signal -import platform.posix.pthread_cond_t -import platform.posix.pthread_cond_wait -import platform.posix.pthread_mutex_destroy -import platform.posix.pthread_mutex_init -import platform.posix.pthread_mutex_lock -import platform.posix.pthread_mutex_t -import platform.posix.pthread_mutex_unlock - -internal actual class PoolLock actual constructor() { - private val isActive = AtomicBoolean(true) - private val mutex = nativeHeap.alloc() - .apply { pthread_mutex_init(ptr, null) } - private val cond = nativeHeap.alloc() - .apply { pthread_cond_init(ptr, null) } - - actual fun withLock( - action: CriticalSection.() -> R - ): R { - check(isActive.value) - pthread_mutex_lock(mutex.ptr) - - val result: R - - try { - result = action(CriticalSection()) - } finally { - pthread_mutex_unlock(mutex.ptr) - } - - return result - } - - actual fun notifyConditionChanged() { - pthread_cond_signal(cond.ptr) - } - - actual fun close(): Boolean { - if (isActive.compareAndSet(expected = true, new = false)) { - pthread_cond_destroy(cond.ptr) - pthread_mutex_destroy(mutex.ptr) - nativeHeap.free(cond) - nativeHeap.free(mutex) - return true - } - - return false - } - - actual inner class CriticalSection { - actual fun loopForConditionalResult(block: () -> R?): R { - check(isActive.value) - - var result = block() - - while (result == null) { - pthread_cond_wait(cond.ptr, mutex.ptr) - result = block() - } - - return result - } - - actual fun loopUntilConditionalResult(block: () -> Boolean) { - check(isActive.value) - - while (!block()) { - pthread_cond_wait(cond.ptr, mutex.ptr) - } - } - } -} diff --git a/drivers/native-driver-strict/src/nativeMain/kotlin/app/cash/sqldelight/driver/native/Borrowed.kt b/drivers/native-driver-strict/src/nativeMain/kotlin/app/cash/sqldelight/driver/native/Borrowed.kt deleted file mode 100644 index f3252179e2a..00000000000 --- a/drivers/native-driver-strict/src/nativeMain/kotlin/app/cash/sqldelight/driver/native/Borrowed.kt +++ /dev/null @@ -1,6 +0,0 @@ -package app.cash.sqldelight.driver.native - -internal interface Borrowed { - val value: T - fun release() -} diff --git a/drivers/native-driver-strict/src/nativeMain/kotlin/app/cash/sqldelight/driver/native/NativeSqlDatabase.kt b/drivers/native-driver-strict/src/nativeMain/kotlin/app/cash/sqldelight/driver/native/NativeSqlDatabase.kt deleted file mode 100644 index 79a2f5a393b..00000000000 --- a/drivers/native-driver-strict/src/nativeMain/kotlin/app/cash/sqldelight/driver/native/NativeSqlDatabase.kt +++ /dev/null @@ -1,414 +0,0 @@ -package app.cash.sqldelight.driver.native - -import app.cash.sqldelight.Query -import app.cash.sqldelight.Transacter -import app.cash.sqldelight.db.Closeable -import app.cash.sqldelight.db.SqlCursor -import app.cash.sqldelight.db.SqlDriver -import app.cash.sqldelight.db.SqlPreparedStatement -import app.cash.sqldelight.driver.native.util.nativeCache -import co.touchlab.sqliter.DatabaseConfiguration -import co.touchlab.sqliter.DatabaseConnection -import co.touchlab.sqliter.DatabaseManager -import co.touchlab.sqliter.Statement -import co.touchlab.sqliter.createDatabaseManager -import co.touchlab.sqliter.withStatement -import co.touchlab.stately.collections.SharedHashMap -import co.touchlab.stately.collections.SharedSet -import co.touchlab.stately.concurrency.ThreadLocalRef -import co.touchlab.stately.concurrency.value -import kotlin.native.concurrent.ensureNeverFrozen - -sealed class ConnectionWrapper : SqlDriver { - internal abstract fun accessConnection( - readOnly: Boolean, - block: ThreadConnection.() -> R - ): R - - final override fun execute( - identifier: Int?, - sql: String, - parameters: Int, - binders: (SqlPreparedStatement.() -> Unit)? - ) { - accessConnection(false) { - val statement = useStatement(identifier, sql) - if (binders != null) { - try { - SqliterStatement(statement).binders() - } catch (t: Throwable) { - statement.resetStatement() - clearIfNeeded(identifier, statement) - throw t - } - } - - statement.execute() - statement.resetStatement() - clearIfNeeded(identifier, statement) - } - } - - final override fun executeQuery( - identifier: Int?, - sql: String, - parameters: Int, - binders: (SqlPreparedStatement.() -> Unit)? - ): SqlCursor { - return accessConnection(true) { - val statement = getStatement(identifier, sql) - - if (binders != null) { - try { - SqliterStatement(statement).binders() - } catch (t: Throwable) { - statement.resetStatement() - safePut(identifier, statement) - throw t - } - } - - val cursor = statement.query() - - SqliterSqlCursor(cursor) { - statement.resetStatement() - if (closed) - statement.finalizeStatement() - safePut(identifier, statement) - } - } - } -} - -/** - * Native driver implementation. - * - * The driver creates two connection pools, which default to 1 connection maximum. There is a reader pool, which - * handles all query requests outside of a transaction. The other pool is the transaction pool, which handles - * all transactions and write requests outside of a transaction. - * - * When a transaction is started, that thread is aligned with a transaction pool connection. Attempting a write or - * starting another transaction, if no connections are available, will cause the caller to wait. - * - * You can have multiple connections in the transaction pool, but this would only be useful for read transactions. Writing - * from multiple connections in an overlapping manner can be problematic. - * - * Aligning a transaction to a thread means you cannot operate on a single transaction from multiple threads. - * However, it would be difficult to find a use case where this would be desirable or safe. Currently, the native - * implementation of kotlinx.coroutines does not use thread pooling. When that changes, we'll need a way to handle - * transaction/connection alignment similar to what the Android/JVM driver implemented. - * - * https://medium.com/androiddevelopers/threading-models-in-coroutines-and-android-sqlite-api-6cab11f7eb90 - * - * To use SqlDelight during create/upgrade processes, you can alternatively wrap a real connection - * with wrapConnection. - * - * SqlPreparedStatement instances also do not point to real resources until either execute or - * executeQuery is called. The SqlPreparedStatement structure also maintains a thread-aligned - * instance which accumulates bind calls. Those are replayed on a real SQLite statement instance - * when execute or executeQuery is called. This avoids race conditions with bind calls. - */ -class NativeSqliteDriver( - private val databaseManager: DatabaseManager, - maxReaderConnections: Int = 1, -) : ConnectionWrapper(), SqlDriver { - constructor( - configuration: DatabaseConfiguration, - maxReaderConnections: Int = 1 - ) : this( - databaseManager = createDatabaseManager(configuration), - maxReaderConnections = maxReaderConnections - ) - - constructor( - schema: SqlDriver.Schema, - name: String, - maxReaderConnections: Int = 1 - ) : this( - configuration = DatabaseConfiguration( - name = name, - version = schema.version, - create = { connection -> - wrapConnection(connection) { schema.create(it) } - }, - upgrade = { connection, oldVersion, newVersion -> - wrapConnection(connection) { schema.migrate(it, oldVersion, newVersion) } - } - ), - maxReaderConnections = maxReaderConnections - ) - - // A pool of reader connections used by all operations not in a transaction - internal val transactionPool: Pool - internal val readerPool: Pool - - // Once a transaction is started and connection borrowed, it will be here, but only for that - // thread - private val borrowedConnectionThread = ThreadLocalRef>() - private val listeners = SharedHashMap>() - - init { - // Single connection for transactions - transactionPool = Pool(1) { - ThreadConnection(databaseManager.createMultiThreadedConnection()) { _ -> - borrowedConnectionThread.let { - it.get()?.release() - it.value = null - } - } - } - - val maxReaderConnectionsForConfig: Int = when { - databaseManager.configuration.inMemory -> 1 // Memory db's are single connection, generally. You can use named connections, but there are other issues that need to be designed for - else -> maxReaderConnections - } - readerPool = Pool(maxReaderConnectionsForConfig) { - val connection = databaseManager.createMultiThreadedConnection() - connection.withStatement("PRAGMA query_only = 1") { execute() } // Ensure read only - ThreadConnection(connection) { - throw UnsupportedOperationException("Should never be in a transaction") - } - } - } - - override fun addListener(listener: Query.Listener, queryKeys: Array) { - queryKeys.forEach { - listeners.getOrPut(it, { SharedSet() }).add(listener) - } - } - - override fun removeListener(listener: Query.Listener, queryKeys: Array) { - queryKeys.forEach { - listeners[it]?.remove(listener) - } - } - - override fun notifyListeners(queryKeys: Array) { - val listenersToNotify = SharedSet() - queryKeys.forEach { listeners[it]?.let(listenersToNotify::addAll) } - listenersToNotify.forEach(Query.Listener::queryResultsChanged) - } - - override fun currentTransaction(): Transacter.Transaction? { - return borrowedConnectionThread.get()?.value?.transaction?.value - } - - override fun newTransaction(): Transacter.Transaction { - val alreadyBorrowed = borrowedConnectionThread.get() - return if (alreadyBorrowed == null) { - val borrowed = transactionPool.borrowEntry() - - try { - val trans = borrowed.value.newTransaction() - - borrowedConnectionThread.value = borrowed - trans - } catch (e: Throwable) { - // Unlock on failure. - borrowed.release() - throw e - } - } else { - alreadyBorrowed.value.newTransaction() - } - } - - /** - * If we're in a transaction, then I have a connection. Otherwise use shared. - */ - override fun accessConnection( - readOnly: Boolean, - block: ThreadConnection.() -> R - ): R { - val mine = borrowedConnectionThread.get() - - return if (readOnly) { - // Code intends to read, which doesn't need to block - if (mine != null) { - mine.value.block() - } else { - readerPool.access(block) - } - } else { - // Code intends to write, for which we're managing locks in code - if (mine != null) { - mine.value.block() - } else { - transactionPool.access(block) - } - } - } - - override fun close() { - transactionPool.close() - readerPool.close() - } -} - -/** - * Sqliter's DatabaseConfiguration takes lambda arguments for it's create and upgrade operations, - * which each take a DatabaseConnection argument. Use wrapConnection to have SqlDelight access this - * passed connection and avoid the pooling that the full SqlDriver instance performs. - * - * Note that queries created during this operation will be cleaned up. If holding onto a cursor from - * a wrap call, it will no longer be viable. - */ -fun wrapConnection( - connection: DatabaseConnection, - block: (SqlDriver) -> Unit -) { - val conn = SqliterWrappedConnection(ThreadConnection(connection) {}) - try { - block(conn) - } finally { - conn.close() - } -} - -/** - * SqlDriverConnection that wraps a Sqliter connection. Useful for migration tasks, or if you - * don't want the polling. - */ -internal class SqliterWrappedConnection( - private val threadConnection: ThreadConnection -) : ConnectionWrapper(), - SqlDriver { - override fun currentTransaction(): Transacter.Transaction? = threadConnection.transaction.value - - override fun newTransaction(): Transacter.Transaction = threadConnection.newTransaction() - - override fun accessConnection( - readOnly: Boolean, - block: ThreadConnection.() -> R - ): R = threadConnection.block() - - override fun addListener(listener: Query.Listener, queryKeys: Array) { - // No-op - } - - override fun removeListener(listener: Query.Listener, queryKeys: Array) { - // No-op - } - - override fun notifyListeners(queryKeys: Array) { - // No-op - } - - override fun close() { - threadConnection.cleanUp() - } -} - -/** - * Wraps and manages a "real" database connection. - * - * SQLite statements are specific to connections, and must be finalized explicitly. Cursors are - * backed by a statement resource, so we keep links to open cursors to allow us to close them out - * properly in cases where the user does not. - */ -internal class ThreadConnection( - private val connection: DatabaseConnection, - private val onEndTransaction: (ThreadConnection) -> Unit -) : Closeable { - internal val transaction = ThreadLocalRef() - internal val closed: Boolean - get() = connection.closed - - internal val statementCache = nativeCache() - - fun safePut(identifier: Int?, statement: Statement) { - val removed = if (identifier == null) { - statement - } else { - statementCache.put(identifier.toString(), statement) - } - removed?.finalizeStatement() - } - - fun getStatement(identifier: Int?, sql: String): Statement { - val statement = removeCreateStatement(identifier, sql) - return statement - } - - fun useStatement(identifier: Int?, sql: String): Statement { - return if (identifier != null) { - statementCache.getOrCreate(identifier.toString()) { - connection.createStatement(sql) - } - } else { - connection.createStatement(sql) - } - } - - fun clearIfNeeded(identifier: Int?, statement: Statement) { - if (identifier == null) { - statement.finalizeStatement() - } - } - - /** - * For cursors. Cursors are actually backed by SQLite statement instances, so they need to be - * removed from the cache when in use. We're giving out a SQLite resource here, so extra care. - */ - private fun removeCreateStatement(identifier: Int?, sql: String): Statement { - if (identifier != null) { - val cached = statementCache.remove(identifier.toString()) - if (cached != null) - return cached - } - - return connection.createStatement(sql) - } - - fun newTransaction(): Transacter.Transaction { - val enclosing = transaction.value - - // Create here, in case we bomb... - if (enclosing == null) { - connection.beginTransaction() - } - - val trans = Transaction(enclosing) - transaction.value = trans - - return trans - } - - /** - * This should only be called directly from wrapConnection. Clean resources without actually closing - * the underlying connection. - */ - internal fun cleanUp() { - statementCache.cleanUp { - it.finalizeStatement() - } - } - - override fun close() { - cleanUp() - connection.close() - } - - private inner class Transaction( - override val enclosingTransaction: Transacter.Transaction? - ) : Transacter.Transaction() { - init { ensureNeverFrozen() } - - override fun endTransaction(successful: Boolean) { - transaction.value = enclosingTransaction - - if (enclosingTransaction == null) { - try { - if (successful) { - connection.setTransactionSuccessful() - } - - connection.endTransaction() - } finally { - // Release if we have - onEndTransaction(this@ThreadConnection) - } - } - } - } -} diff --git a/drivers/native-driver-strict/src/nativeMain/kotlin/app/cash/sqldelight/driver/native/Pool.kt b/drivers/native-driver-strict/src/nativeMain/kotlin/app/cash/sqldelight/driver/native/Pool.kt deleted file mode 100644 index 9f213039fa4..00000000000 --- a/drivers/native-driver-strict/src/nativeMain/kotlin/app/cash/sqldelight/driver/native/Pool.kt +++ /dev/null @@ -1,122 +0,0 @@ -package app.cash.sqldelight.driver.native - -import app.cash.sqldelight.db.Closeable -import app.cash.sqldelight.driver.native.util.PoolLock -import co.touchlab.stately.concurrency.AtomicBoolean -import kotlin.native.concurrent.AtomicReference -import kotlin.native.concurrent.freeze - -/** - * A shared pool of connections. Borrowing is blocking when all connections are in use, and the pool has reached its - * designated capacity. - */ -internal class Pool(internal val capacity: Int, private val producer: () -> T) { - /** - * Hold a list of active connections. If it is null, it means the MultiPool has been closed. - */ - private val entriesRef = AtomicReference?>(listOf().freeze()) - private val poolLock = PoolLock() - - /** - * For test purposes only - */ - internal fun entryCount(): Int = poolLock.withLock { - entriesRef.value?.size ?: 0 - } - - fun borrowEntry(): Borrowed { - val snapshot = entriesRef.value ?: throw ClosedMultiPoolException - - // Fastpath: Borrow the first available entry. - val firstAvailable = snapshot.firstOrNull { it.tryToAcquire() } - - if (firstAvailable != null) { - return firstAvailable.asBorrowed(poolLock) - } - - // Slowpath: Create a new entry if capacity limit has not been reached, or wait for the next available entry. - val nextAvailable = poolLock.withLock { - // Reload the list since it could've been updated by other threads concurrently. - val entries = entriesRef.value ?: throw ClosedMultiPoolException - - if (entries.count() < capacity) { - // Capacity hasn't been reached — create a new entry to serve this call. - val newEntry = Entry(producer()) - val done = newEntry.tryToAcquire() - check(done) - - entriesRef.value = (entries + listOf(newEntry)).freeze() - return@withLock newEntry - } else { - // Capacity is reached — wait for the next available entry. - return@withLock loopForConditionalResult { - // Reload the list, since the thread can be suspended here while the list of entries has been modified. - val innerEntries = entriesRef.value ?: throw ClosedMultiPoolException - innerEntries.firstOrNull { it.tryToAcquire() } - } - } - } - - return nextAvailable.asBorrowed(poolLock) - } - - fun access(action: (T) -> R): R { - val borrowed = borrowEntry() - return try { - action(borrowed.value) - } finally { - borrowed.release() - } - } - - fun close() { - if (!poolLock.close()) - return - - val entries = entriesRef.value - val done = entriesRef.compareAndSet(entries, null) - check(done) - - entries?.forEach { it.value.close() } - } - - inner class Entry(val value: T) { - val isAvailable = AtomicBoolean(true) - - init { freeze() } - - fun tryToAcquire(): Boolean = isAvailable.compareAndSet(expected = true, new = false) - - fun asBorrowed(poolLock: PoolLock): Borrowed = object : Borrowed { - override val value: T - get() = this@Entry.value - - override fun release() { - /** - * Mark-as-available should be done before signalling blocked threads via [PoolLock.notifyConditionChanged], - * since the happens-before relationship guarantees the woken thread to see the - * available entry (if not having been taken by other threads during the wake-up lead time). - */ - - val done = isAvailable.compareAndSet(expected = false, new = true) - check(done) - - // While signalling blocked threads does not require locking, doing so avoids a subtle race - // condition in which: - // - // 1. a [loopForConditionalResult] iteration in [borrowEntry] slow path is happening concurrently; - // 2. the iteration fails to see the atomic `isAvailable = true` above; - // 3. we signal availability here but it is a no-op due to no waiting blocker; and finally - // 4. the iteration entered an indefinite blocking wait, not being aware of us having signalled availability here. - // - // By acquiring the pool lock first, signalling cannot happen concurrently with the loop - // iterations in [borrowEntry], thus eliminating the race condition. - poolLock.withLock { - poolLock.notifyConditionChanged() - } - } - } - } -} - -private val ClosedMultiPoolException get() = IllegalStateException("Attempt to access a closed MultiPool.") diff --git a/drivers/native-driver-strict/src/nativeMain/kotlin/app/cash/sqldelight/driver/native/SqliterSqlCursor.kt b/drivers/native-driver-strict/src/nativeMain/kotlin/app/cash/sqldelight/driver/native/SqliterSqlCursor.kt deleted file mode 100644 index 67dad49ac39..00000000000 --- a/drivers/native-driver-strict/src/nativeMain/kotlin/app/cash/sqldelight/driver/native/SqliterSqlCursor.kt +++ /dev/null @@ -1,37 +0,0 @@ -package app.cash.sqldelight.driver.native - -import app.cash.sqldelight.db.SqlCursor -import co.touchlab.sqliter.Cursor -import co.touchlab.sqliter.getBytesOrNull -import co.touchlab.sqliter.getDoubleOrNull -import co.touchlab.sqliter.getLongOrNull -import co.touchlab.sqliter.getStringOrNull - -/** - * Wrapper for cursor calls. Cursors point to real SQLite statements, so we need to be careful with - * them. If dev closes the outer structure, this will get closed as well, which means it could start - * throwing errors if you're trying to access it. - */ -internal class SqliterSqlCursor( - private val cursor: Cursor, - private val recycler: () -> Unit -) : SqlCursor { - - override fun close() { - recycler() - } - - override fun getBytes(index: Int): ByteArray? = cursor.getBytesOrNull(index) - - override fun getDouble(index: Int): Double? = cursor.getDoubleOrNull(index) - - override fun getLong(index: Int): Long? = cursor.getLongOrNull(index) - - override fun getString(index: Int): String? = cursor.getStringOrNull(index) - - override fun getBoolean(index: Int): Boolean? { - return (cursor.getLongOrNull(index) ?: return null) == 1L - } - - override fun next(): Boolean = cursor.next() -} diff --git a/drivers/native-driver-strict/src/nativeMain/kotlin/app/cash/sqldelight/driver/native/SqliterStatement.kt b/drivers/native-driver-strict/src/nativeMain/kotlin/app/cash/sqldelight/driver/native/SqliterStatement.kt deleted file mode 100644 index 87e7ceec2ee..00000000000 --- a/drivers/native-driver-strict/src/nativeMain/kotlin/app/cash/sqldelight/driver/native/SqliterStatement.kt +++ /dev/null @@ -1,42 +0,0 @@ -package app.cash.sqldelight.driver.native - -import app.cash.sqldelight.db.SqlPreparedStatement -import co.touchlab.sqliter.Statement -import co.touchlab.sqliter.bindBlob -import co.touchlab.sqliter.bindDouble -import co.touchlab.sqliter.bindLong -import co.touchlab.sqliter.bindString - -/** - * @param [recycle] A function which recycles any resources this statement is backed by. - */ -internal class SqliterStatement( - private val statement: Statement -) : SqlPreparedStatement { - override fun bindBytes(index: Int, bytes: ByteArray?) { - statement.bindBlob(index, bytes) - } - - override fun bindLong(index: Int, long: Long?) { - statement.bindLong(index, long) - } - - override fun bindDouble(index: Int, double: Double?) { - statement.bindDouble(index, double) - } - - override fun bindString(index: Int, string: String?) { - statement.bindString(index, string) - } - - override fun bindBoolean(index: Int, boolean: Boolean?) { - statement.bindLong( - index, - when (boolean) { - null -> null - true -> 1L - false -> 0L - } - ) - } -} diff --git a/drivers/native-driver-strict/src/nativeMain/kotlin/app/cash/sqldelight/driver/native/util/NativeCache.kt b/drivers/native-driver-strict/src/nativeMain/kotlin/app/cash/sqldelight/driver/native/util/NativeCache.kt deleted file mode 100644 index 1fa67375ce3..00000000000 --- a/drivers/native-driver-strict/src/nativeMain/kotlin/app/cash/sqldelight/driver/native/util/NativeCache.kt +++ /dev/null @@ -1,10 +0,0 @@ -package app.cash.sqldelight.driver.native.util - -internal interface NativeCache { - fun put(key: String, value: T?): T? - fun getOrCreate(key: String, block: () -> T): T - fun remove(key: String): T? - fun cleanUp(block: (T) -> Unit) -} - -internal expect fun nativeCache(): NativeCache diff --git a/drivers/native-driver-strict/src/nativeMain/kotlin/app/cash/sqldelight/driver/native/util/PoolLock.kt b/drivers/native-driver-strict/src/nativeMain/kotlin/app/cash/sqldelight/driver/native/util/PoolLock.kt deleted file mode 100644 index ac852d0f82e..00000000000 --- a/drivers/native-driver-strict/src/nativeMain/kotlin/app/cash/sqldelight/driver/native/util/PoolLock.kt +++ /dev/null @@ -1,32 +0,0 @@ -package app.cash.sqldelight.driver.native.util - -@Suppress("NO_ACTUAL_FOR_EXPECT") -internal expect class PoolLock() { - fun withLock( - action: CriticalSection.() -> R - ): R - - /** - * Select one blocked thread in [CriticalSection.loopForConditionalResult] to be woken up for - * re-evaluation, if any. - */ - fun notifyConditionChanged() - - fun close(): Boolean - - inner class CriticalSection { - /** - * Evaluate the given lambda of a conditional result in an infinite loop, until the result is - * available. - * - * If null is produced, the current thread enters suspension, and are only woken up for - * re-evaluation by a subsequent [PoolLock.notifyConditionChanged] call. Note that the lock - * would not be held by the current thread during its suspension. This allows resources - * protected by the same lock to remain accessible by other threads, provided that they do not - * depend on the same conditional result. - */ - fun loopForConditionalResult(block: () -> R?): R - - fun loopUntilConditionalResult(block: () -> Boolean) - } -} diff --git a/drivers/native-driver-strict/src/nativeTest/kotlin/com/squareup/sqldelight/drivers/native/LazyDriverBaseTest.kt b/drivers/native-driver-strict/src/nativeTest/kotlin/com/squareup/sqldelight/drivers/native/LazyDriverBaseTest.kt deleted file mode 100644 index b008d712700..00000000000 --- a/drivers/native-driver-strict/src/nativeTest/kotlin/com/squareup/sqldelight/drivers/native/LazyDriverBaseTest.kt +++ /dev/null @@ -1,108 +0,0 @@ -package com.squareup.sqldelight.drivers.native - -import app.cash.sqldelight.TransacterImpl -import app.cash.sqldelight.db.SqlDriver -import app.cash.sqldelight.driver.native.NativeSqliteDriver -import app.cash.sqldelight.driver.native.wrapConnection -import co.touchlab.sqliter.DatabaseConfiguration -import co.touchlab.sqliter.DatabaseFileContext.deleteDatabase -import co.touchlab.sqliter.DatabaseManager -import co.touchlab.sqliter.createDatabaseManager -import co.touchlab.stately.freeze -import kotlin.test.AfterTest -import kotlin.test.BeforeTest - -abstract class LazyDriverBaseTest { - protected lateinit var driver: NativeSqliteDriver - private var manager: DatabaseManager? = null - - protected abstract val memory: Boolean - - private val transacterInternal: TransacterImpl by lazy { - object : TransacterImpl(driver) {} - } - - protected val transacter: TransacterImpl - get() { - val t = transacterInternal - t.freeze() - return t - } - - @BeforeTest fun setup() { - driver = setupDatabase(schema = defaultSchema()) - } - - @AfterTest fun tearDown() { - driver.close() - } - - protected fun defaultSchema(): SqlDriver.Schema { - return object : SqlDriver.Schema { - override val version: Int = 1 - - override fun create(driver: SqlDriver) { - driver.execute( - 20, - """ - |CREATE TABLE test ( - | id INTEGER PRIMARY KEY, - | value TEXT - |); - """.trimMargin(), - 0 - ) - driver.execute( - 30, - """ - |CREATE TABLE nullability_test ( - | id INTEGER PRIMARY KEY, - | integer_value INTEGER, - | text_value TEXT, - | blob_value BLOB, - | real_value REAL - |); - """.trimMargin(), - 0 - ) - } - - override fun migrate( - driver: SqlDriver, - oldVersion: Int, - newVersion: Int - ) { - // No-op. - } - } - } - - protected fun altInit(config: DatabaseConfiguration) { - driver.close() - driver = setupDatabase(defaultSchema(), config) - } - - private fun setupDatabase( - schema: SqlDriver.Schema, - config: DatabaseConfiguration = defaultConfiguration(schema) - ): NativeSqliteDriver { - deleteDatabase(config.name!!) - // This isn't pretty, but just for test - manager = createDatabaseManager(config) - return NativeSqliteDriver(manager!!) - } - - protected fun defaultConfiguration(schema: SqlDriver.Schema): DatabaseConfiguration { - return DatabaseConfiguration( - name = "testdb", - version = 1, - create = { connection -> - wrapConnection(connection) { - schema.create(it) - } - }, - extendedConfig = DatabaseConfiguration.Extended(busyTimeout = 20_000), - inMemory = true - ) - } -} diff --git a/drivers/native-driver-strict/src/nativeTest/kotlin/com/squareup/sqldelight/drivers/native/NativeDriverTest.kt b/drivers/native-driver-strict/src/nativeTest/kotlin/com/squareup/sqldelight/drivers/native/NativeDriverTest.kt deleted file mode 100644 index 9b1d04141d7..00000000000 --- a/drivers/native-driver-strict/src/nativeTest/kotlin/com/squareup/sqldelight/drivers/native/NativeDriverTest.kt +++ /dev/null @@ -1,14 +0,0 @@ -package com.squareup.sqldelight.drivers.native - -import app.cash.sqldelight.db.SqlDriver -import app.cash.sqldelight.driver.native.NativeSqliteDriver -import co.touchlab.sqliter.DatabaseFileContext.deleteDatabase -import com.squareup.sqldelight.driver.test.DriverTest - -class NativeDriverTest : DriverTest() { - override fun setupDatabase(schema: SqlDriver.Schema): SqlDriver { - val name = "testdb" - deleteDatabase(name) - return NativeSqliteDriver(schema, name) - } -} diff --git a/drivers/native-driver-strict/src/nativeTest/kotlin/com/squareup/sqldelight/drivers/native/NativeQueryTest.kt b/drivers/native-driver-strict/src/nativeTest/kotlin/com/squareup/sqldelight/drivers/native/NativeQueryTest.kt deleted file mode 100644 index 62669adc566..00000000000 --- a/drivers/native-driver-strict/src/nativeTest/kotlin/com/squareup/sqldelight/drivers/native/NativeQueryTest.kt +++ /dev/null @@ -1,14 +0,0 @@ -package com.squareup.sqldelight.drivers.native - -import app.cash.sqldelight.db.SqlDriver -import app.cash.sqldelight.driver.native.NativeSqliteDriver -import co.touchlab.sqliter.DatabaseFileContext -import com.squareup.sqldelight.driver.test.QueryTest - -class NativeQueryTest : QueryTest() { - override fun setupDatabase(schema: SqlDriver.Schema): SqlDriver { - val name = "testdb" - DatabaseFileContext.deleteDatabase(name) - return NativeSqliteDriver(schema, name) - } -} diff --git a/drivers/native-driver-strict/src/nativeTest/kotlin/com/squareup/sqldelight/drivers/native/NativeSqliteDriverTest.kt.ignore b/drivers/native-driver-strict/src/nativeTest/kotlin/com/squareup/sqldelight/drivers/native/NativeSqliteDriverTest.kt.ignore deleted file mode 100644 index 64eaaa3d620..00000000000 --- a/drivers/native-driver-strict/src/nativeTest/kotlin/com/squareup/sqldelight/drivers/native/NativeSqliteDriverTest.kt.ignore +++ /dev/null @@ -1,412 +0,0 @@ -package com.squareup.sqldelight.drivers.native - -import co.touchlab.sqliter.DatabaseConfiguration -import co.touchlab.sqliter.createDatabaseManager -import co.touchlab.stately.collections.frozenLinkedList -import co.touchlab.stately.concurrency.AtomicBoolean -import co.touchlab.stately.concurrency.AtomicInt -import co.touchlab.testhelp.concurrency.ThreadOperations -import co.touchlab.testhelp.concurrency.sleep -import app.cash.sqldelight.Transacter -import app.cash.sqldelight.db.SqlCursor -import kotlin.test.Test -import kotlin.test.assertEquals -import kotlin.test.assertFails -import kotlin.test.assertFalse -import kotlin.test.assertSame -import kotlin.test.assertFails - -//Run tests with WAL db -class NativeSqliteDriverTestWAL : NativeSqliteDriverTest() { - override val memory: Boolean = false -} - -//Run tests with memory db -class NativeSqliteDriverTestMemory : NativeSqliteDriverTest() { - override val memory: Boolean = true - - @Test - fun `wrapConnection does not close connection`() { - val closed = AtomicBoolean(true) - val config = DatabaseConfiguration( - name = "memorydb", - version = 1, - inMemory = true, - create = { connection -> - wrapConnection(connection) { - defaultSchema().create(it) - } - - closed.value = connection.closed - }) - val dbm = createDatabaseManager(config) - dbm.createMultiThreadedConnection().close() - - assertFalse(closed.value) - } -} - -abstract class NativeSqliteDriverTest : LazyDriverBaseTest() { - - /*@Test - fun `close with open transaction fails`(){ - transacter.transaction { - assertFails { driver.close() } - } - - //Still working? There's probably a better general test for this. - driver.queryPool.access { - val stmt = it.getStatement(null, "select * from test") - stmt.finalizeStatement() - } - }*/ - - //Kind of a sanity check - @Test - fun `threads share statement main connection multithreaded`() { - altInit(defaultConfiguration(defaultSchema()).copy(inMemory = true)) - val ops = ThreadOperations { } - val INSERTS = 10_000 - for (i in 0 until INSERTS) { - ops.exe { - driver.execute(1, "insert into test(id, value)values(?, ?)", 2) { - bindLong(1, i.toLong()) - bindString(2, "Hey $i") - } - } - } - - ops.run(10) - - assertEquals(INSERTS.toLong(), countTestRows(driver)) - val strSet = mutableSetOf() - val query = driver.executeQuery(2, "select id, value from test", 0) - var sum = 0L - while (query.next()) { - strSet.add(query.getString(1)!!) - sum += query.getLong(0)!! - } - - assertEquals(sum, (0 until INSERTS).fold(0L) { a, b -> a + b }) - assertEquals(INSERTS, strSet.size) - } - - @Test - fun `failing transaction clears lock`() { - assertFails { - transacter.transaction { - driver.execute(1, "insert into test(id, value)values(?, ?)", 2) { - bindLong(1, 1) - bindString(2, "asdf") - } - - throw IllegalStateException("Fail") - } - } - - transacter.transaction { - try { - driver.execute(1, "insert into test(id, value)values(?, ?)", 2) { - bindLong(1, 1) - bindString(2, "asdf") - } - - } catch (e: Exception) { - e.printStackTrace() - throw e - } - } - - assertEquals(1, countTestRows(driver)) - } - - @Test - fun `bad bind doens't taint future binding`() { - transacter.transaction { - assertFails { - driver.execute(1, "insert into test(id, value)values(?, ?)", 2) { - bindLong(1, 1) - bindString(3, "asdf") - } - } - } - - transacter.transaction { - driver.execute(1, "insert into test(id, value)values(?, ?)", 2) { - bindLong(1, 1) - bindString(2, "asdf") - } - } - - assertEquals(1, countTestRows(driver)) - } - - @Test - fun `failed bind dumps sqlite statement`() { - assertFails { - driver.execute(1, "insert into test(id, value) values(?, ?)", 2) { - assertEquals(0, driver.queryPool.entry.statementCache.size) - bindLong(1, 1L) - throw assertFails { bindLong(3, 1L) } - } - } - assertEquals(1, driver.queryPool.entry.statementCache.size) - } - - @Test - fun `failures don't leak resources`() { - val transacter = transacter - - val ops = ThreadOperations { } - val threads = 3 - for (i in 1..10) { - ops.exe { - exeQuiet { - transacter.transaction { - - for (i in 0 until 10) { - driver.execute(1, "insert into test(id, value)values(?, ?)", 2) { - bindLong(1, i.toLong()) - bindString(2, "Hey $i") - } - } - - throw IllegalStateException("Nah") - } - } - - exeQuiet { - driver.execute(1, "insert into test(id, value)values(?, ?)", 2) { - bindLong(1, i.toLong()) - bindString(3, "Hey $i") - } - } - - driver.executeQuery(2, "select id, value from test", 0).next() - - exeQuiet { - driver.executeQuery(3, "select id, value from toast", 0).next() - } - } - } - - ops.run(threads) - - assertEquals(10, driver.queryPool.entry.cursorCollection.size) - assertEquals(0, countTestRows(driver)) - - //If we've leaked anything the test cleanup will fail... - } - - fun exeQuiet(proc: () -> Unit) { - try { - proc() - } catch (e: Exception) { - } - } - - @Test - fun `multiple thread transactions wait and complete successfully`() { - val THREADS = 25 - val LOOPS = 50 - - val GLOBALLOOPS = 100 - - for (i in 0 until GLOBALLOOPS) { - insertThreadLoop(THREADS * LOOPS * i, THREADS, transacter, LOOPS) - } - - assertEquals((THREADS * LOOPS * GLOBALLOOPS).toLong(), countTestRows(driver)) - } - - private fun countTestRows(conn: NativeSqliteDriver): Long { - val query = conn.executeQuery(10, "select count(*) from test", 0) - query.next() - val count = query.getLong(0) - query.close() - return count!! - } - - private fun insertThreadLoop( - start: Int, - THREADS: Int, - transacter: Transacter, - LOOPS: Int - ) { - val ops = ThreadOperations { driver } - - for (i in 0 until THREADS) { - ops.exe { conn -> - - transacter.transaction { - try { - for (j in 0 until LOOPS) { - conn.execute(1, "insert into test(id, value)values(?, ?)", 2) { - val idInt = i * LOOPS + j + start - bindLong(1, idInt.toLong()) - bindString(2, "row $idInt") - } - } - } catch (e: Throwable) { - e.printStackTrace() - throw e - } - } - } - } - - ops.run(THREADS) - } - - @Test - fun `query statements cached but only 1`() { - val stmt = { driver.executeQuery(1, "select * from test", 0) } - - assertEquals(0, driver.queryPool.entry.statementCache.size) - assertEquals(0, driver.queryPool.entry.cursorCollection.size) - - val query = stmt() - - assertEquals(0, driver.queryPool.entry.statementCache.size) - assertEquals(1, driver.queryPool.entry.cursorCollection.size) - query.close() - - assertEquals(1, driver.queryPool.entry.statementCache.size) - assertEquals(0, driver.queryPool.entry.cursorCollection.size) - - val queryA = stmt() - val queryB = stmt() - - assertEquals(0, driver.queryPool.entry.statementCache.size) - assertEquals(2, driver.queryPool.entry.cursorCollection.size) - - queryA.close() - queryB.close() - - assertEquals(1, driver.queryPool.entry.statementCache.size) - assertEquals(0, driver.queryPool.entry.cursorCollection.size) - - val ops = ThreadOperations { stmt } - val THREAD = 4 - val collectCursors = frozenLinkedList() - for (i in 0 until THREAD) { - ops.exe { - collectCursors.add(stmt()) - } - } - - ops.run(THREAD) - - assertEquals(0, driver.queryPool.entry.statementCache.size) - assertEquals(THREAD, driver.queryPool.entry.cursorCollection.size) - collectCursors.forEach { it.close() } - assertEquals(1, driver.queryPool.entry.statementCache.size) - assertEquals(0, driver.queryPool.entry.cursorCollection.size) - } - - @Test - fun `query exception clears statement`() { - assertFails { - driver.executeQuery(1, "select * from test", 0) { - throw assertFails { bindLong(1, 2L) } - } - } - - assertEquals(1, driver.queryPool.entry.statementCache.size) - } - - @Test - fun `SinglePool access locked`() { - val ops = ThreadOperations { SinglePool { AtomicInt(0) } } - val failed = AtomicBoolean(false) - for (i in 0 until 150) { - ops.exe { - it.access { - val atStart = it.incrementAndGet() - if (atStart != 1) - failed.value = true - - sleep(10) - - val atEnd = it.decrementAndGet() - if (atEnd != 0) - failed.value = true - } - } - } - - ops.run(5) - assertFalse(failed.value) - } - - @Test - fun `SinglePool re-borrow fails`() { - val pool = SinglePool {} - val borrowed = pool.borrowEntry() - assertFails { pool.borrowEntry() } - borrowed.release() - } - - @Test - fun `caching by index works as expected`() { - val transacter = transacter - driver.execute(1, "insert into test(id, value)values(?, ?)", 2) { - bindLong(1, 22L) - bindString(2, "Hey 22") - } - - assertEquals(1, driver.queryPool.entry.statementCache.size) - assertEquals(0, driver.transactionPool.entry.statementCache.size) - - transacter.transaction { - driver.execute(1, "insert into test(id, value)values(?, ?)", 2) { - bindLong(1, 33L) - bindString(2, "Hey 33") - } - } - - assertEquals(1, driver.queryPool.entry.statementCache.size) - assertEquals(1, driver.transactionPool.entry.statementCache.size) - - val statement = - driver.transactionPool.entry.statementCache.entries.iterator().next().value - - transacter.transaction { - driver.execute(1, "insert into test(id, value)values(?, ?)", 2) { - bindLong(1, 34L) - bindString(2, "Hey 34") - } - } - - assertEquals(1, driver.queryPool.entry.statementCache.size) - assertEquals(1, driver.transactionPool.entry.statementCache.size) - - assertSame( - driver.transactionPool.entry.statementCache.entries.iterator().next().value, - statement) - } - - @Test - fun `null identifier doesn't cache`() { - val transacter = transacter - driver.execute(null, "insert into test(id, value)values(?, ?)", 2) { - bindLong(1, 22L) - bindString(2, "Hey 22") - } - - - assertEquals(0, driver.queryPool.entry.statementCache.size) - assertEquals(0, driver.transactionPool.entry.statementCache.size) - - transacter.transaction { - driver.execute(null, "insert into test(id, value)values(?, ?)", 2) { - bindLong(1, 23L) - bindString(2, "Hey 23") - } - } - - assertEquals(0, driver.queryPool.entry.statementCache.size) - assertEquals(0, driver.transactionPool.entry.statementCache.size) - } - -} \ No newline at end of file diff --git a/drivers/native-driver-strict/src/nativeTest/kotlin/com/squareup/sqldelight/drivers/native/NativeTransacterTest.kt b/drivers/native-driver-strict/src/nativeTest/kotlin/com/squareup/sqldelight/drivers/native/NativeTransacterTest.kt deleted file mode 100644 index 820046d2721..00000000000 --- a/drivers/native-driver-strict/src/nativeTest/kotlin/com/squareup/sqldelight/drivers/native/NativeTransacterTest.kt +++ /dev/null @@ -1,14 +0,0 @@ -package com.squareup.sqldelight.drivers.native - -import app.cash.sqldelight.db.SqlDriver -import app.cash.sqldelight.driver.native.NativeSqliteDriver -import co.touchlab.sqliter.DatabaseFileContext.deleteDatabase -import com.squareup.sqldelight.driver.test.TransacterTest - -class NativeTransacterTest : TransacterTest() { - override fun setupDatabase(schema: SqlDriver.Schema): SqlDriver { - val name = "testdb" - deleteDatabase(name) - return NativeSqliteDriver(schema, name) - } -} diff --git a/drivers/native-driver-strict/src/nativeTest/kotlin/com/squareup/sqldelight/drivers/native/connectionpool/BaseConcurrencyTest.kt b/drivers/native-driver-strict/src/nativeTest/kotlin/com/squareup/sqldelight/drivers/native/connectionpool/BaseConcurrencyTest.kt deleted file mode 100644 index 6790d461f89..00000000000 --- a/drivers/native-driver-strict/src/nativeTest/kotlin/com/squareup/sqldelight/drivers/native/connectionpool/BaseConcurrencyTest.kt +++ /dev/null @@ -1,175 +0,0 @@ -package com.squareup.sqldelight.drivers.native.connectionpool - -import app.cash.sqldelight.db.SqlDriver -import app.cash.sqldelight.driver.native.NativeSqliteDriver -import app.cash.sqldelight.driver.native.wrapConnection -import co.touchlab.sqliter.DatabaseConfiguration -import co.touchlab.sqliter.DatabaseFileContext -import co.touchlab.sqliter.JournalMode -import co.touchlab.testhelp.concurrency.currentTimeMillis -import co.touchlab.testhelp.concurrency.sleep -import kotlin.native.concurrent.AtomicInt -import kotlin.native.concurrent.Worker -import kotlin.test.AfterTest - -abstract class BaseConcurrencyTest { - fun countRows(myDriver: SqlDriver = driver): Long { - val cur = myDriver.executeQuery(0, "SELECT count(*) FROM test", 0) - try { - cur.next() - val count = cur.getLong(0) - return count!! - } finally { - cur.close() - } - } - - private var _driver: SqlDriver? = null - private var dbName: String? = null - internal val driver: SqlDriver - get() = _driver!! - - internal inner class ConcurrentContext { - private val myWorkers = arrayListOf() - internal fun createWorker(): Worker { - val w = Worker.start() - myWorkers.add(w) - return w - } - - internal fun stopWorkers() { - myWorkers.forEach { it.requestTermination() } - } - } - - internal fun runConcurrent(block: ConcurrentContext.() -> Unit) { - val context = ConcurrentContext() - try { - context.block() - } finally { - context.stopWorkers() - } - } - - fun setupDatabase( - schema: SqlDriver.Schema, - dbType: DbType, - configBase: DatabaseConfiguration, - maxReaderConnections: Int = 4 - ): SqlDriver { - // Some failing tests can leave the db in a weird state, so on each run we have a different db per test - val name = "testdb_${globalDbCount.addAndGet(1)}" - dbName = name - DatabaseFileContext.deleteDatabase(name) - val configCommon = configBase.copy( - name = name, - version = 1, - create = { conn -> - wrapConnection(conn) { driver -> - schema.create(driver) - } - } - ) - return when (dbType) { - DbType.RegularWal -> { - NativeSqliteDriver( - configCommon, - maxReaderConnections = maxReaderConnections - ) - } - DbType.RegularDelete -> { - val config = configCommon.copy(journalMode = JournalMode.DELETE) - NativeSqliteDriver( - config, - maxReaderConnections = maxReaderConnections - ) - } - DbType.InMemoryShared -> { - val config = configCommon.copy(inMemory = true) - NativeSqliteDriver( - config, - maxReaderConnections = maxReaderConnections - ) - } - DbType.InMemorySingle -> { - val config = configCommon.copy(name = null, inMemory = true) - NativeSqliteDriver( - config, - maxReaderConnections = maxReaderConnections - ) - } - } - } - - enum class DbType { - RegularWal, RegularDelete, InMemoryShared, InMemorySingle - } - - fun createDriver( - dbType: DbType, - configBase: DatabaseConfiguration = DatabaseConfiguration(name = null, version = 1, create = {}), - ): SqlDriver { - return setupDatabase( - schema = object : SqlDriver.Schema { - override val version: Int = 1 - - override fun create(driver: SqlDriver) { - driver.execute( - null, - """ - CREATE TABLE test ( - id INTEGER NOT NULL PRIMARY KEY, - value TEXT NOT NULL - ); - """.trimIndent(), - 0 - ) - } - - override fun migrate( - driver: SqlDriver, - oldVersion: Int, - newVersion: Int - ) { - // No-op. - } - }, - dbType, - configBase - ) - } - - internal fun waitFor(timeout: Long = 10_000, block: () -> Boolean) { - val start = currentTimeMillis() - var wasTimeout = false - - while (!block() && !wasTimeout) { - sleep(200) - wasTimeout = (currentTimeMillis() - start) > timeout - } - - if (wasTimeout) - throw IllegalStateException("Timeout $timeout exceeded") - } - - fun initDriver(dbType: DbType) { - _driver = createDriver(dbType) - } - - @AfterTest - fun tearDown() { - _driver?.close() - dbName?.let { DatabaseFileContext.deleteDatabase(it) } - } - - internal fun insertTestData(testData: TestData, driver: SqlDriver = this.driver) { - driver.execute(1, "INSERT INTO test VALUES (?, ?)", 2) { - bindLong(1, testData.id) - bindString(2, testData.value) - } - } - - internal data class TestData(val id: Long, val value: String) -} - -private val globalDbCount = AtomicInt(0) diff --git a/drivers/native-driver-strict/src/nativeTest/kotlin/com/squareup/sqldelight/drivers/native/connectionpool/NativeSqliteDriverConfigTest.kt b/drivers/native-driver-strict/src/nativeTest/kotlin/com/squareup/sqldelight/drivers/native/connectionpool/NativeSqliteDriverConfigTest.kt deleted file mode 100644 index c92e938c468..00000000000 --- a/drivers/native-driver-strict/src/nativeTest/kotlin/com/squareup/sqldelight/drivers/native/connectionpool/NativeSqliteDriverConfigTest.kt +++ /dev/null @@ -1,19 +0,0 @@ -package com.squareup.sqldelight.drivers.native.connectionpool - -import app.cash.sqldelight.driver.native.NativeSqliteDriver -import kotlin.test.Test -import kotlin.test.assertEquals - -/** - * Testing driver config impact on internals. This is basically verifying that connection max config is set properly - * internally. - */ -class NativeSqliteDriverConfigTest : BaseConcurrencyTest() { - @Test - fun limitReaderConnectionsForMemoryDb() { - assertEquals(1, (createDriver(DbType.InMemorySingle) as NativeSqliteDriver).readerPool.capacity) - assertEquals(1, (createDriver(DbType.InMemoryShared) as NativeSqliteDriver).readerPool.capacity) - assertEquals(4, (createDriver(DbType.RegularWal) as NativeSqliteDriver).readerPool.capacity) - assertEquals(4, (createDriver(DbType.RegularDelete) as NativeSqliteDriver).readerPool.capacity) - } -} diff --git a/drivers/native-driver-strict/src/nativeTest/kotlin/com/squareup/sqldelight/drivers/native/connectionpool/WalConcurrencyTest.kt b/drivers/native-driver-strict/src/nativeTest/kotlin/com/squareup/sqldelight/drivers/native/connectionpool/WalConcurrencyTest.kt deleted file mode 100644 index fc7d9c8fec0..00000000000 --- a/drivers/native-driver-strict/src/nativeTest/kotlin/com/squareup/sqldelight/drivers/native/connectionpool/WalConcurrencyTest.kt +++ /dev/null @@ -1,202 +0,0 @@ -package com.squareup.sqldelight.drivers.native.connectionpool - -import app.cash.sqldelight.Query -import app.cash.sqldelight.TransacterImpl -import app.cash.sqldelight.db.SqlCursor -import app.cash.sqldelight.driver.native.NativeSqliteDriver -import co.touchlab.testhelp.concurrency.ThreadOperations -import co.touchlab.testhelp.concurrency.sleep -import kotlin.native.concurrent.AtomicInt -import kotlin.native.concurrent.TransferMode -import kotlin.native.concurrent.Worker -import kotlin.native.concurrent.freeze -import kotlin.test.BeforeTest -import kotlin.test.Ignore -import kotlin.test.Test -import kotlin.test.assertEquals -import kotlin.test.assertTrue - -/** - * Testing multiple read and transaction pool connections. These were - * written when it a was a single pool, so this will need some refactor. - * The reader pool is much more likely to be multiple with a single transaction pool - * connection, which removes a lot of the potential concurrency issues, but introduces new things - * we should probably test. - */ -class WalConcurrencyTest : BaseConcurrencyTest() { - @BeforeTest - fun setup() { - initDriver(DbType.RegularWal) - } - - /** - * This is less important now that we have a separate reader pool again, but will revisit. - */ - @Test - fun writeNotBlockRead() { - assertEquals(countRows(), 0) - - val transacter: TransacterImpl = object : TransacterImpl(driver) {} - val worker = Worker.start() - val counter = AtomicInt(0) - val transactionStarted = AtomicInt(0) - - val block = { - transacter.transaction { - insertTestData(TestData(1L, "arst 1")) - transactionStarted.increment() - sleep(1500) - counter.increment() - } - } - - val future = worker.execute(TransferMode.SAFE, { block.freeze() }) { it() } - - // When ready, transaction started but sleeping - waitFor { transactionStarted.value > 0 } - - // These three should run before transaction is done (not blocking) - assertEquals(counter.value, 0) - assertEquals(0L, countRows()) - assertEquals(counter.value, 0) - - future.result - - worker.requestTermination() - } - - /** - * Reader pool stress test - */ - @Test @Ignore - fun manyReads() = runConcurrent { - val transacter: TransacterImpl = object : TransacterImpl(driver) {} - val dataSize = 2_000 - transacter.transaction { - repeat(dataSize) { - insertTestData(TestData(it.toLong(), "Data $it")) - } - } - - val ops = ThreadOperations {} - val totalCount = AtomicInt(0) - val queryRuns = 100 - repeat(queryRuns) { - ops.exe { - totalCount.addAndGet(testDataQuery().executeAsList().size) - } - } - ops.run(6) - assertEquals(totalCount.value, dataSize * queryRuns) - val readerPool = (driver as NativeSqliteDriver).readerPool - // Make sure we actually created all of the connections - assertTrue(readerPool.entryCount() > 1, "Reader pool size ${readerPool.entryCount()}") - } - - private val mapper = { cursor: SqlCursor -> - TestData( - cursor.getLong(0)!!, cursor.getString(1)!! - ) - } - - private fun testDataQuery(): Query { - return object : Query(mapper) { - override fun execute(): SqlCursor { - return driver.executeQuery(0, "SELECT * FROM test", 0) - } - - override fun addListener(listener: Listener) { - driver.addListener(listener, arrayOf("test")) - } - - override fun removeListener(listener: Listener) { - driver.removeListener(listener, arrayOf("test")) - } - } - } - - @Test - fun writeBlocksWrite() { - val transacter: TransacterImpl = object : TransacterImpl(driver) {} - val worker = Worker.start() - val counter = AtomicInt(0) - val transactionStarted = AtomicInt(0) - - val block = { - transacter.transaction { - insertTestData(TestData(1L, "arst 1")) - transactionStarted.increment() - sleep(1500) - counter.increment() - } - } - - val future = worker.execute(TransferMode.SAFE, { block.freeze() }) { it() } - - // Transaction with write started but sleeping - waitFor { transactionStarted.value > 0 } - - assertEquals(counter.value, 0) - insertTestData(TestData(2L, "arst 2")) // This waits on transaction to wrap up - assertEquals(counter.value, 1) // Counter would be zero if write didn't block (see above) - - future.result - - worker.requestTermination() - } - - @Test - fun multipleWritesDontTimeOut() { - val transacter: TransacterImpl = object : TransacterImpl(driver) {} - val worker = Worker.start() - val transactionStarted = AtomicInt(0) - - val block = { - transacter.transaction { - insertTestData(TestData(1L, "arst 1"), driver) - transactionStarted.increment() - sleep(1500) - insertTestData(TestData(5L, "arst 1"), driver) - } - } - - val future = worker.execute(TransferMode.SAFE, { block.freeze() }) { it() } - - // When we get here, first transaction has run a write command, and is sleeping - waitFor { transactionStarted.value > 0 } - transacter.transaction { - insertTestData(TestData(2L, "arst 2"), driver) - } - - future.result - worker.requestTermination() - } - - /** - * Just a bunch of inserts on multiple threads. More of a stress test. - */ - @Test - fun multiWrite() { - val ops = ThreadOperations {} - val times = 10_000 - val transacter: TransacterImpl = object : TransacterImpl(driver) {} - - repeat(times) { index -> - ops.exe { - transacter.transaction { - insertTestData(TestData(index.toLong(), "arst $index")) - - val id2 = index.toLong() + times - insertTestData(TestData(id2, "arst $id2")) - - val id3 = index.toLong() + times + times - insertTestData(TestData(id3, "arst $id3")) - } - } - } - - ops.run(10) - - assertEquals(countRows(), times.toLong() * 3) - } -} diff --git a/drivers/native-driver/src/nativeMain/kotlin/app/cash/sqldelight/driver/native/NativeSqlDatabase.kt b/drivers/native-driver/src/nativeMain/kotlin/app/cash/sqldelight/driver/native/NativeSqlDatabase.kt index dd59d099182..64881549a16 100644 --- a/drivers/native-driver/src/nativeMain/kotlin/app/cash/sqldelight/driver/native/NativeSqlDatabase.kt +++ b/drivers/native-driver/src/nativeMain/kotlin/app/cash/sqldelight/driver/native/NativeSqlDatabase.kt @@ -6,15 +6,15 @@ import app.cash.sqldelight.db.Closeable import app.cash.sqldelight.db.SqlCursor import app.cash.sqldelight.db.SqlDriver import app.cash.sqldelight.db.SqlPreparedStatement +import app.cash.sqldelight.driver.native.util.BasicMap import app.cash.sqldelight.driver.native.util.MutableCache +import app.cash.sqldelight.driver.native.util.basicConcurrentMap import co.touchlab.sqliter.DatabaseConfiguration import co.touchlab.sqliter.DatabaseConnection import co.touchlab.sqliter.DatabaseManager import co.touchlab.sqliter.Statement import co.touchlab.sqliter.createDatabaseManager import co.touchlab.sqliter.withStatement -import co.touchlab.stately.collections.SharedHashMap -import co.touchlab.stately.collections.SharedSet import co.touchlab.stately.concurrency.ThreadLocalRef import co.touchlab.stately.concurrency.value import kotlin.native.concurrent.ensureNeverFrozen @@ -25,61 +25,45 @@ sealed class ConnectionWrapper : SqlDriver { block: ThreadConnection.() -> R ): R - final override fun execute( + private fun accessStatement( + readOnly: Boolean, identifier: Int?, sql: String, - parameters: Int, - binders: (SqlPreparedStatement.() -> Unit)? - ): Long { - return accessConnection(false) { + binders: (SqlPreparedStatement.() -> Unit)?, + block: (Statement) -> R + ): R { + return accessConnection(readOnly) { val statement = useStatement(identifier, sql) - if (binders != null) { - try { + try { + if (binders != null) { SqliterStatement(statement).binders() - } catch (t: Throwable) { - statement.resetStatement() - clearIfNeeded(identifier, statement) - throw t } - } - val result = statement.executeUpdateDelete().toLong() - statement.resetStatement() - clearIfNeeded(identifier, statement) - return@accessConnection result + block(statement) + } finally { + statement.resetStatement() + clearIfNeeded(identifier, statement) + } } } + final override fun execute( + identifier: Int?, + sql: String, + parameters: Int, + binders: (SqlPreparedStatement.() -> Unit)? + ): Long = accessStatement(false, identifier, sql, binders) { statement -> + statement.executeUpdateDelete().toLong() + } + final override fun executeQuery( identifier: Int?, sql: String, mapper: (SqlCursor) -> R, parameters: Int, binders: (SqlPreparedStatement.() -> Unit)? - ): R { - return accessConnection(true) { - val statement = getStatement(identifier, sql) - - if (binders != null) { - try { - SqliterStatement(statement).binders() - } catch (t: Throwable) { - statement.resetStatement() - safePut(identifier, statement) - throw t - } - } - - try { - val cursor = statement.query() - mapper(SqliterSqlCursor(cursor)) - } finally { - statement.resetStatement() - if (closed) - statement.finalizeStatement() - safePut(identifier, statement) - } - } + ): R = accessStatement(true, identifier, sql, binders) { statement -> + mapper(SqliterSqlCursor(statement.query())) } } @@ -148,7 +132,7 @@ class NativeSqliteDriver( // Once a transaction is started and connection borrowed, it will be here, but only for that // thread private val borrowedConnectionThread = ThreadLocalRef>() - private val listeners = SharedHashMap>() + private val listeners = basicConcurrentMap>() init { // Single connection for transactions @@ -176,19 +160,19 @@ class NativeSqliteDriver( override fun addListener(listener: Query.Listener, queryKeys: Array) { queryKeys.forEach { - listeners.getOrPut(it, { SharedSet() }).add(listener) + listeners.getOrPut(it) { basicConcurrentMap() }.put(listener, Unit) } } override fun removeListener(listener: Query.Listener, queryKeys: Array) { queryKeys.forEach { - listeners[it]?.remove(listener) + listeners.get(it)?.remove(listener) } } override fun notifyListeners(queryKeys: Array) { - val listenersToNotify = SharedSet() - queryKeys.forEach { listeners[it]?.let(listenersToNotify::addAll) } + val listenersToNotify = mutableSetOf() + queryKeys.forEach { key -> listeners.get(key)?.let { listenersToNotify.addAll(it.keys) } } listenersToNotify.forEach(Query.Listener::queryResultsChanged) } @@ -319,23 +303,9 @@ internal class ThreadConnection( internal val statementCache = MutableCache() - fun safePut(identifier: Int?, statement: Statement) { - val removed = if (identifier == null) { - statement - } else { - statementCache.put(identifier.toString(), statement) - } - removed?.finalizeStatement() - } - - fun getStatement(identifier: Int?, sql: String): Statement { - val statement = removeCreateStatement(identifier, sql) - return statement - } - fun useStatement(identifier: Int?, sql: String): Statement { return if (identifier != null) { - statementCache.getOrCreate(identifier.toString()) { + statementCache.getOrCreate(identifier) { connection.createStatement(sql) } } else { @@ -344,25 +314,11 @@ internal class ThreadConnection( } fun clearIfNeeded(identifier: Int?, statement: Statement) { - if (identifier == null) { + if (identifier == null || closed) { statement.finalizeStatement() } } - /** - * For cursors. Cursors are actually backed by SQLite statement instances, so they need to be - * removed from the cache when in use. We're giving out a SQLite resource here, so extra care. - */ - private fun removeCreateStatement(identifier: Int?, sql: String): Statement { - if (identifier != null) { - val cached = statementCache.remove(identifier.toString()) - if (cached != null) - return cached - } - - return connection.createStatement(sql) - } - fun newTransaction(): Transacter.Transaction { val enclosing = transaction.value @@ -395,7 +351,9 @@ internal class ThreadConnection( private inner class Transaction( override val enclosingTransaction: Transacter.Transaction? ) : Transacter.Transaction() { - init { ensureNeverFrozen() } + init { + ensureNeverFrozen() + } override fun endTransaction(successful: Boolean) { transaction.value = enclosingTransaction diff --git a/drivers/native-driver/src/nativeMain/kotlin/app/cash/sqldelight/driver/native/Pool.kt b/drivers/native-driver/src/nativeMain/kotlin/app/cash/sqldelight/driver/native/Pool.kt index ee22dcd4c57..3196003ffc6 100644 --- a/drivers/native-driver/src/nativeMain/kotlin/app/cash/sqldelight/driver/native/Pool.kt +++ b/drivers/native-driver/src/nativeMain/kotlin/app/cash/sqldelight/driver/native/Pool.kt @@ -2,6 +2,7 @@ package app.cash.sqldelight.driver.native import app.cash.sqldelight.db.Closeable import app.cash.sqldelight.driver.native.util.PoolLock +import app.cash.sqldelight.driver.native.util.maybeFreeze import co.touchlab.stately.concurrency.AtomicBoolean import kotlin.native.concurrent.AtomicReference @@ -13,7 +14,7 @@ internal class Pool(internal val capacity: Int, private val produ /** * Hold a list of active connections. If it is null, it means the MultiPool has been closed. */ - private val entriesRef = AtomicReference?>(listOf()) + private val entriesRef = AtomicReference?>(listOf().maybeFreeze()) private val poolLock = PoolLock() /** @@ -44,7 +45,7 @@ internal class Pool(internal val capacity: Int, private val produ val done = newEntry.tryToAcquire() check(done) - entriesRef.value = (entries + listOf(newEntry)) + entriesRef.value = (entries + listOf(newEntry)).maybeFreeze() return@withLock newEntry } else { // Capacity is reached — wait for the next available entry. @@ -82,6 +83,8 @@ internal class Pool(internal val capacity: Int, private val produ inner class Entry(val value: T) { val isAvailable = AtomicBoolean(true) + init { maybeFreeze() } + fun tryToAcquire(): Boolean = isAvailable.compareAndSet(expected = true, new = false) fun asBorrowed(poolLock: PoolLock): Borrowed = object : Borrowed { diff --git a/drivers/native-driver/src/nativeMain/kotlin/app/cash/sqldelight/driver/native/util/MemoryModel.kt b/drivers/native-driver/src/nativeMain/kotlin/app/cash/sqldelight/driver/native/util/MemoryModel.kt new file mode 100644 index 00000000000..57cc8183aaf --- /dev/null +++ b/drivers/native-driver/src/nativeMain/kotlin/app/cash/sqldelight/driver/native/util/MemoryModel.kt @@ -0,0 +1,9 @@ +package app.cash.sqldelight.driver.native.util + +import kotlin.native.concurrent.freeze + +internal inline fun T.maybeFreeze(): T = if (Platform.memoryModel == MemoryModel.STRICT) { + this.freeze() +} else { + this +} diff --git a/drivers/native-driver/src/nativeMain/kotlin/app/cash/sqldelight/driver/native/util/MutableCache.kt b/drivers/native-driver/src/nativeMain/kotlin/app/cash/sqldelight/driver/native/util/MutableCache.kt index 85e2a95afb2..e056071a1a4 100644 --- a/drivers/native-driver/src/nativeMain/kotlin/app/cash/sqldelight/driver/native/util/MutableCache.kt +++ b/drivers/native-driver/src/nativeMain/kotlin/app/cash/sqldelight/driver/native/util/MutableCache.kt @@ -1,26 +1,105 @@ package app.cash.sqldelight.driver.native.util +import kotlin.native.concurrent.AtomicReference +import kotlin.native.concurrent.freeze + internal class MutableCache { - private val dictionary = mutableMapOf() + private val dictionary: BasicMap = basicConcurrentMap() + private val lock = PoolLock() - fun put(key: String, value: T?): T? = lock.withLock { - if (value == null) { - dictionary.remove(key) + fun getOrCreate(key: Int, block: () -> T): T = lock.withLock { + dictionary.getOrPut(key, block) + } + + fun cleanUp(block: (T) -> Unit) = lock.withLock { + dictionary.values.forEach(block) + } +} + +internal fun basicConcurrentMap(): BasicMap = + if (Platform.memoryModel == MemoryModel.STRICT) { + BasicMapStrict() + } else { + BasicMapMutable() + } + +internal interface BasicMap { + fun getOrPut(key: K, block: () -> V): V + fun put(key:K, value: V) + fun get(key:K):V? + fun remove(key: K) + val keys: Collection + val values: Collection +} + +/** + * New memory model only. + */ +private class BasicMapMutable : BasicMap { + private val data = mutableMapOf() + + override fun getOrPut(key: K, block: () -> V): V = data.getOrPut(key, block) + + override val values: Collection + get() = data.values + + override fun put(key: K, value: V) { + data[key] = value + } + + override fun get(key: K): V? = data[key] + + override fun remove(key: K) { + data.remove(key) + } + + override val keys: Collection + get() = data.keys +} + +/** + * Slow, but compatible with the strict memory model + */ +private class BasicMapStrict : BasicMap { + private val mapReference = AtomicReference(mutableMapOf().freeze()) + + override fun getOrPut(key: K, block: () -> V): V { + val result = mapReference.value[key] + return if (result == null) { + val created = block() + put(key, created) + created } else { - dictionary.put(key, value) + result } } - fun getOrCreate(key: String, block: () -> T): T = lock.withLock { - dictionary.getOrPut(key, block) - } + override val values: Collection + get() = mapReference.value.values - fun remove(key: String): T? = lock.withLock { - dictionary.remove(key) + override fun put(key: K, value: V) { + mapReference.value = mutableMapOf( + Pair(key, value), + *mapReference.value.map { entry -> + Pair(entry.key, entry.value) + }.toTypedArray() + ).freeze() } - fun cleanUp(block: (T) -> Unit) = lock.withLock { - dictionary.values.forEach(block) + override fun get(key: K): V? = mapReference.value[key] + + override fun remove(key: K) { + val sourceMap = mapReference.value + val resultMap = mutableMapOf() + sourceMap.keys.subtract(listOf(key)).forEach { key -> + val value = sourceMap[key] + if(value != null) + resultMap[key] = value + } + mapReference.value = resultMap.freeze() } + + override val keys: Collection + get() = mapReference.value.keys } diff --git a/drivers/native-driver/src/nativeTest/kotlin/com/squareup/sqldelight/drivers/native/LazyDriverBaseTest.kt b/drivers/native-driver/src/nativeTest/kotlin/com/squareup/sqldelight/drivers/native/LazyDriverBaseTest.kt index 65fc8b7b1a4..bf6263b14d6 100644 --- a/drivers/native-driver/src/nativeTest/kotlin/com/squareup/sqldelight/drivers/native/LazyDriverBaseTest.kt +++ b/drivers/native-driver/src/nativeTest/kotlin/com/squareup/sqldelight/drivers/native/LazyDriverBaseTest.kt @@ -3,6 +3,7 @@ package com.squareup.sqldelight.drivers.native import app.cash.sqldelight.TransacterImpl import app.cash.sqldelight.db.SqlDriver import app.cash.sqldelight.driver.native.NativeSqliteDriver +import app.cash.sqldelight.driver.native.util.maybeFreeze import app.cash.sqldelight.driver.native.wrapConnection import co.touchlab.sqliter.DatabaseConfiguration import co.touchlab.sqliter.DatabaseFileContext.deleteDatabase @@ -23,7 +24,9 @@ abstract class LazyDriverBaseTest { protected val transacter: TransacterImpl get() { - return transacterInternal + val t = transacterInternal + t.maybeFreeze() + return t } @BeforeTest fun setup() { diff --git a/drivers/native-driver/src/nativeTest/kotlin/com/squareup/sqldelight/drivers/native/connectionpool/WalConcurrencyTest.kt b/drivers/native-driver/src/nativeTest/kotlin/com/squareup/sqldelight/drivers/native/connectionpool/WalConcurrencyTest.kt index 8728f3da6a8..0bc6b913057 100644 --- a/drivers/native-driver/src/nativeTest/kotlin/com/squareup/sqldelight/drivers/native/connectionpool/WalConcurrencyTest.kt +++ b/drivers/native-driver/src/nativeTest/kotlin/com/squareup/sqldelight/drivers/native/connectionpool/WalConcurrencyTest.kt @@ -4,6 +4,7 @@ import app.cash.sqldelight.Query import app.cash.sqldelight.TransacterImpl import app.cash.sqldelight.db.SqlCursor import app.cash.sqldelight.driver.native.NativeSqliteDriver +import app.cash.sqldelight.driver.native.util.maybeFreeze import co.touchlab.testhelp.concurrency.ThreadOperations import co.touchlab.testhelp.concurrency.sleep import kotlin.native.concurrent.AtomicInt @@ -49,7 +50,7 @@ class WalConcurrencyTest : BaseConcurrencyTest() { } } - val future = worker.execute(TransferMode.SAFE, { block }) { it() } + val future = worker.execute(TransferMode.SAFE, { block.maybeFreeze() }) { it() } // When ready, transaction started but sleeping waitFor { transactionStarted.value > 0 } @@ -130,7 +131,7 @@ class WalConcurrencyTest : BaseConcurrencyTest() { } } - val future = worker.execute(TransferMode.SAFE, { block }) { it() } + val future = worker.execute(TransferMode.SAFE, { block.maybeFreeze() }) { it() } // Transaction with write started but sleeping waitFor { transactionStarted.value > 0 } @@ -159,7 +160,7 @@ class WalConcurrencyTest : BaseConcurrencyTest() { } } - val future = worker.execute(TransferMode.SAFE, { block }) { it() } + val future = worker.execute(TransferMode.SAFE, { block.maybeFreeze() }) { it() } // When we get here, first transaction has run a write command, and is sleeping waitFor { transactionStarted.value > 0 } diff --git a/runtime/build.gradle b/runtime/build.gradle index 7b905c79f6d..9ab42896c7e 100644 --- a/runtime/build.gradle +++ b/runtime/build.gradle @@ -63,12 +63,14 @@ kotlin { nativeMain { dependsOn(commonMain) dependencies { - implementation deps.stately.core - implementation deps.stately.collections + implementation deps.stately.concurrency } } nativeTest{ dependsOn(commonTest) + dependencies { + implementation deps.stately.collections + } } configure([targets.iosX64, targets.iosArm32, targets.iosArm64, targets.tvosX64, targets.tvosArm64, targets.watchosX86, targets.watchosX64, targets.watchosArm32, targets.watchosArm64, targets.macosX64, targets.mingwX86, targets.mingwX64, targets.linuxX64, targets.macosArm64, targets.iosSimulatorArm64, targets.watchosSimulatorArm64, targets.tvosSimulatorArm64]) { diff --git a/settings.gradle b/settings.gradle index 97e32709fdd..abe7e0c59a1 100644 --- a/settings.gradle +++ b/settings.gradle @@ -26,7 +26,6 @@ include ':dialects:sqlite-3-38' include ':drivers:android-driver' include ':drivers:jdbc-driver' include ':drivers:native-driver' -include ':drivers:native-driver-strict' include ':drivers:sqlite-driver' include ':drivers:sqljs-driver' include ':drivers:driver-test' From e287b33056710a8232edc790eaa2650eccf3e546 Mon Sep 17 00:00:00 2001 From: Kevin Galligan Date: Tue, 3 May 2022 21:21:47 -0400 Subject: [PATCH 6/8] Fix format for spotless --- .../app/cash/sqldelight/driver/native/util/MutableCache.kt | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/drivers/native-driver/src/nativeMain/kotlin/app/cash/sqldelight/driver/native/util/MutableCache.kt b/drivers/native-driver/src/nativeMain/kotlin/app/cash/sqldelight/driver/native/util/MutableCache.kt index e056071a1a4..c768e22d598 100644 --- a/drivers/native-driver/src/nativeMain/kotlin/app/cash/sqldelight/driver/native/util/MutableCache.kt +++ b/drivers/native-driver/src/nativeMain/kotlin/app/cash/sqldelight/driver/native/util/MutableCache.kt @@ -26,8 +26,8 @@ internal fun basicConcurrentMap(): BasicMap = internal interface BasicMap { fun getOrPut(key: K, block: () -> V): V - fun put(key:K, value: V) - fun get(key:K):V? + fun put(key: K, value: V) + fun get(key: K): V? fun remove(key: K) val keys: Collection val values: Collection @@ -94,7 +94,7 @@ private class BasicMapStrict : BasicMap { val resultMap = mutableMapOf() sourceMap.keys.subtract(listOf(key)).forEach { key -> val value = sourceMap[key] - if(value != null) + if (value != null) resultMap[key] = value } mapReference.value = resultMap.freeze() From b92e332024fb6cdac5ec0df08f1c5280f514fafc Mon Sep 17 00:00:00 2001 From: Kevin Galligan Date: Wed, 4 May 2022 14:21:42 -0400 Subject: [PATCH 7/8] Update map and lock --- .../sqldelight/driver/native/util/PoolLock.kt | 17 ++- .../sqldelight/driver/native/util/PoolLock.kt | 17 ++- .../driver/native/NativeSqlDatabase.kt | 13 +- .../driver/native/util/BasicMutableMap.kt | 119 ++++++++++++++++++ .../driver/native/util/MutableCache.kt | 105 ---------------- .../sqldelight/driver/native/util/PoolLock.kt | 2 +- .../native/util/BasicMutableMapTest.kt | 38 ++++++ gradle.properties | 1 + 8 files changed, 195 insertions(+), 117 deletions(-) create mode 100644 drivers/native-driver/src/nativeMain/kotlin/app/cash/sqldelight/driver/native/util/BasicMutableMap.kt delete mode 100644 drivers/native-driver/src/nativeMain/kotlin/app/cash/sqldelight/driver/native/util/MutableCache.kt create mode 100644 drivers/native-driver/src/nativeTest/kotlin/com/squareup/sqldelight/drivers/native/util/BasicMutableMapTest.kt diff --git a/drivers/native-driver/src/mingwMain/kotlin/app/cash/sqldelight/driver/native/util/PoolLock.kt b/drivers/native-driver/src/mingwMain/kotlin/app/cash/sqldelight/driver/native/util/PoolLock.kt index 331f846dadb..a114504e0b7 100644 --- a/drivers/native-driver/src/mingwMain/kotlin/app/cash/sqldelight/driver/native/util/PoolLock.kt +++ b/drivers/native-driver/src/mingwMain/kotlin/app/cash/sqldelight/driver/native/util/PoolLock.kt @@ -5,6 +5,7 @@ import kotlinx.cinterop.alloc import kotlinx.cinterop.free import kotlinx.cinterop.nativeHeap import kotlinx.cinterop.ptr +import platform.posix.PTHREAD_MUTEX_RECURSIVE import platform.posix.pthread_cond_destroy import platform.posix.pthread_cond_init import platform.posix.pthread_cond_signal @@ -15,11 +16,21 @@ import platform.posix.pthread_mutex_init import platform.posix.pthread_mutex_lock import platform.posix.pthread_mutex_tVar import platform.posix.pthread_mutex_unlock +import platform.posix.pthread_mutexattr_destroy +import platform.posix.pthread_mutexattr_init +import platform.posix.pthread_mutexattr_settype +import platform.posix.pthread_mutexattr_tVar -internal actual class PoolLock actual constructor() { +internal actual class PoolLock actual constructor(reentrant: Boolean) { private val isActive = AtomicBoolean(true) + private val attr = nativeHeap.alloc() + .apply { + pthread_mutexattr_init(ptr) + if (reentrant) + pthread_mutexattr_settype(ptr, PTHREAD_MUTEX_RECURSIVE.toInt()) + } private val mutex = nativeHeap.alloc() - .apply { pthread_mutex_init(ptr, null) } + .apply { pthread_mutex_init(ptr, attr.ptr) } private val cond = nativeHeap.alloc() .apply { pthread_cond_init(ptr, null) } @@ -48,8 +59,10 @@ internal actual class PoolLock actual constructor() { if (isActive.compareAndSet(expected = true, new = false)) { pthread_cond_destroy(cond.ptr) pthread_mutex_destroy(mutex.ptr) + pthread_mutexattr_destroy(attr.ptr) nativeHeap.free(cond) nativeHeap.free(mutex) + nativeHeap.free(attr) return true } diff --git a/drivers/native-driver/src/nativeLinuxLikeMain/kotlin/app/cash/sqldelight/driver/native/util/PoolLock.kt b/drivers/native-driver/src/nativeLinuxLikeMain/kotlin/app/cash/sqldelight/driver/native/util/PoolLock.kt index 3896457a75e..7b31219242a 100644 --- a/drivers/native-driver/src/nativeLinuxLikeMain/kotlin/app/cash/sqldelight/driver/native/util/PoolLock.kt +++ b/drivers/native-driver/src/nativeLinuxLikeMain/kotlin/app/cash/sqldelight/driver/native/util/PoolLock.kt @@ -5,6 +5,7 @@ import kotlinx.cinterop.alloc import kotlinx.cinterop.free import kotlinx.cinterop.nativeHeap import kotlinx.cinterop.ptr +import platform.posix.PTHREAD_MUTEX_RECURSIVE import platform.posix.pthread_cond_destroy import platform.posix.pthread_cond_init import platform.posix.pthread_cond_signal @@ -15,11 +16,21 @@ import platform.posix.pthread_mutex_init import platform.posix.pthread_mutex_lock import platform.posix.pthread_mutex_t import platform.posix.pthread_mutex_unlock +import platform.posix.pthread_mutexattr_destroy +import platform.posix.pthread_mutexattr_init +import platform.posix.pthread_mutexattr_settype +import platform.posix.pthread_mutexattr_t -internal actual class PoolLock actual constructor() { +internal actual class PoolLock actual constructor(reentrant: Boolean) { private val isActive = AtomicBoolean(true) + private val attr = nativeHeap.alloc() + .apply { + pthread_mutexattr_init(ptr) + if (reentrant) + pthread_mutexattr_settype(ptr, PTHREAD_MUTEX_RECURSIVE.toInt()) + } private val mutex = nativeHeap.alloc() - .apply { pthread_mutex_init(ptr, null) } + .apply { pthread_mutex_init(ptr, attr.ptr) } private val cond = nativeHeap.alloc() .apply { pthread_cond_init(ptr, null) } @@ -48,8 +59,10 @@ internal actual class PoolLock actual constructor() { if (isActive.compareAndSet(expected = true, new = false)) { pthread_cond_destroy(cond.ptr) pthread_mutex_destroy(mutex.ptr) + pthread_mutexattr_destroy(attr.ptr) nativeHeap.free(cond) nativeHeap.free(mutex) + nativeHeap.free(attr) return true } diff --git a/drivers/native-driver/src/nativeMain/kotlin/app/cash/sqldelight/driver/native/NativeSqlDatabase.kt b/drivers/native-driver/src/nativeMain/kotlin/app/cash/sqldelight/driver/native/NativeSqlDatabase.kt index 64881549a16..5197ffe2e4e 100644 --- a/drivers/native-driver/src/nativeMain/kotlin/app/cash/sqldelight/driver/native/NativeSqlDatabase.kt +++ b/drivers/native-driver/src/nativeMain/kotlin/app/cash/sqldelight/driver/native/NativeSqlDatabase.kt @@ -6,9 +6,8 @@ import app.cash.sqldelight.db.Closeable import app.cash.sqldelight.db.SqlCursor import app.cash.sqldelight.db.SqlDriver import app.cash.sqldelight.db.SqlPreparedStatement -import app.cash.sqldelight.driver.native.util.BasicMap -import app.cash.sqldelight.driver.native.util.MutableCache -import app.cash.sqldelight.driver.native.util.basicConcurrentMap +import app.cash.sqldelight.driver.native.util.BasicMutableMap +import app.cash.sqldelight.driver.native.util.basicMutableMap import co.touchlab.sqliter.DatabaseConfiguration import co.touchlab.sqliter.DatabaseConnection import co.touchlab.sqliter.DatabaseManager @@ -132,7 +131,7 @@ class NativeSqliteDriver( // Once a transaction is started and connection borrowed, it will be here, but only for that // thread private val borrowedConnectionThread = ThreadLocalRef>() - private val listeners = basicConcurrentMap>() + private val listeners = basicMutableMap>() init { // Single connection for transactions @@ -160,7 +159,7 @@ class NativeSqliteDriver( override fun addListener(listener: Query.Listener, queryKeys: Array) { queryKeys.forEach { - listeners.getOrPut(it) { basicConcurrentMap() }.put(listener, Unit) + listeners.getOrPut(it) { basicMutableMap() }.put(listener, Unit) } } @@ -301,11 +300,11 @@ internal class ThreadConnection( internal val closed: Boolean get() = connection.closed - internal val statementCache = MutableCache() + internal val statementCache = basicMutableMap() fun useStatement(identifier: Int?, sql: String): Statement { return if (identifier != null) { - statementCache.getOrCreate(identifier) { + statementCache.getOrPut(identifier) { connection.createStatement(sql) } } else { diff --git a/drivers/native-driver/src/nativeMain/kotlin/app/cash/sqldelight/driver/native/util/BasicMutableMap.kt b/drivers/native-driver/src/nativeMain/kotlin/app/cash/sqldelight/driver/native/util/BasicMutableMap.kt new file mode 100644 index 00000000000..ea5f1cadd3b --- /dev/null +++ b/drivers/native-driver/src/nativeMain/kotlin/app/cash/sqldelight/driver/native/util/BasicMutableMap.kt @@ -0,0 +1,119 @@ +package app.cash.sqldelight.driver.native.util + +import kotlin.native.concurrent.AtomicReference +import kotlin.native.concurrent.freeze + +/** + * Basic map functionality, implemented differently by memory model. Both are safe to use with + * multiple threads. + */ +internal interface BasicMutableMap { + fun getOrPut(key: K, block: () -> V): V + fun put(key: K, value: V) + fun get(key: K): V? + fun remove(key: K) + val keys: Collection + val values: Collection + fun cleanUp(block: (V) -> Unit) +} + +internal fun basicMutableMap(): BasicMutableMap = + if (Platform.memoryModel == MemoryModel.STRICT) { + BasicMutableMapStrict() + } else { + BasicMutableMapShared() + } + +/** + * New memory model only. May be able to remove the lock or use a more optimized version of a concurrent + * map, but relative to where it's being used, there isn't likely to be much of a performance hit. + */ +private class BasicMutableMapShared : BasicMutableMap { + private val lock = PoolLock(reentrant = true) + private val data = mutableMapOf() + + override fun getOrPut(key: K, block: () -> V): V = lock.withLock { data.getOrPut(key, block) } + + override val values: Collection + get() = lock.withLock { data.values } + + override fun put(key: K, value: V) { + lock.withLock { data[key] = value } + } + + override fun get(key: K): V? = lock.withLock { data[key] } + + override fun remove(key: K) { + lock.withLock { data.remove(key) } + } + + override val keys: Collection + get() = lock.withLock { data.keys } + + override fun cleanUp(block: (V) -> Unit) { + lock.withLock { + data.values.forEach(block) + } + } +} + +/** + * Slow, but compatible with the strict memory model. + */ +private class BasicMutableMapStrict : BasicMutableMap { + private val lock = PoolLock(reentrant = true) + private val mapReference = AtomicReference(mutableMapOf().freeze()) + + override fun getOrPut(key: K, block: () -> V): V = lock.withLock { + val result = mapReference.value[key] + if (result == null) { + val created = block() + _put(key, created) + created + } else { + result + } + } + + override val values: Collection + get() = lock.withLock { mapReference.value.values } + + override fun put(key: K, value: V) { + lock.withLock { + _put(key, value) + } + } + + private fun _put(key: K, value: V) { + mapReference.value = mutableMapOf( + Pair(key, value), + *mapReference.value.map { entry -> + Pair(entry.key, entry.value) + }.toTypedArray() + ).freeze() + } + + override fun get(key: K): V? = lock.withLock { mapReference.value[key] } + + override fun remove(key: K) { + lock.withLock { + val sourceMap = mapReference.value + val resultMap = mutableMapOf() + sourceMap.keys.subtract(listOf(key)).forEach { key -> + val value = sourceMap[key] + if (value != null) + resultMap[key] = value + } + mapReference.value = resultMap.freeze() + } + } + + override val keys: Collection + get() = lock.withLock { mapReference.value.keys } + + override fun cleanUp(block: (V) -> Unit) { + lock.withLock { + mapReference.value.values.forEach(block) + } + } +} diff --git a/drivers/native-driver/src/nativeMain/kotlin/app/cash/sqldelight/driver/native/util/MutableCache.kt b/drivers/native-driver/src/nativeMain/kotlin/app/cash/sqldelight/driver/native/util/MutableCache.kt deleted file mode 100644 index c768e22d598..00000000000 --- a/drivers/native-driver/src/nativeMain/kotlin/app/cash/sqldelight/driver/native/util/MutableCache.kt +++ /dev/null @@ -1,105 +0,0 @@ -package app.cash.sqldelight.driver.native.util - -import kotlin.native.concurrent.AtomicReference -import kotlin.native.concurrent.freeze - -internal class MutableCache { - private val dictionary: BasicMap = basicConcurrentMap() - - private val lock = PoolLock() - - fun getOrCreate(key: Int, block: () -> T): T = lock.withLock { - dictionary.getOrPut(key, block) - } - - fun cleanUp(block: (T) -> Unit) = lock.withLock { - dictionary.values.forEach(block) - } -} - -internal fun basicConcurrentMap(): BasicMap = - if (Platform.memoryModel == MemoryModel.STRICT) { - BasicMapStrict() - } else { - BasicMapMutable() - } - -internal interface BasicMap { - fun getOrPut(key: K, block: () -> V): V - fun put(key: K, value: V) - fun get(key: K): V? - fun remove(key: K) - val keys: Collection - val values: Collection -} - -/** - * New memory model only. - */ -private class BasicMapMutable : BasicMap { - private val data = mutableMapOf() - - override fun getOrPut(key: K, block: () -> V): V = data.getOrPut(key, block) - - override val values: Collection - get() = data.values - - override fun put(key: K, value: V) { - data[key] = value - } - - override fun get(key: K): V? = data[key] - - override fun remove(key: K) { - data.remove(key) - } - - override val keys: Collection - get() = data.keys -} - -/** - * Slow, but compatible with the strict memory model - */ -private class BasicMapStrict : BasicMap { - private val mapReference = AtomicReference(mutableMapOf().freeze()) - - override fun getOrPut(key: K, block: () -> V): V { - val result = mapReference.value[key] - return if (result == null) { - val created = block() - put(key, created) - created - } else { - result - } - } - - override val values: Collection - get() = mapReference.value.values - - override fun put(key: K, value: V) { - mapReference.value = mutableMapOf( - Pair(key, value), - *mapReference.value.map { entry -> - Pair(entry.key, entry.value) - }.toTypedArray() - ).freeze() - } - - override fun get(key: K): V? = mapReference.value[key] - - override fun remove(key: K) { - val sourceMap = mapReference.value - val resultMap = mutableMapOf() - sourceMap.keys.subtract(listOf(key)).forEach { key -> - val value = sourceMap[key] - if (value != null) - resultMap[key] = value - } - mapReference.value = resultMap.freeze() - } - - override val keys: Collection - get() = mapReference.value.keys -} diff --git a/drivers/native-driver/src/nativeMain/kotlin/app/cash/sqldelight/driver/native/util/PoolLock.kt b/drivers/native-driver/src/nativeMain/kotlin/app/cash/sqldelight/driver/native/util/PoolLock.kt index ac852d0f82e..44ca1a7637d 100644 --- a/drivers/native-driver/src/nativeMain/kotlin/app/cash/sqldelight/driver/native/util/PoolLock.kt +++ b/drivers/native-driver/src/nativeMain/kotlin/app/cash/sqldelight/driver/native/util/PoolLock.kt @@ -1,7 +1,7 @@ package app.cash.sqldelight.driver.native.util @Suppress("NO_ACTUAL_FOR_EXPECT") -internal expect class PoolLock() { +internal expect class PoolLock(reentrant: Boolean = false) { fun withLock( action: CriticalSection.() -> R ): R diff --git a/drivers/native-driver/src/nativeTest/kotlin/com/squareup/sqldelight/drivers/native/util/BasicMutableMapTest.kt b/drivers/native-driver/src/nativeTest/kotlin/com/squareup/sqldelight/drivers/native/util/BasicMutableMapTest.kt new file mode 100644 index 00000000000..0cc3e1d8467 --- /dev/null +++ b/drivers/native-driver/src/nativeTest/kotlin/com/squareup/sqldelight/drivers/native/util/BasicMutableMapTest.kt @@ -0,0 +1,38 @@ +package com.squareup.sqldelight.drivers.native.util + +import app.cash.sqldelight.driver.native.util.basicMutableMap +import kotlin.test.Test +import kotlin.test.assertEquals +import kotlin.test.assertNull + +class MutableCacheTest { + @Test + fun basicOperations() { + val map = basicMutableMap() + map.put("abc", SomeData("123")) + map.put("8675", SomeData("309")) + + assertEquals(map.get("abc"), SomeData("123")) + + map.remove("abc") + + assertNull(map.get("abc")) + + assertEquals(map.keys.toList(), listOf("8675")) + assertEquals(map.values.toList(), listOf(SomeData("309"))) + } + + @Test + fun cleanUp() { + val map = basicMutableMap() + repeat(20) { index -> + map.put("i$index", SomeData("v$index")) + } + + var count = 0 + map.cleanUp { count++ } + assertEquals(count, 20) + } +} + +data class SomeData(val s: String) diff --git a/gradle.properties b/gradle.properties index 5d1fa3e57d2..39d5259ca8d 100644 --- a/gradle.properties +++ b/gradle.properties @@ -27,5 +27,6 @@ kotlin.mpp.enableGranularSourceSetsMetadata=true kotlin.native.enableDependencyPropagation=false kotlin.mpp.enableCInteropCommonization=true kotlin.mpp.commonizerLogLevel=info +kotlin.mpp.enableCompatibilityMetadataVariant=true kotlin.native.binary.memoryModel=experimental \ No newline at end of file From 4d14c744240463e8ab0edd54f859880c675ec27e Mon Sep 17 00:00:00 2001 From: Kevin Galligan Date: Wed, 4 May 2022 14:26:26 -0400 Subject: [PATCH 8/8] Removed HMPP properties --- gradle.properties | 6 ------ 1 file changed, 6 deletions(-) diff --git a/gradle.properties b/gradle.properties index 39d5259ca8d..a7f46c8d098 100644 --- a/gradle.properties +++ b/gradle.properties @@ -23,10 +23,4 @@ kotlin.js.compiler=both kotlin.mpp.stability.nowarn=true kotlin.native.ignoreDisabledTargets=true -kotlin.mpp.enableGranularSourceSetsMetadata=true -kotlin.native.enableDependencyPropagation=false -kotlin.mpp.enableCInteropCommonization=true -kotlin.mpp.commonizerLogLevel=info -kotlin.mpp.enableCompatibilityMetadataVariant=true - kotlin.native.binary.memoryModel=experimental \ No newline at end of file