From 1747ec731e7031e2b77c1d1e1cf732c6686ddad3 Mon Sep 17 00:00:00 2001 From: Weihao Ding <158090588+weihao-statsig@users.noreply.github.com> Date: Tue, 19 Mar 2024 16:28:31 -0700 Subject: [PATCH] Add Local Data File Option To Read Configuration Locally (#242) Added the ability to read configuration from a file when in local mode. --- src/main/kotlin/com/statsig/sdk/SpecStore.kt | 16 ++- .../kotlin/com/statsig/sdk/StatsigOptions.kt | 2 + .../kotlin/com/statsig/sdk/StatsigServer.kt | 13 ++ .../statsig/sdk/{ => datastore}/IDataStore.kt | 2 +- .../sdk/datastore/LocalFileDataStore.kt | 49 ++++++++ .../java/com/statsig/sdk/DataStoreTest.kt | 7 +- .../com/statsig/sdk/LocalFileDataStoreTest.kt | 116 ++++++++++++++++++ 7 files changed, 200 insertions(+), 5 deletions(-) rename src/main/kotlin/com/statsig/sdk/{ => datastore}/IDataStore.kt (81%) create mode 100644 src/main/kotlin/com/statsig/sdk/datastore/LocalFileDataStore.kt create mode 100644 src/test/java/com/statsig/sdk/LocalFileDataStoreTest.kt diff --git a/src/main/kotlin/com/statsig/sdk/SpecStore.kt b/src/main/kotlin/com/statsig/sdk/SpecStore.kt index 8d3ffe1..b5bb161 100644 --- a/src/main/kotlin/com/statsig/sdk/SpecStore.kt +++ b/src/main/kotlin/com/statsig/sdk/SpecStore.kt @@ -4,6 +4,7 @@ import com.google.gson.Gson import com.google.gson.GsonBuilder import com.google.gson.ToNumberPolicy import com.google.gson.reflect.TypeToken +import com.statsig.sdk.datastore.LocalFileDataStore import kotlinx.coroutines.CoroutineScope import kotlinx.coroutines.Job import kotlinx.coroutines.delay @@ -360,7 +361,11 @@ internal class SpecStore constructor( var downloadedConfigs: APIDownloadedConfigs? = null if (options.dataStore != null) { - downloadedConfigs = this.loadConfigSpecsFromStorageAdapter() + if (options.dataStore is LocalFileDataStore) { + downloadedConfigs = this.downloadConfigSpecsToLocal() + } else { + downloadedConfigs = this.loadConfigSpecsFromStorageAdapter() + } initReason = if (downloadedConfigs == null) EvaluationReason.UNINITIALIZED else EvaluationReason.DATA_ADAPTER } else if (options.bootstrapValues != null) { @@ -390,6 +395,15 @@ internal class SpecStore constructor( } } + private suspend fun downloadConfigSpecsToLocal(): APIDownloadedConfigs? { + val response = this.downloadConfigSpecs() + + val specs: String = gson.toJson(response) + val localDataStore = options.dataStore as LocalFileDataStore + localDataStore.set(localDataStore.filePath, specs) + return response + } + private suspend fun downloadConfigSpecsFromNetwork(): APIDownloadedConfigs? { return this.downloadConfigSpecs() } diff --git a/src/main/kotlin/com/statsig/sdk/StatsigOptions.kt b/src/main/kotlin/com/statsig/sdk/StatsigOptions.kt index 712136a..d55e227 100644 --- a/src/main/kotlin/com/statsig/sdk/StatsigOptions.kt +++ b/src/main/kotlin/com/statsig/sdk/StatsigOptions.kt @@ -1,5 +1,7 @@ package com.statsig.sdk +import com.statsig.sdk.datastore.IDataStore + private const val TIER_KEY: String = "tier" private const val DEFAULT_INIT_TIME_OUT_MS: Long = 3000L private const val CONFIG_SYNC_INTERVAL_MS: Long = 10 * 1000 diff --git a/src/main/kotlin/com/statsig/sdk/StatsigServer.kt b/src/main/kotlin/com/statsig/sdk/StatsigServer.kt index d175a7d..dcf0751 100644 --- a/src/main/kotlin/com/statsig/sdk/StatsigServer.kt +++ b/src/main/kotlin/com/statsig/sdk/StatsigServer.kt @@ -2,6 +2,7 @@ package com.statsig.sdk import com.google.gson.GsonBuilder import com.google.gson.ToNumberPolicy +import com.statsig.sdk.datastore.LocalFileDataStore import kotlinx.coroutines.* import kotlinx.coroutines.future.future import kotlinx.coroutines.sync.Mutex @@ -222,6 +223,8 @@ sealed class StatsigServer { @JvmSynthetic internal abstract fun getCustomLogger(): LoggerInterface + abstract fun localDataStoreSetUp(serverSecret: String) + companion object { @JvmStatic @@ -278,6 +281,7 @@ private class StatsigServerImpl() : ) } setupAndStartDiagnostics() + localDataStoreSetUp(serverSecret) configEvaluator = Evaluator(network, options, statsigScope, errorBoundary, diagnostics, statsigMetadata, serverSecret) configEvaluator.initialize() @@ -291,6 +295,15 @@ private class StatsigServerImpl() : ) } + override fun localDataStoreSetUp(serverSecret: String) { + if (options.dataStore == null || options.dataStore !is LocalFileDataStore) { + return + } + + val localFileDataStore = options.dataStore as LocalFileDataStore + localFileDataStore.filePath = Hashing.djb2(serverSecret) + } + override fun isInitialized(): Boolean { return initialized } diff --git a/src/main/kotlin/com/statsig/sdk/IDataStore.kt b/src/main/kotlin/com/statsig/sdk/datastore/IDataStore.kt similarity index 81% rename from src/main/kotlin/com/statsig/sdk/IDataStore.kt rename to src/main/kotlin/com/statsig/sdk/datastore/IDataStore.kt index 4c7469a..8a8eb7c 100644 --- a/src/main/kotlin/com/statsig/sdk/IDataStore.kt +++ b/src/main/kotlin/com/statsig/sdk/datastore/IDataStore.kt @@ -1,4 +1,4 @@ -package com.statsig.sdk +package com.statsig.sdk.datastore abstract class IDataStore { abstract fun get(key: String): String? diff --git a/src/main/kotlin/com/statsig/sdk/datastore/LocalFileDataStore.kt b/src/main/kotlin/com/statsig/sdk/datastore/LocalFileDataStore.kt new file mode 100644 index 0000000..5464ed1 --- /dev/null +++ b/src/main/kotlin/com/statsig/sdk/datastore/LocalFileDataStore.kt @@ -0,0 +1,49 @@ +package com.statsig.sdk.datastore + +import java.io.File +import java.util.* + +/** + * LocalFileDataStore class implements IDataStore interface to provide data storage operations using local files. + */ +class LocalFileDataStore() : IDataStore() { + + var workingDirectory: String = "/tmp/statsig/" + var filePath: String = "" + + init { + workingDirectory = resolvePath(workingDirectory) + if (!File(workingDirectory).exists()) { + File(workingDirectory).mkdirs() // Ensure the working directory exists + } + } + + override fun get(key: String): String? { + val path = "$workingDirectory${Base64.getEncoder().encodeToString(filePath.toByteArray())}" + return try { + File(path).readText() + } catch (e: Exception) { + null + } + } + + override fun set(key: String, value: String) { + val path = "$workingDirectory${Base64.getEncoder().encodeToString(filePath.toByteArray())}" + File(path).writeText(value) + } + + override fun shutdown() { + // No explicit shutdown operations needed for this class + } + + internal fun resolvePath(path: String): String { + var resolvedPath = path + if (!resolvedPath.endsWith("/")) { + resolvedPath += "/" + } + if (!resolvedPath.startsWith("/")) { + resolvedPath = File("").absolutePath + "/" + resolvedPath + } + return resolvedPath + } +} diff --git a/src/test/java/com/statsig/sdk/DataStoreTest.kt b/src/test/java/com/statsig/sdk/DataStoreTest.kt index b109a8e..909c100 100644 --- a/src/test/java/com/statsig/sdk/DataStoreTest.kt +++ b/src/test/java/com/statsig/sdk/DataStoreTest.kt @@ -2,6 +2,7 @@ package com.statsig.sdk import com.google.gson.GsonBuilder import com.google.gson.ToNumberPolicy +import com.statsig.sdk.datastore.IDataStore import kotlinx.coroutines.CompletableDeferred import okhttp3.mockwebserver.Dispatcher import okhttp3.mockwebserver.MockResponse @@ -12,10 +13,10 @@ import org.junit.Assert import org.junit.Before import org.junit.Test -private class TestDataAdapter : IDataStore() { - var data = +class TestDataAdapter : IDataStore() { + private var data = DataStoreTest::class.java.getResource("/data_adapter.json")?.readText() ?: "" - var dataStore = mutableMapOf( + private var dataStore = mutableMapOf( STORAGE_ADAPTER_KEY to data, ) diff --git a/src/test/java/com/statsig/sdk/LocalFileDataStoreTest.kt b/src/test/java/com/statsig/sdk/LocalFileDataStoreTest.kt new file mode 100644 index 0000000..8e935c7 --- /dev/null +++ b/src/test/java/com/statsig/sdk/LocalFileDataStoreTest.kt @@ -0,0 +1,116 @@ +package com.statsig.sdk + +import com.statsig.sdk.datastore.LocalFileDataStore +import okhttp3.mockwebserver.Dispatcher +import okhttp3.mockwebserver.MockResponse +import okhttp3.mockwebserver.MockWebServer +import okhttp3.mockwebserver.RecordedRequest +import org.junit.After +import org.junit.Assert +import org.junit.Before +import org.junit.Test +import java.io.File + +class LocalFileDataStoreTest { + private lateinit var localDataStore: LocalFileDataStore + private lateinit var downloadConfigSpecsResponse: String + private lateinit var statsigServer: StatsigServer + private lateinit var options: StatsigOptions + private lateinit var mockServer: MockWebServer + private var didCallDownloadConfig = false + private val user = StatsigUser("test-user") + + @Before + fun setUp() { + downloadConfigSpecsResponse = StatsigE2ETest::class.java.getResource("/download_config_specs.json")?.readText() ?: "" + + mockServer = MockWebServer() + mockServer.start(8899) + mockServer.apply { + dispatcher = object : Dispatcher() { + @Throws(InterruptedException::class) + override fun dispatch(request: RecordedRequest): MockResponse { + if (request.path == null) { + return MockResponse().setResponseCode(404) + } + if ("/v1/download_config_specs" in request.path!!) { + didCallDownloadConfig = true + return MockResponse().setResponseCode(200).setBody(downloadConfigSpecsResponse) + } + return MockResponse().setResponseCode(404) + } + } + } + + localDataStore = LocalFileDataStore() + } + + @After + fun tearDown() { + File(localDataStore.workingDirectory).deleteRecursively() // clean up the folder when finished tests + mockServer.shutdown() + } + + @Test + fun testLocalDataStoreIsLoaded() { + options = StatsigOptions( + api = mockServer.url("/v1").toString(), + dataStore = localDataStore, + disableDiagnostics = true, + ) + + statsigServer = StatsigServer.create() + statsigServer.initializeAsync("test-key", options).get() + + val gateRes1 = statsigServer.checkGateSync(user, "always_on_gate") + Assert.assertTrue(gateRes1) + + user.email = "test@statsig.com" + val gateRes2 = statsigServer.checkGateSync(user, "on_for_statsig_email") + Assert.assertTrue(gateRes2) + statsigServer.shutdown() + } + + @Test + fun testNetworkNotCallWhenBootstrapIsPresent() { + options = StatsigOptions( + api = mockServer.url("/v1").toString(), + bootstrapValues = downloadConfigSpecsResponse, + ) + statsigServer = StatsigServer.create() + statsigServer.initializeAsync("secret-local", options).get() + + Assert.assertFalse(didCallDownloadConfig) + } + + @Test + fun testCallsNetworkWhenAdapterIsEmpty() { + val options = StatsigOptions( + api = mockServer.url("/v1").toString(), + ) + statsigServer = StatsigServer.create() + statsigServer.initializeAsync("secret-local", options).get() + + Assert.assertTrue(didCallDownloadConfig) + statsigServer.shutdown() + } + + @Test + fun testNetworkNotCalledWhenAdapterEnable() { + // if dataStore(cached one) is enabled + // should not trigger network request + val options = StatsigOptions( + api = mockServer.url("/v1").toString(), + dataStore = TestDataAdapter() + ) + statsigServer = StatsigServer.create() + statsigServer.initializeAsync("secret-local", options).get() + + Assert.assertFalse(didCallDownloadConfig) + + // Test dataStore still works + val dataStoreGateRes = statsigServer.checkGateSync(user, "gate_from_adapter_always_on") + Assert.assertTrue(dataStoreGateRes) + statsigServer.shutdown() + } +}