From 571e7c403f8c190d4bf7a8b04a9ea91ce27fec80 Mon Sep 17 00:00:00 2001 From: WhiredPlanck Date: Thu, 2 May 2024 17:20:17 +0800 Subject: [PATCH] feat: smarter and faster assets syncing Finally I remember to utilize the checksums pre-calculated on build, comparing the sha256 among the files to determine how to handle them. Ref: https://github.com/fcitx5-android/fcitx5-android/blob/59558c5b624359455911082b10750f4dcbd10fe8/app/src/main/java/org/fcitx/fcitx5/android/core/data --- .../java/com/osfans/trime/data/AppPrefs.kt | 12 -- .../osfans/trime/data/base/DataChecksums.kt | 13 ++ .../com/osfans/trime/data/base/DataDiff.kt | 55 +++++++++ .../com/osfans/trime/data/base/DataManager.kt | 116 ++++++++++-------- .../java/com/osfans/trime/util/FileUtils.kt | 27 ++++ .../com/osfans/trime/util/ResourceUtils.kt | 21 ++++ 6 files changed, 182 insertions(+), 62 deletions(-) create mode 100644 app/src/main/java/com/osfans/trime/data/base/DataChecksums.kt create mode 100644 app/src/main/java/com/osfans/trime/data/base/DataDiff.kt create mode 100644 app/src/main/java/com/osfans/trime/util/FileUtils.kt create mode 100644 app/src/main/java/com/osfans/trime/util/ResourceUtils.kt diff --git a/app/src/main/java/com/osfans/trime/data/AppPrefs.kt b/app/src/main/java/com/osfans/trime/data/AppPrefs.kt index accadcdcf8..16dcdf9ec2 100644 --- a/app/src/main/java/com/osfans/trime/data/AppPrefs.kt +++ b/app/src/main/java/com/osfans/trime/data/AppPrefs.kt @@ -121,20 +121,12 @@ class AppPrefs( class Internal(private val prefs: AppPrefs) { companion object { - const val LAST_VERSION_NAME = "general__last_version_name" const val PID = "general__pid" - const val LAST_BUILD_GIT_HASH = "general__last_build_git_hash" } - var lastVersionName: String - get() = prefs.getPref(LAST_VERSION_NAME, "") - set(v) = prefs.setPref(LAST_VERSION_NAME, v) var pid: Int get() = prefs.getPref(PID, 0) set(v) = prefs.setPref(PID, v) - var lastBuildGitHash: String - get() = prefs.getPref(LAST_BUILD_GIT_HASH, "") - set(v) = prefs.setPref(LAST_BUILD_GIT_HASH, v) } /** @@ -420,7 +412,6 @@ class AppPrefs( const val UI_MODE = "other__ui_mode" const val SHOW_APP_ICON = "other__show_app_icon" const val SHOW_STATUS_BAR_ICON = "other__show_status_bar_icon" - const val DESTROY_ON_QUIT = "other__destroy_on_quit" } var uiMode: String @@ -432,8 +423,5 @@ class AppPrefs( var showStatusBarIcon: Boolean = false get() = prefs.getPref(SHOW_STATUS_BAR_ICON, false) private set - var destroyOnQuit: Boolean = false - get() = prefs.getPref(DESTROY_ON_QUIT, false) - private set } } diff --git a/app/src/main/java/com/osfans/trime/data/base/DataChecksums.kt b/app/src/main/java/com/osfans/trime/data/base/DataChecksums.kt new file mode 100644 index 0000000000..c73926d8d9 --- /dev/null +++ b/app/src/main/java/com/osfans/trime/data/base/DataChecksums.kt @@ -0,0 +1,13 @@ +// SPDX-FileCopyrightText: 2015 - 2024 Rime community +// +// SPDX-License-Identifier: GPL-3.0-or-later + +package com.osfans.trime.data.base + +import kotlinx.serialization.Serializable + +@Serializable +data class DataChecksums( + val sha256: String, + val files: Map, +) diff --git a/app/src/main/java/com/osfans/trime/data/base/DataDiff.kt b/app/src/main/java/com/osfans/trime/data/base/DataDiff.kt new file mode 100644 index 0000000000..c9b35caa13 --- /dev/null +++ b/app/src/main/java/com/osfans/trime/data/base/DataDiff.kt @@ -0,0 +1,55 @@ +// SPDX-FileCopyrightText: 2015 - 2024 Rime community +// +// SPDX-License-Identifier: GPL-3.0-or-later + +package com.osfans.trime.data.base + +sealed interface DataDiff { + val path: String + + val ordinal: Int + + data class CreateFile(override val path: String) : DataDiff { + override val ordinal: Int + get() = 3 + } + + data class UpdateFile(override val path: String) : DataDiff { + override val ordinal: Int + get() = 2 + } + + data class DeleteDir(override val path: String) : DataDiff { + override val ordinal: Int + get() = 1 + } + + data class DeleteFile(override val path: String) : DataDiff { + override val ordinal: Int + get() = 0 + } + + companion object { + fun diff( + old: DataChecksums, + new: DataChecksums, + ): List { + if (old.sha256 == new.sha256) return emptyList() + return new.files.mapNotNull { (path, sha256) -> + when { + path !in old.files && sha256.isNotBlank() -> CreateFile(path) + old.files[path] != sha256 -> + if (sha256.isNotBlank()) UpdateFile(path) else null + else -> null + } + }.toMutableList().apply { + addAll( + old.files.filterKeys { it !in new.files } + .map { (path, sha256) -> + if (sha256.isNotBlank()) DeleteFile(path) else DeleteDir(path) + }, + ) + } + } + } +} diff --git a/app/src/main/java/com/osfans/trime/data/base/DataManager.kt b/app/src/main/java/com/osfans/trime/data/base/DataManager.kt index 8e30e6b10d..4c12a118ee 100644 --- a/app/src/main/java/com/osfans/trime/data/base/DataManager.kt +++ b/app/src/main/java/com/osfans/trime/data/base/DataManager.kt @@ -4,16 +4,50 @@ package com.osfans.trime.data.base +import android.content.res.AssetManager +import android.os.Build import com.blankj.utilcode.util.PathUtils -import com.blankj.utilcode.util.ResourceUtils import com.osfans.trime.data.AppPrefs -import com.osfans.trime.util.Const +import com.osfans.trime.util.FileUtils +import com.osfans.trime.util.ResourceUtils import com.osfans.trime.util.WeakHashSet +import com.osfans.trime.util.appContext +import kotlinx.serialization.json.Json import timber.log.Timber import java.io.File +import java.util.concurrent.locks.ReentrantLock +import kotlin.concurrent.withLock object DataManager { private const val DEFAULT_CUSTOM_FILE_NAME = "default.custom.yaml" + + private const val DATA_CHECKSUMS_NAME = "checksums.json" + + private val lock = ReentrantLock() + + private val json by lazy { Json } + + private fun deserializeDataChecksums(raw: String): DataChecksums { + return json.decodeFromString(raw) + } + + // If Android version supports direct boot, we put the hierarchy in device encrypted storage + // instead of credential encrypted storage so that data can be accessed before user unlock + private val dataDir: File = + if (Build.VERSION.SDK_INT >= Build.VERSION_CODES.N) { + Timber.d("Using device protected storage") + appContext.createDeviceProtectedStorageContext().dataDir + } else { + File(appContext.applicationInfo.dataDir) + } + + private fun AssetManager.dataChecksums(): DataChecksums { + return open(DATA_CHECKSUMS_NAME) + .bufferedReader() + .use { it.readText() } + .let { deserializeDataChecksums(it) } + } + private val prefs get() = AppPrefs.defaultInstance() val defaultDataDirectory = File(PathUtils.getExternalStoragePath(), "rime") @@ -44,14 +78,6 @@ object DataManager { val userDataDir get() = File(prefs.profile.userDataDir) - sealed class Diff { - object New : Diff() - - object Update : Diff() - - object Keep : Diff() - } - /** * Return the absolute path of the compiled config file * based on given resource id. @@ -72,48 +98,38 @@ object DataManager { return defaultPath.absolutePath } - private fun diff( - old: String, - new: String, - ): Diff { - return when { - old.isBlank() -> Diff.New - !new.contentEquals(old) -> Diff.Update - else -> Diff.Keep - } - } - - @JvmStatic - fun sync() { - val newHash = Const.buildCommitHash - val oldHash = prefs.internal.lastBuildGitHash - - diff(oldHash, newHash).run { - Timber.d("Diff: $this") - when (this) { - is Diff.New -> - ResourceUtils.copyFileFromAssets( - "rime", - sharedDataDir.absolutePath, - ) - is Diff.Update -> - ResourceUtils.copyFileFromAssets( - "rime", - sharedDataDir.absolutePath, - ) - is Diff.Keep -> {} + fun sync() = + lock.withLock { + val oldChecksumsFile = File(dataDir, DATA_CHECKSUMS_NAME) + val oldChecksums = + oldChecksumsFile + .runCatching { deserializeDataChecksums(bufferedReader().use { it.readText() }) } + .getOrElse { DataChecksums("", emptyMap()) } + + val newChecksums = appContext.assets.dataChecksums() + + DataDiff.diff(oldChecksums, newChecksums).sortedByDescending { it.ordinal }.forEach { + Timber.d("Diff: $it") + when (it) { + is DataDiff.CreateFile, + is DataDiff.UpdateFile, + -> ResourceUtils.copyFile(it.path, sharedDataDir) + is DataDiff.DeleteDir, + is DataDiff.DeleteFile, + -> FileUtils.delete(sharedDataDir.resolve(it.path)).getOrThrow() + } } - } - // FIXME：缺失 default.custom.yaml 会导致方案列表为空 - with(File(sharedDataDir, DEFAULT_CUSTOM_FILE_NAME)) { - val customDefault = this - if (!customDefault.exists()) { - Timber.d("Creating empty default.custom.yaml ...") - customDefault.createNewFile() + ResourceUtils.copyFile(DATA_CHECKSUMS_NAME, dataDir) + + // FIXME：缺失 default.custom.yaml 会导致方案列表为空 + File(sharedDataDir, DEFAULT_CUSTOM_FILE_NAME).let { + if (!it.exists()) { + Timber.d("Creating empty default.custom.yaml") + it.bufferedWriter().use { w -> w.write("") } + } } - } - Timber.i("Synced!") - } + Timber.d("Synced!") + } } diff --git a/app/src/main/java/com/osfans/trime/util/FileUtils.kt b/app/src/main/java/com/osfans/trime/util/FileUtils.kt new file mode 100644 index 0000000000..32b87072be --- /dev/null +++ b/app/src/main/java/com/osfans/trime/util/FileUtils.kt @@ -0,0 +1,27 @@ +// SPDX-FileCopyrightText: 2015 - 2024 Rime community +// +// SPDX-License-Identifier: GPL-3.0-or-later + +package com.osfans.trime.util + +import java.io.File +import java.io.IOException + +object FileUtils { + fun delete(file: File) = + runCatching { + if (!file.exists()) return Result.success(Unit) + val res = + if (file.isDirectory) { + file.walkBottomUp() + .fold(true) { acc, file -> + if (file.exists()) file.delete() else acc + } + } else { + file.delete() + } + if (!res) { + throw IOException("Cannot delete ${file.path}") + } + } +} diff --git a/app/src/main/java/com/osfans/trime/util/ResourceUtils.kt b/app/src/main/java/com/osfans/trime/util/ResourceUtils.kt new file mode 100644 index 0000000000..5181bd9b02 --- /dev/null +++ b/app/src/main/java/com/osfans/trime/util/ResourceUtils.kt @@ -0,0 +1,21 @@ +// SPDX-FileCopyrightText: 2015 - 2024 Rime community +// +// SPDX-License-Identifier: GPL-3.0-or-later + +package com.osfans.trime.util + +import java.io.File + +object ResourceUtils { + fun copyFile( + filename: String, + dest: File, + ) = runCatching { + appContext.assets.open(filename).use { i -> + File(dest, filename) + .also { it.parentFile?.mkdirs() } + .outputStream() + .use { o -> i.copyTo(o) } + } + } +}