From efaefe551ebb73da1fe809f30dbc13ab1ac7827a Mon Sep 17 00:00:00 2001 From: Mike Lin Date: Sun, 26 Feb 2023 21:15:53 -1000 Subject: [PATCH] update zstd_vfs with VFS logging --- CMakeLists.txt | 2 +- docs/guide_db.md | 2 +- src/genomicsqlite.cc | 8 +++++--- 3 files changed, 7 insertions(+), 5 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index a5cd2ee..1bc5852 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -18,7 +18,7 @@ endif() FetchContent_Declare( sqlite_zstd_vfs GIT_REPOSITORY https://github.com/mlin/sqlite_zstd_vfs.git - GIT_TAG 25e962e + GIT_TAG 2e9c7db ) FetchContent_MakeAvailable(sqlite_zstd_vfs) FetchContent_MakeAvailable(sqlitecpp) diff --git a/docs/guide_db.md b/docs/guide_db.md index e8beed7..7b8abea 100644 --- a/docs/guide_db.md +++ b/docs/guide_db.md @@ -205,7 +205,7 @@ The **GenomicSQLite Open** routine and the `genomicsqlite` shell also accept htt Under the hood, the extension uses [libcurl](https://curl.se/libcurl/) to send web requests for necessary portions of the database file as queries proceed, with adaptive batching & prefetching to balance the number and size of these requests. This works well for point lookups and queries that scan largely-contiguous slices of tables and indexes (and a modest number thereof). It's less suitable for big multi-way joins and other aggressively random access patterns; in such cases, it'd be better to download the database file upfront to open locally. * Reading large databases over the web, budget an additional ~600MiB of memory for HTTP prefetch buffers. -* The HTTP driver writes log messages to standard error when requests fail or had to be retried, which can be disabled by setting configuration web_log = 0 or environment SQLITE_WEB_LOG=0; or increased up to 5 to log every request and other details. +* The HTTP driver writes log messages to standard error when requests fail or had to be retried, which can be disabled by setting configuration vfs_log = 0 or environment SQLITE_VFS_LOG=0; or increased up to 5 for extensive debug logging. * To disable TLS certificate and hostname verification, set web_insecure = true in the GenomicSQLite configuration, or SQLITE_WEB_INSECURE=1 in the environment. * The above-described `genomicsqlite DB_FILENAME --compact` optimizes a database for web access by making the request pattern more contiguous. diff --git a/src/genomicsqlite.cc b/src/genomicsqlite.cc index 350b985..45e1248 100644 --- a/src/genomicsqlite.cc +++ b/src/genomicsqlite.cc @@ -68,7 +68,7 @@ std::string GenomicSQLiteDefaultConfigJSON() { "zstd_level": 6, "inner_page_KiB": 16, "outer_page_KiB": 32, - "web_log": 2, + "vfs_log": -1, "web_insecure": false, "web_dbi_url": "", "web_nodbi": false, @@ -173,9 +173,11 @@ string GenomicSQLiteURI(const string &dbfile, const string &config_json = "") { bool web = dbfile.substr(0, 5) == "http:" || dbfile.substr(0, 6) == "https:"; ostringstream uri; uri << "file:" << (web ? "/__web__" : SQLiteNested::urlencode(dbfile, true)) << "?vfs=zstd"; + if (cfg.GetInt("$.vfs_log") >= 0) { + uri << "&vfs_log=" << cfg.GetInt("$.vfs_log"); + } if (web) { - uri << "&mode=ro&immutable=1&web_url=" << SQLiteNested::urlencode(dbfile) - << "&web_log=" << cfg.GetInt("$.web_log"); + uri << "&mode=ro&immutable=1&web_url=" << SQLiteNested::urlencode(dbfile); if (cfg.GetBool("$.web_insecure")) { uri << "&web_insecure=1"; }