Skip to content

Commit

Permalink
improve robustness of extension init routines
Browse files Browse the repository at this point in the history
  • Loading branch information
@mlin
mlin committed May 31, 2021
1 parent 5789a9b commit 3efb82e
Show file tree
Hide file tree
Showing 6 changed files with 52 additions and 40 deletions.
2 changes: 1 addition & 1 deletion CMakeLists.txt
View file
Original file line number Diff line number Diff line change
Expand Up @@ -18,7 +18,7 @@ endif()
FetchContent_Declare(
sqlite_zstd_vfs
GIT_REPOSITORY https://github.com/mlin/sqlite_zstd_vfs.git
GIT_TAG eadb758
GIT_TAG df70c88
)
FetchContent_MakeAvailable(sqlite_zstd_vfs)
FetchContent_MakeAvailable(sqlitecpp)
Expand Down
15 changes: 9 additions & 6 deletions include/genomicsqlite.h
View file
Original file line number Diff line number Diff line change
Expand Up @@ -38,10 +38,11 @@ char *genomicsqlite_default_config_json();
*/
int genomicsqlite_init(int (*)(const char *, sqlite3 **, int, const char *),
int (*)(sqlite3 *, int),
int (*)(sqlite3 *, const char *, const char *, char **), char **pzErrMsg);
int (*)(sqlite3 *, const char *, const char *, char **), int (*)(sqlite3 *),
void (*)(void *), char **pzErrMsg);
#define GENOMICSQLITE_C_INIT(pzErrMsg) \
genomicsqlite_init(sqlite3_open_v2, sqlite3_enable_load_extension, sqlite3_load_extension, \
pzErrMsg);
sqlite3_close, sqlite3_free, pzErrMsg);

/*
* Wrap sqlite3_open() and initialize the "connection" for use with GenomicSQLite. config_json if
Expand Down Expand Up @@ -160,11 +161,13 @@ std::string GenomicSQLiteDefaultConfigJSON();
* ...
* }
*/
void GenomicSQLiteInit(int (*open_v2)(const char *, sqlite3 **, int, const char *),
int (*enable_load_extension)(sqlite3 *, int),
int (*load_extension)(sqlite3 *, const char *, const char *, char **));
void GenomicSQLiteInit(int (*)(const char *, sqlite3 **, int, const char *),
int (*)(sqlite3 *, int),
int (*)(sqlite3 *, const char *, const char *, char **), int (*)(sqlite3 *),
void (*)(void *));
#define GENOMICSQLITE_CXX_INIT() \
GenomicSQLiteInit(sqlite3_open_v2, sqlite3_enable_load_extension, sqlite3_load_extension);
GenomicSQLiteInit(sqlite3_open_v2, sqlite3_enable_load_extension, sqlite3_load_extension, \
sqlite3_close, sqlite3_free);

int GenomicSQLiteOpen(const std::string &dbfile, sqlite3 **ppDb, std::string &errmsg_out,
int flags = 0, const std::string &config_json = "{}") noexcept;
Expand Down
6 changes: 4 additions & 2 deletions loaders/sam_into_sqlite.cc
View file
Original file line number Diff line number Diff line change
Expand Up @@ -275,9 +275,11 @@ int main(int argc, char *argv[]) {

try {
// open output database
if (sqlite3_config(SQLITE_CONFIG_MEMSTATUS, 0) != SQLITE_OK ||
sqlite3_config(SQLITE_CONFIG_LOOKASIDE, 4096, 256) != SQLITE_OK) {
throw runtime_error("sqlite3_config() failed");
}
GENOMICSQLITE_CXX_INIT();
sqlite3_config(SQLITE_CONFIG_MEMSTATUS, 0);
sqlite3_config(SQLITE_CONFIG_LOOKASIDE, 2048, 128);
string config_json = R"({"unsafe_load": true, "zstd_level":)" + to_string(level) +
R"(,"inner_page_KiB":)" + to_string(inner_page_KiB) +
R"(,"outer_page_KiB":)" + to_string(outer_page_KiB) + "}";
Expand Down
6 changes: 4 additions & 2 deletions loaders/vcf_into_sqlite.cc
View file
Original file line number Diff line number Diff line change
Expand Up @@ -667,9 +667,11 @@ int main(int argc, char *argv[]) {
}

// open output database
if (sqlite3_config(SQLITE_CONFIG_MEMSTATUS, 0) != SQLITE_OK ||
sqlite3_config(SQLITE_CONFIG_LOOKASIDE, 4096, 256) != SQLITE_OK) {
throw runtime_error("sqlite3_config() failed");
}
GENOMICSQLITE_CXX_INIT();
sqlite3_config(SQLITE_CONFIG_MEMSTATUS, 0);
sqlite3_config(SQLITE_CONFIG_LOOKASIDE, 2048, 128);
string config_json = R"({"unsafe_load": true, "zstd_level":)" + to_string(level) +
R"(,"inner_page_KiB":)" + to_string(inner_page_KiB) +
R"(,"outer_page_KiB":)" + to_string(outer_page_KiB) + "}";
Expand Down
6 changes: 4 additions & 2 deletions loaders/vcf_lines_into_sqlite.cc
View file
Original file line number Diff line number Diff line change
Expand Up @@ -136,9 +136,11 @@ int main(int argc, char *argv[]) {
}

// open output database
if (sqlite3_config(SQLITE_CONFIG_MEMSTATUS, 0) != SQLITE_OK ||
sqlite3_config(SQLITE_CONFIG_LOOKASIDE, 4096, 256) != SQLITE_OK) {
throw runtime_error("sqlite3_config() failed");
}
GENOMICSQLITE_CXX_INIT();
sqlite3_config(SQLITE_CONFIG_MEMSTATUS, 0);
sqlite3_config(SQLITE_CONFIG_LOOKASIDE, 2048, 128);
auto db = GenomicSQLiteOpen(
outfilename, SQLITE_OPEN_CREATE | SQLITE_OPEN_READWRITE | SQLITE_OPEN_NOMUTEX,
R"( {"unsafe_load": true, "zstd_level": )" + to_string(level) + "}");
Expand Down
57 changes: 30 additions & 27 deletions src/genomicsqlite.cc
View file
Original file line number Diff line number Diff line change
Expand Up @@ -205,9 +205,10 @@ string GenomicSQLiteURI(const string &dbfile, const string &config_json = "") {
uri << "&immutable=1";
}
if (cfg.GetBool("$.unsafe_load")) {
uri << "&nolock=1&outer_unsafe";
uri << "&nolock=1&outer_unsafe=1";
}
}
_DBG << uri.str() << endl;
return uri.str();
}

Expand Down Expand Up @@ -298,6 +299,7 @@ string GenomicSQLiteTuningSQL(const string &config_json, const string &schema =
for (const auto &p : pragmas) {
out << "; PRAGMA " << p.first << "=" << p.second;
}
_DBG << out.str() << endl;
return out.str();
}

Expand All @@ -315,14 +317,13 @@ static void sqlfn_genomicsqlite_tuning_sql(sqlite3_context *ctx, int argc, sqlit

void GenomicSQLiteInit(int (*open_v2)(const char *, sqlite3 **, int, const char *),
int (*enable_load_extension)(sqlite3 *, int),
int (*load_extension)(sqlite3 *, const char *, const char *, char **)) {
static bool ext_loaded = false;
if (!ext_loaded) {

int (*load_extension)(sqlite3 *, const char *, const char *, char **),
int (*close_)(sqlite3 *), void (*free_)(void *)) {
if (!sqlite3_api) {
sqlite3 *memdb = nullptr;
int rc = open_v2(":memory:", &memdb, SQLITE_OPEN_CREATE | SQLITE_OPEN_READWRITE, nullptr);
if (rc != SQLITE_OK) {
sqlite3_close(memdb);
close_(memdb);
throw runtime_error("GenomicSQLiteInit() unable to open temporary SQLite connection");
}
enable_load_extension(memdb, 1);
Expand All @@ -331,30 +332,28 @@ void GenomicSQLiteInit(int (*open_v2)(const char *, sqlite3 **, int, const char
if (rc != SQLITE_OK) {
string err = "GenomicSQLiteInit() unable to load the extension" +
(zErrMsg ? ": " + string(zErrMsg) : "");
sqlite3_free(zErrMsg);
sqlite3_close(memdb);
free_(zErrMsg);
close_(memdb);
throw runtime_error(err);
}
sqlite3_free(zErrMsg);
rc = sqlite3_close(memdb);
free_(zErrMsg);
rc = close_(memdb);
if (rc != SQLITE_OK) {
throw runtime_error("GenomicSQLiteInit() unable to close temporary SQLite connection");
}
}
if (open_v2 != sqlite3_api->open_v2) {
} else if (open_v2 != sqlite3_api->open_v2) {
throw std::runtime_error(
"GenomicSQLiteInit() saw inconsistent libsqlite3/libgenomicsqlite library linkage in this process");
"Two distinct copies of SQLite in this process attempted to load Genomics Extension");
}
ext_loaded = true;
}

extern "C" int genomicsqlite_init(int (*open_v2)(const char *, sqlite3 **, int, const char *),
int (*enable_load_extension)(sqlite3 *, int),
int (*load_extension)(sqlite3 *, const char *, const char *,
char **),
char **pzErrMsg) {
extern "C" int
genomicsqlite_init(int (*open_v2)(const char *, sqlite3 **, int, const char *),
int (*enable_load_extension)(sqlite3 *, int),
int (*load_extension)(sqlite3 *, const char *, const char *, char **),
int (*close_)(sqlite3 *), void (*free_)(void *), char **pzErrMsg) {
try {
GenomicSQLiteInit(open_v2, enable_load_extension, load_extension);
GenomicSQLiteInit(open_v2, enable_load_extension, load_extension, close_, free_);
return SQLITE_OK;
} catch (std::bad_alloc &) {
return SQLITE_NOMEM;
Expand Down Expand Up @@ -458,7 +457,7 @@ static void sqlfn_genomicsqlite_attach_sql(sqlite3_context *ctx, int argc, sqlit
}

string GenomicSQLiteVacuumIntoSQL(const string &destfile, const string &config_json) {
string desturi = GenomicSQLiteURI(destfile, config_json) + "&outer_unsafe=true";
string desturi = GenomicSQLiteURI(destfile, config_json) + "&outer_unsafe=1";

ConfigParser cfg(config_json);
ostringstream ans;
Expand Down Expand Up @@ -1786,6 +1785,14 @@ static int register_genomicsqlite_functions(sqlite3 *db, const char **pzErrMsg,
*/
extern "C" int sqlite3_genomicsqlite_init(sqlite3 *db, char **pzErrMsg,
const sqlite3_api_routines *pApi) {
if (sqlite3_api && sqlite3_api != pApi) {
if (pzErrMsg) {
*pzErrMsg = sqlite3_mprintf(
"Two distinct copies of SQLite in this process attempted to load Genomics Extension %s",
GIT_REVISION);
}
return SQLITE_ERROR;
}
SQLITE_EXTENSION_INIT2(pApi);

// The newest SQLite feature currently required is "Generated Columns"
Expand All @@ -1800,10 +1807,7 @@ extern "C" int sqlite3_genomicsqlite_init(sqlite3 *db, char **pzErrMsg,
return SQLITE_ERROR;
}

/* disabled b/c JDBC bindings "legitimately" entail two different dynamically-linked sqlite libs!
// Check that SQLiteCpp is using the same SQLite library that's loading us. This may not be the
// case when different versions of SQLite are linked into the running process, one static and
// one dynamic.
// Check that SQLiteCpp is using the same SQLite library that's loading us.
string static_version(pApi->libversion()), dynamic_version("UNKNOWN");
{
SQLite::Database tmpdb(":memory:", SQLITE_OPEN_CREATE | SQLITE_OPEN_READWRITE);
Expand All @@ -1815,12 +1819,11 @@ extern "C" int sqlite3_genomicsqlite_init(sqlite3 *db, char **pzErrMsg,
if (static_version != dynamic_version) {
if (pzErrMsg) {
*pzErrMsg = sqlite3_mprintf(
"Two distinct versions of SQLite (%s & %s) detected by Genomics Extension %s. Eliminate static linking of SQLite from the main executable.",
"Two distinct versions of SQLite (%s & %s) in this process detected by Genomics Extension %s",
static_version.c_str(), dynamic_version.c_str(), GIT_REVISION);
}
return SQLITE_ERROR;
}
*/

int rc = (new WebVFS::VFS())->Register("web");
if (rc != SQLITE_OK) {
Expand Down

0 comments on commit 3efb82e

Please sign in to comment.