GPKG: add a NOLOCK=YES option to open a file without any lock (for re…

…ad-only access) (helps fixing qgis/QGIS#23991, but requires QGIS changes as well)
OSGeo · Feb 1, 2022 · af70e66 · af70e66
1 parent 34f2bc6
commit af70e66
Show file tree

Hide file tree

Showing 6 changed files with 220 additions and 76 deletions.
diff --git a/autotest/ogr/ogr_gpkg.py b/autotest/ogr/ogr_gpkg.py
@@ -4154,6 +4154,64 @@ def test_ogr_gpkg_wal():
     gdal.Unlink(filename + '-wal')
     gdal.Unlink(filename + '-shm')
 
+###############################################################################
+# Test NOLOCK open option
+
+
+def test_ogr_gpkg_nolock():
+
+    def get_nolock(ds):
+        sql_lyr = ds.ExecuteSQL('SELECT nolock', dialect='DEBUG')
+        f = sql_lyr.GetNextFeature()
+        res = True if f[0] == 1 else False
+        ds.ReleaseResultSet(sql_lyr)
+        return res
+
+    # needs to be a real file
+    filename = 'tmp/test_ogr_gpkg_nolock.gpkg'
+
+    ds = gdaltest.gpkg_dr.CreateDataSource(filename)
+    lyr = ds.CreateLayer('foo')
+    f = ogr.Feature(lyr.GetLayerDefn())
+    lyr.CreateFeature(f)
+    f = None
+    ds = None
+
+    ds = gdal.OpenEx(filename, gdal.OF_VECTOR, open_options=['NOLOCK=YES'])
+    assert ds
+    assert get_nolock(ds)
+
+    lyr = ds.GetLayer(0)
+    f = lyr.GetNextFeature()
+    ds2 = ogr.Open(filename, update = 1)
+    lyr2 = ds2.GetLayer(0)
+    f = ogr.Feature(lyr2.GetLayerDefn())
+    # Without lockless mode on ds, this would timeout and fail
+    assert lyr2.CreateFeature(f) == ogr.OGRERR_NONE
+    f = None
+    ds2 = None
+    ds = None
+
+    # Lockless mode should NOT be honored by GDAL in upate mode
+    ds = gdal.OpenEx(filename, gdal.OF_VECTOR | gdal.OF_UPDATE, open_options=['NOLOCK=YES'])
+    assert ds
+    assert not get_nolock(ds)
+    ds = None
+
+    # Now turn on WAL
+    ds = ogr.Open(filename, update = 1)
+    ds.ExecuteSQL('PRAGMA journal_mode = WAL')
+    ds = None
+
+    # Lockless mode should NOT be honored by GDAL on a WAL enabled file
+    ds = gdal.OpenEx(filename, gdal.OF_VECTOR, open_options=['NOLOCK=YES'])
+    assert ds
+    assert not get_nolock(ds)
+    ds = None
+
+    gdal.Unlink(filename)
+    gdal.Unlink(filename + '-wal')
+    gdal.Unlink(filename + '-shm')
 
 ###############################################################################
 # Run test_ogrsf

diff --git a/doc/source/drivers/vector/gpkg.rst b/doc/source/drivers/vector/gpkg.rst
@@ -167,6 +167,13 @@ The following open options are available:
         The attached database must be a GeoPackage one too, so
         that its geometry blobs are properly recognized (so typically not a Spatialite one)
 
+-  **NOLOCK**\= YES/NO (GDAL >= 3.4.2). Defaults is NO.
+   Whether the database should be used without doing any file locking. Setting
+   it to YES will only be honoured when opening in read-only mode and if the
+   journal mode is not WAL.
+   This corresponds to the nolock=1 query parameter described at
+   https://www.sqlite.org/uri.html
+
 Note: open options are typically specified with "-oo name=value" syntax
 in most OGR utilities, or with the GDALOpenEx() API call.
 

diff --git a/ogr/ogrsf_frmts/gpkg/ogrgeopackagedatasource.cpp b/ogr/ogrsf_frmts/gpkg/ogrgeopackagedatasource.cpp
@@ -1226,6 +1226,12 @@ int GDALGeoPackageDataset::Open( GDALOpenInfo* poOpenInfo )
     eAccess = poOpenInfo->eAccess;
     m_pszFilename = CPLStrdup( osFilename );
 
+    if( poOpenInfo->papszOpenOptions )
+    {
+        CSLDestroy(papszOpenOptions);
+        papszOpenOptions = CSLDuplicate(poOpenInfo->papszOpenOptions);
+    }
+
 #ifdef ENABLE_SQL_GPKG_FORMAT
     if( poOpenInfo->pabyHeader &&
         STARTS_WITH((const char*)poOpenInfo->pabyHeader, "-- SQL GPKG") &&
@@ -5855,6 +5861,16 @@ OGRLayer * GDALGeoPackageDataset::ExecuteSQL( const char *pszSQLCommand,
         }
     }
 
+/* -------------------------------------------------------------------- */
+/*      DEBUG "SELECT nolock" command.                                  */
+/* -------------------------------------------------------------------- */
+    if( pszDialect != nullptr && EQUAL(pszDialect, "DEBUG") &&
+        EQUAL(osSQLCommand, "SELECT nolock") )
+    {
+        return new OGRSQLiteSingleFeatureLayer
+                                (osSQLCommand, m_bNoLock ? 1 : 0 );
+    }
+
 /* -------------------------------------------------------------------- */
 /*      Special case DELLAYER: command.                                 */
 /* -------------------------------------------------------------------- */

diff --git a/ogr/ogrsf_frmts/gpkg/ogrgeopackagedriver.cpp b/ogr/ogrsf_frmts/gpkg/ogrgeopackagedriver.cpp
@@ -450,6 +450,7 @@ void RegisterOGRGeoPackage()
 "  <Option name='WHERE' type='string' scope='raster' description='SQL WHERE clause to be appended to tile requests'/>"
 COMPRESSION_OPTIONS
 "  <Option name='PRELUDE_STATEMENTS' type='string' scope='raster,vector' description='SQL statement(s) to send on the SQLite connection before any other ones'/>"
+"  <Option name='NOLOCK' type='boolean' description='Whether the database should be opened in nolock mode'/>"
 "</OpenOptionList>");
 
     poDriver->SetMetadataItem( GDAL_DS_LAYER_CREATIONOPTIONLIST,

diff --git a/ogr/ogrsf_frmts/sqlite/ogrsqlitebase.h b/ogr/ogrsf_frmts/sqlite/ogrsqlitebase.h
@@ -120,6 +120,8 @@ class OGRSQLiteBaseDataSource CPL_NON_FINAL: public GDALPamDataset
 {
   protected:
     char               *m_pszFilename = nullptr;
+    std::string         m_osFilenameForSQLiteOpen{}; // generally m_pszFilename, but can be also file:{m_pszFilename}?nolock=1
+    bool                m_bNoLock = false;
     std::string         m_osFinalFilename{}; // use when generating a network hosted file with CPL_VSIL_USE_TEMP_FILE_FOR_RANDOM_WRITE=YES
     bool                m_bCallUndeclareFileNotToOpen = false;
 

diff --git a/ogr/ogrsf_frmts/sqlite/ogrsqlitedatasource.cpp b/ogr/ogrsf_frmts/sqlite/ogrsqlitedatasource.cpp
@@ -830,117 +830,170 @@ int OGRSQLiteBaseDataSource::OpenOrCreateDB(int flagsIn, bool bRegisterOGR2SQLit
     if( bRegisterOGR2SQLiteExtensions )
         OGR2SQLITE_Register();
 
+    const bool bUseOGRVFS =
+        CPLTestBool(CPLGetConfigOption("SQLITE_USE_OGR_VFS", "NO"));
+
+#ifdef SQLITE_OPEN_URI
+    if ( m_osFilenameForSQLiteOpen.empty() &&
+          (flagsIn & SQLITE_OPEN_READWRITE) == 0 &&
+          !(bUseOGRVFS || STARTS_WITH(m_pszFilename, "/vsi")) &&
+          !STARTS_WITH(m_pszFilename, "file:") &&
+          CPLTestBool(CSLFetchNameValueDef(papszOpenOptions, "NOLOCK", "NO")) )
+    {
+        m_osFilenameForSQLiteOpen = "file:";
+        m_osFilenameForSQLiteOpen += m_pszFilename;
+        m_osFilenameForSQLiteOpen += "?nolock=1";
+    }
+#endif
+    if( m_osFilenameForSQLiteOpen.empty() )
+    {
+        m_osFilenameForSQLiteOpen = m_pszFilename;
+    }
+
     // No mutex since OGR objects are not supposed to be used concurrently
     // from multiple threads.
     int flags = flagsIn | SQLITE_OPEN_NOMUTEX;
 #ifdef SQLITE_OPEN_URI
     // This code enables support for named memory databases in SQLite.
     // SQLITE_USE_URI is checked only to enable backward compatibility, in
     // case we accidentally hijacked some other format.
-    if( STARTS_WITH(m_pszFilename, "file:") &&
+    if( STARTS_WITH(m_osFilenameForSQLiteOpen.c_str(), "file:") &&
         CPLTestBool(CPLGetConfigOption("SQLITE_USE_URI", "YES")) )
     {
         flags |= SQLITE_OPEN_URI;
     }
 #endif
 
-    int rc = SQLITE_OK;
-
-    const bool bUseOGRVFS =
-        CPLTestBool(CPLGetConfigOption("SQLITE_USE_OGR_VFS", "NO"));
-    if (bUseOGRVFS || STARTS_WITH(m_pszFilename, "/vsi"))
-    {
-        pMyVFS = OGRSQLiteCreateVFS(OGRSQLiteBaseDataSourceNotifyFileOpened, this);
-        sqlite3_vfs_register(pMyVFS, 0);
-        rc = sqlite3_open_v2( m_pszFilename, &hDB, flags, pMyVFS->zName );
-    }
-    else
-    {
-        rc = sqlite3_open_v2( m_pszFilename, &hDB, flags, nullptr );
-    }
+    bool bPageSizeFound = false;
 
-    if( rc != SQLITE_OK )
-    {
-        CPLError( CE_Failure, CPLE_OpenFailed,
-                  "sqlite3_open(%s) failed: %s",
-                  m_pszFilename, sqlite3_errmsg( hDB ) );
-        return FALSE;
-    }
+    const char* pszSqlitePragma =
+                            CPLGetConfigOption("OGR_SQLITE_PRAGMA", nullptr);
+    CPLString osJournalMode =
+                        CPLGetConfigOption("OGR_SQLITE_JOURNAL", "");
 
-#ifdef SQLITE_DBCONFIG_DEFENSIVE
-    // SQLite builds on recent MacOS enable defensive mode by default, which
-    // causes issues in the VDV driver (when updating a deleted database),
-    // or in the GPKG driver (when modifying a CREATE TABLE DDL with writable_schema=ON)
-    // So disable it.
-    int bDefensiveOldValue = 0;
-    if( sqlite3_db_config(hDB, SQLITE_DBCONFIG_DEFENSIVE, -1, &bDefensiveOldValue) == SQLITE_OK &&
-        bDefensiveOldValue == 1 )
+    for( int iterOpen = 0; iterOpen < 2 ; iterOpen++ )
     {
-        if( sqlite3_db_config(hDB, SQLITE_DBCONFIG_DEFENSIVE, 0, nullptr) == SQLITE_OK )
+        int rc;
+        if (bUseOGRVFS || STARTS_WITH(m_pszFilename, "/vsi"))
         {
-            CPLDebug("SQLITE", "Disabling defensive mode succeeded");
+            pMyVFS = OGRSQLiteCreateVFS(OGRSQLiteBaseDataSourceNotifyFileOpened, this);
+            sqlite3_vfs_register(pMyVFS, 0);
+            rc = sqlite3_open_v2( m_osFilenameForSQLiteOpen.c_str(), &hDB, flags, pMyVFS->zName );
         }
         else
         {
-            CPLDebug("SQLITE", "Could not disable defensive mode");
+            rc = sqlite3_open_v2( m_osFilenameForSQLiteOpen.c_str(), &hDB, flags, nullptr );
         }
-    }
-#endif
 
-#ifdef SQLITE_FCNTL_PERSIST_WAL
-    int nPersistentWAL = -1;
-    sqlite3_file_control(hDB, "main", SQLITE_FCNTL_PERSIST_WAL, &nPersistentWAL);
-    if( nPersistentWAL == 1 )
-    {
-        nPersistentWAL = 0;
-        if( sqlite3_file_control(hDB, "main", SQLITE_FCNTL_PERSIST_WAL, &nPersistentWAL) == SQLITE_OK )
+        if( rc != SQLITE_OK )
         {
-            CPLDebug("SQLITE", "Disabling persistent WAL succeeded");
+            CPLError( CE_Failure, CPLE_OpenFailed,
+                      "sqlite3_open(%s) failed: %s",
+                      m_pszFilename, sqlite3_errmsg( hDB ) );
+            return FALSE;
         }
-        else
-        {
-            CPLDebug("SQLITE", "Could not disable persistent WAL");
+
+#ifdef SQLITE_DBCONFIG_DEFENSIVE
+        // SQLite builds on recent MacOS enable defensive mode by default, which
+        // causes issues in the VDV driver (when updating a deleted database),
+        // or in the GPKG driver (when modifying a CREATE TABLE DDL with writable_schema=ON)
+        // So disable it.
+        int bDefensiveOldValue = 0;
+        if( sqlite3_db_config(hDB, SQLITE_DBCONFIG_DEFENSIVE, -1, &bDefensiveOldValue) == SQLITE_OK &&
+            bDefensiveOldValue == 1 )
+        {
+            if( sqlite3_db_config(hDB, SQLITE_DBCONFIG_DEFENSIVE, 0, nullptr) == SQLITE_OK )
+            {
+                CPLDebug("SQLITE", "Disabling defensive mode succeeded");
+            }
+            else
+            {
+                CPLDebug("SQLITE", "Could not disable defensive mode");
+            }
         }
-    }
 #endif
 
-    const char* pszSqlitePragma =
-                            CPLGetConfigOption("OGR_SQLITE_PRAGMA", nullptr);
-    CPLString osJournalMode =
-                        CPLGetConfigOption("OGR_SQLITE_JOURNAL", "");
+#ifdef SQLITE_FCNTL_PERSIST_WAL
+        int nPersistentWAL = -1;
+        sqlite3_file_control(hDB, "main", SQLITE_FCNTL_PERSIST_WAL, &nPersistentWAL);
+        if( nPersistentWAL == 1 )
+        {
+            nPersistentWAL = 0;
+            if( sqlite3_file_control(hDB, "main", SQLITE_FCNTL_PERSIST_WAL, &nPersistentWAL) == SQLITE_OK )
+            {
+                CPLDebug("SQLITE", "Disabling persistent WAL succeeded");
+            }
+            else
+            {
+                CPLDebug("SQLITE", "Could not disable persistent WAL");
+            }
+        }
+#endif
 
-    bool bPageSizeFound = false;
-    if (pszSqlitePragma != nullptr)
-    {
-        char** papszTokens = CSLTokenizeString2( pszSqlitePragma, ",",
-                                                 CSLT_HONOURSTRINGS );
-        for(int i=0; papszTokens[i] != nullptr; i++ )
+        if (pszSqlitePragma != nullptr)
         {
-            if( STARTS_WITH_CI(papszTokens[i], "PAGE_SIZE") )
-                bPageSizeFound = true;
-            if( STARTS_WITH_CI(papszTokens[i], "JOURNAL_MODE") )
+            char** papszTokens = CSLTokenizeString2( pszSqlitePragma, ",",
+                                                     CSLT_HONOURSTRINGS );
+            for(int i=0; papszTokens[i] != nullptr; i++ )
             {
-                const char* pszEqual = strchr(papszTokens[i], '=');
-                if( pszEqual )
+                if( STARTS_WITH_CI(papszTokens[i], "PAGE_SIZE") )
+                    bPageSizeFound = true;
+                if( STARTS_WITH_CI(papszTokens[i], "JOURNAL_MODE") )
                 {
-                    osJournalMode = pszEqual + 1;
-                    osJournalMode.Trim();
-                    // Only apply journal_mode after changing page_size
-                    continue;
+                    const char* pszEqual = strchr(papszTokens[i], '=');
+                    if( pszEqual )
+                    {
+                        osJournalMode = pszEqual + 1;
+                        osJournalMode.Trim();
+                        // Only apply journal_mode after changing page_size
+                        continue;
+                    }
                 }
-            }
 
-            const char* pszSQL = CPLSPrintf("PRAGMA %s", papszTokens[i]);
+                const char* pszSQL = CPLSPrintf("PRAGMA %s", papszTokens[i]);
 
-            CPL_IGNORE_RET_VAL(
-                sqlite3_exec( hDB, pszSQL, nullptr, nullptr, nullptr ) );
+                CPL_IGNORE_RET_VAL(
+                    sqlite3_exec( hDB, pszSQL, nullptr, nullptr, nullptr ) );
+            }
+            CSLDestroy(papszTokens);
+        }
+
+        const char* pszVal = CPLGetConfigOption("SQLITE_BUSY_TIMEOUT", "5000");
+        if ( pszVal != nullptr ) {
+            sqlite3_busy_timeout(hDB, atoi(pszVal));
         }
-        CSLDestroy(papszTokens);
-    }
 
-    const char* pszVal = CPLGetConfigOption("SQLITE_BUSY_TIMEOUT", "5000");
-    if ( pszVal != nullptr ) {
-        sqlite3_busy_timeout(hDB, atoi(pszVal));
+#ifdef SQLITE_OPEN_URI
+        if( iterOpen == 0 && m_osFilenameForSQLiteOpen != m_pszFilename &&
+            m_osFilenameForSQLiteOpen.find("?nolock=1") != std::string::npos )
+        {
+            int nRowCount = 0, nColCount = 0;
+            char** papszResult = nullptr;
+            rc = sqlite3_get_table( hDB,
+                            "PRAGMA journal_mode",
+                            &papszResult, &nRowCount, &nColCount,
+                            nullptr );
+            bool bWal = false;
+            // rc == SQLITE_CANTOPEN seems to be what we get when issuing the
+            // above in nolock mode on a wal enabled file
+            if( rc != SQLITE_OK || (nRowCount == 1 && nColCount == 1 &&
+                papszResult[1] && EQUAL(papszResult[1], "wal")) )
+            {
+                bWal = true;
+            }
+            sqlite3_free_table(papszResult);
+            if( bWal )
+            {
+                flags &= ~SQLITE_OPEN_URI;
+                sqlite3_close(hDB);
+                hDB = nullptr;
+                CPLDebug("SQLite", "Cannot open %s in nolock mode because it is presumably in -wal mode", m_pszFilename);
+                m_osFilenameForSQLiteOpen = m_pszFilename;
+                continue;
+            }
+        }
+#endif
+        break;
     }
 
     if( (flagsIn & SQLITE_OPEN_CREATE) == 0 )
@@ -957,7 +1010,7 @@ int OGRSQLiteBaseDataSource::OpenOrCreateDB(int flagsIn, bool bRegisterOGR2SQLit
         int nRowCount = 0, nColCount = 0;
         char** papszResult = nullptr;
         char* pszErrMsg = nullptr;
-        rc = sqlite3_get_table( hDB,
+        int rc = sqlite3_get_table( hDB,
                         "SELECT 1 FROM sqlite_master "
                         "WHERE (type = 'trigger' OR type = 'view') AND ("
                         "sql LIKE '%%ogr_geocode%%' OR "
@@ -1015,6 +1068,13 @@ int OGRSQLiteBaseDataSource::OpenOrCreateDB(int flagsIn, bool bRegisterOGR2SQLit
         }
     }
 
+    if( m_osFilenameForSQLiteOpen != m_pszFilename &&
+            m_osFilenameForSQLiteOpen.find("?nolock=1") != std::string::npos )
+    {
+        m_bNoLock = true;
+        CPLDebug("SQLite", "%s open in nolock mode", m_pszFilename);
+    }
+
     if( !bPageSizeFound && (flagsIn & SQLITE_OPEN_CREATE) != 0 )
     {
         // Since sqlite 3.12 the default page_size is now 4096. But we