diff --git a/src/itemdb.d b/src/itemdb.d index 8b91b70af..24be37d93 100644 --- a/src/itemdb.d +++ b/src/itemdb.d @@ -45,6 +45,9 @@ struct Item { string size; } +class Lock { +} + // Construct an Item DB struct from a JSON driveItem Item makeDatabaseItem(JSONValue driveItem) { @@ -63,6 +66,7 @@ Item makeDatabaseItem(JSONValue driveItem) { item.mtime = SysTime(0); } else { // Item is not in a deleted state + string lastModifiedTimestamp; // Resolve 'Key not found: fileSystemInfo' when then item is a remote item // https://github.com/abraunegg/onedrive/issues/11 if (isItemRemote(driveItem)) { @@ -72,17 +76,51 @@ Item makeDatabaseItem(JSONValue driveItem) { // See: https://github.com/abraunegg/onedrive/issues/1533 if ("fileSystemInfo" in driveItem["remoteItem"]) { // 'fileSystemInfo' is in 'remoteItem' which will be the majority of cases - item.mtime = SysTime.fromISOExtString(driveItem["remoteItem"]["fileSystemInfo"]["lastModifiedDateTime"].str); + lastModifiedTimestamp = strip(driveItem["remoteItem"]["fileSystemInfo"]["lastModifiedDateTime"].str); + // is lastModifiedTimestamp valid? + if (isValidUTCDateTime(lastModifiedTimestamp)) { + // string is a valid timestamp + item.mtime = SysTime.fromISOExtString(lastModifiedTimestamp); + } else { + // invalid timestamp from JSON file + addLogEntry("WARNING: Invalid timestamp provided by the Microsoft OneDrive API: " ~ lastModifiedTimestamp); + // Set mtime to Clock.currTime(UTC()) to ensure we have a valid UTC value + item.mtime = Clock.currTime(UTC()); + } } else { // is a remote item, but 'fileSystemInfo' is missing from 'remoteItem' - if ("fileSystemInfo" in driveItem) { - item.mtime = SysTime.fromISOExtString(driveItem["fileSystemInfo"]["lastModifiedDateTime"].str); + lastModifiedTimestamp = strip(driveItem["fileSystemInfo"]["lastModifiedDateTime"].str); + // is lastModifiedTimestamp valid? + if (isValidUTCDateTime(lastModifiedTimestamp)) { + // string is a valid timestamp + item.mtime = SysTime.fromISOExtString(lastModifiedTimestamp); + } else { + // invalid timestamp from JSON file + addLogEntry("WARNING: Invalid timestamp provided by the Microsoft OneDrive API: " ~ lastModifiedTimestamp); + // Set mtime to Clock.currTime(UTC()) to ensure we have a valid UTC value + item.mtime = Clock.currTime(UTC()); } } } else { // Does fileSystemInfo exist at all ? if ("fileSystemInfo" in driveItem) { - item.mtime = SysTime.fromISOExtString(driveItem["fileSystemInfo"]["lastModifiedDateTime"].str); + // fileSystemInfo exists + lastModifiedTimestamp = strip(driveItem["fileSystemInfo"]["lastModifiedDateTime"].str); + // is lastModifiedTimestamp valid? + if (isValidUTCDateTime(lastModifiedTimestamp)) { + // string is a valid timestamp + item.mtime = SysTime.fromISOExtString(lastModifiedTimestamp); + } else { + // invalid timestamp from JSON file + addLogEntry("WARNING: Invalid timestamp provided by the Microsoft OneDrive API: " ~ lastModifiedTimestamp); + // Set mtime to Clock.currTime(UTC()) to ensure we have a valid UTC value + item.mtime = Clock.currTime(UTC()); + } + } else { + // no timestamp from JSON file + addLogEntry("WARNING: No timestamp provided by the Microsoft OneDrive API - using current system time for item!"); + // Set mtime to Clock.currTime(UTC()) to ensure we have a valid UTC value + item.mtime = Clock.currTime(UTC()); } } } @@ -188,7 +226,7 @@ Item makeDatabaseItem(JSONValue driveItem) { final class ItemDatabase { // increment this for every change in the db schema - immutable int itemDatabaseVersion = 13; + immutable int itemDatabaseVersion = 14; Database db; string insertItemStmt; @@ -198,6 +236,7 @@ final class ItemDatabase { string selectItemByParentIdStmt; string deleteItemByIdStmt; bool databaseInitialised = false; + shared(Lock) lock = new Lock(); this(string filename) { db = Database(filename); @@ -256,7 +295,7 @@ final class ItemDatabase { // What is the threadsafe value auto threadsafeValue = db.getThreadsafeValue(); - addLogEntry("Threadsafe database value: " ~ to!string(threadsafeValue), ["debug"]); + addLogEntry("SQLite Threadsafe database value: " ~ to!string(threadsafeValue), ["debug"]); try { // Set the enforcement of foreign key constraints. @@ -288,12 +327,11 @@ final class ItemDatabase { // The synchronous setting determines how carefully SQLite writes data to disk, balancing between performance and data safety. // https://sqlite.org/pragma.html#pragma_synchronous // PRAGMA synchronous = 0 | OFF | 1 | NORMAL | 2 | FULL | 3 | EXTRA; - db.exec("PRAGMA synchronous=NORMAL;"); + db.exec("PRAGMA synchronous=FULL;"); } catch (SqliteException exception) { detailSQLErrorMessage(exception); } - insertItemStmt = " INSERT OR REPLACE INTO item (driveId, id, name, remoteName, type, eTag, cTag, mtime, parentId, quickXorHash, sha256Hash, remoteDriveId, remoteParentId, remoteId, remoteType, syncStatus, size) VALUES (?1, ?2, ?3, ?4, ?5, ?6, ?7, ?8, ?9, ?10, ?11, ?12, ?13, ?14, ?15, ?16, ?17) @@ -397,24 +435,28 @@ final class ItemDatabase { } void insert(const ref Item item) { - auto p = db.prepare(insertItemStmt); - scope(exit) p.finalise(); // Ensure that the prepared statement is finalised after execution. - try { - bindItem(item, p); - p.exec(); - } catch (SqliteException exception) { - detailSQLErrorMessage(exception); + synchronized(lock) { + auto p = db.prepare(insertItemStmt); + scope(exit) p.finalise(); // Ensure that the prepared statement is finalised after execution. + try { + bindItem(item, p); + p.exec(); + } catch (SqliteException exception) { + detailSQLErrorMessage(exception); + } } } void update(const ref Item item) { - auto p = db.prepare(updateItemStmt); - scope(exit) p.finalise(); // Ensure that the prepared statement is finalised after execution. - try { - bindItem(item, p); - p.exec(); - } catch (SqliteException exception) { - detailSQLErrorMessage(exception); + synchronized(lock) { + auto p = db.prepare(updateItemStmt); + scope(exit) p.finalise(); // Ensure that the prepared statement is finalised after execution. + try { + bindItem(item, p); + p.exec(); + } catch (SqliteException exception) { + detailSQLErrorMessage(exception); + } } } @@ -423,200 +465,217 @@ final class ItemDatabase { } int db_checkpoint() { - return db.db_checkpoint(); + synchronized(lock) { + return db.db_checkpoint(); + } } void upsert(const ref Item item) { - Statement selectStmt = db.prepare("SELECT COUNT(*) FROM item WHERE driveId = ? AND id = ?"); - Statement executionStmt = Statement.init; // Initialise executionStmt to avoid uninitialised variable usage - - scope(exit) { - selectStmt.finalise(); - executionStmt.finalise(); - } - - try { - selectStmt.bind(1, item.driveId); - selectStmt.bind(2, item.id); - auto result = selectStmt.exec(); - size_t count = result.front[0].to!size_t; + synchronized(lock) { + Statement selectStmt = db.prepare("SELECT COUNT(*) FROM item WHERE driveId = ? AND id = ?"); + Statement executionStmt = Statement.init; // Initialise executionStmt to avoid uninitialised variable usage - if (count == 0) { - executionStmt = db.prepare(insertItemStmt); - } else { - executionStmt = db.prepare(updateItemStmt); + scope(exit) { + selectStmt.finalise(); + executionStmt.finalise(); + } + + try { + selectStmt.bind(1, item.driveId); + selectStmt.bind(2, item.id); + auto result = selectStmt.exec(); + size_t count = result.front[0].to!size_t; + + if (count == 0) { + executionStmt = db.prepare(insertItemStmt); + } else { + executionStmt = db.prepare(updateItemStmt); + } + + bindItem(item, executionStmt); + executionStmt.exec(); + } catch (SqliteException exception) { + // Handle errors appropriately + detailSQLErrorMessage(exception); } - - bindItem(item, executionStmt); - executionStmt.exec(); - } catch (SqliteException exception) { - // Handle errors appropriately - detailSQLErrorMessage(exception); } } Item[] selectChildren(const(char)[] driveId, const(char)[] id) { - - Item[] items; - auto p = db.prepare(selectItemByParentIdStmt); - scope(exit) p.finalise(); // Ensure that the prepared statement is finalised after execution. - - try { - p.bind(1, driveId); - p.bind(2, id); - auto res = p.exec(); + synchronized(lock) { + Item[] items; + auto p = db.prepare(selectItemByParentIdStmt); + scope(exit) p.finalise(); // Ensure that the prepared statement is finalised after execution. - while (!res.empty) { - items ~= buildItem(res); - res.step(); + try { + p.bind(1, driveId); + p.bind(2, id); + auto res = p.exec(); + + while (!res.empty) { + items ~= buildItem(res); + res.step(); + } + return items; + } catch (SqliteException exception) { + // Handle errors appropriately + detailSQLErrorMessage(exception); + items = []; + return items; // Return an empty array on error } - return items; - } catch (SqliteException exception) { - // Handle errors appropriately - detailSQLErrorMessage(exception); - items = []; - return items; // Return an empty array on error } } bool selectById(const(char)[] driveId, const(char)[] id, out Item item) { - auto p = db.prepare(selectItemByIdStmt); - scope(exit) p.finalise(); // Ensure that the prepared statement is finalised after execution. - - try { - p.bind(1, driveId); - p.bind(2, id); - auto r = p.exec(); - if (!r.empty) { - item = buildItem(r); - return true; + synchronized(lock) { + auto p = db.prepare(selectItemByIdStmt); + scope(exit) p.finalise(); // Ensure that the prepared statement is finalised after execution. + + try { + p.bind(1, driveId); + p.bind(2, id); + auto r = p.exec(); + if (!r.empty) { + item = buildItem(r); + return true; + } + } catch (SqliteException exception) { + // Handle the error appropriately + detailSQLErrorMessage(exception); } - } catch (SqliteException exception) { - // Handle the error appropriately - detailSQLErrorMessage(exception); - } - return false; + return false; + } } bool selectByRemoteId(const(char)[] remoteDriveId, const(char)[] remoteId, out Item item) { - auto p = db.prepare(selectItemByRemoteIdStmt); - scope(exit) p.finalise(); // Ensure that the prepared statement is finalised after execution. - try { - p.bind(1, remoteDriveId); - p.bind(2, remoteId); - auto r = p.exec(); - if (!r.empty) { - item = buildItem(r); - return true; + synchronized(lock) { + auto p = db.prepare(selectItemByRemoteIdStmt); + scope(exit) p.finalise(); // Ensure that the prepared statement is finalised after execution. + try { + p.bind(1, remoteDriveId); + p.bind(2, remoteId); + auto r = p.exec(); + if (!r.empty) { + item = buildItem(r); + return true; + } + } catch (SqliteException exception) { + // Handle the error appropriately + detailSQLErrorMessage(exception); } - } catch (SqliteException exception) { - // Handle the error appropriately - detailSQLErrorMessage(exception); - } - return false; + return false; + } } // returns true if an item id is in the database bool idInLocalDatabase(const(string) driveId, const(string) id) { - auto p = db.prepare(selectItemByIdStmt); - scope(exit) p.finalise(); // Ensure that the prepared statement is finalised after execution. - try { - p.bind(1, driveId); - p.bind(2, id); - auto r = p.exec(); - return !r.empty; - } catch (SqliteException exception) { - // Handle the error appropriately - detailSQLErrorMessage(exception); - return false; + synchronized(lock) { + auto p = db.prepare(selectItemByIdStmt); + scope(exit) p.finalise(); // Ensure that the prepared statement is finalised after execution. + try { + p.bind(1, driveId); + p.bind(2, id); + auto r = p.exec(); + return !r.empty; + } catch (SqliteException exception) { + // Handle the error appropriately + detailSQLErrorMessage(exception); + return false; + } } } // returns the item with the given path // the path is relative to the sync directory ex: "./Music/file_name.mp3" bool selectByPath(const(char)[] path, string rootDriveId, out Item item) { - Item currItem = { driveId: rootDriveId }; - - // Issue https://github.com/abraunegg/onedrive/issues/578 - path = "root/" ~ (startsWith(path, "./") || path == "." ? path.chompPrefix(".") : path); - - auto s = db.prepare("SELECT * FROM item WHERE name = ?1 AND driveId IS ?2 AND parentId IS ?3"); - scope(exit) s.finalise(); // Ensure that the prepared statement is finalised after execution. + synchronized(lock) { + Item currItem = { driveId: rootDriveId }; + + // Issue https://github.com/abraunegg/onedrive/issues/578 + path = "root/" ~ (startsWith(path, "./") || path == "." ? path.chompPrefix(".") : path); + + auto s = db.prepare("SELECT * FROM item WHERE name = ?1 AND driveId IS ?2 AND parentId IS ?3"); + scope(exit) s.finalise(); // Ensure that the prepared statement is finalised after execution. - try { - foreach (name; pathSplitter(path)) { - s.bind(1, name); - s.bind(2, currItem.driveId); - s.bind(3, currItem.id); - auto r = s.exec(); - if (r.empty) return false; - currItem = buildItem(r); - - // If the item is of type remote, substitute it with the child - if (currItem.type == ItemType.remote) { - addLogEntry("Record is a Remote Object: " ~ to!string(currItem), ["debug"]); - Item child; - if (selectById(currItem.remoteDriveId, currItem.remoteId, child)) { - assert(child.type != ItemType.remote, "The type of the child cannot be remote"); - currItem = child; - addLogEntry("Selecting Record that is NOT Remote Object: " ~ to!string(currItem), ["debug"]); + try { + foreach (name; pathSplitter(path)) { + s.bind(1, name); + s.bind(2, currItem.driveId); + s.bind(3, currItem.id); + auto r = s.exec(); + if (r.empty) return false; + currItem = buildItem(r); + + // If the item is of type remote, substitute it with the child + if (currItem.type == ItemType.remote) { + addLogEntry("Record is a Remote Object: " ~ to!string(currItem), ["debug"]); + Item child; + if (selectById(currItem.remoteDriveId, currItem.remoteId, child)) { + assert(child.type != ItemType.remote, "The type of the child cannot be remote"); + currItem = child; + addLogEntry("Selecting Record that is NOT Remote Object: " ~ to!string(currItem), ["debug"]); + } } } + item = currItem; + return true; + } catch (SqliteException exception) { + // Handle the error appropriately + detailSQLErrorMessage(exception); + return false; } - item = currItem; - return true; - } catch (SqliteException exception) { - // Handle the error appropriately - detailSQLErrorMessage(exception); - return false; } } // same as selectByPath() but it does not traverse remote folders, returns the remote element if that is what is required bool selectByPathIncludingRemoteItems(const(char)[] path, string rootDriveId, out Item item) { - Item currItem = { driveId: rootDriveId }; + synchronized(lock) { + Item currItem = { driveId: rootDriveId }; - // Issue https://github.com/abraunegg/onedrive/issues/578 - path = "root/" ~ (startsWith(path, "./") || path == "." ? path.chompPrefix(".") : path); + // Issue https://github.com/abraunegg/onedrive/issues/578 + path = "root/" ~ (startsWith(path, "./") || path == "." ? path.chompPrefix(".") : path); - auto s = db.prepare("SELECT * FROM item WHERE name IS ?1 AND driveId IS ?2 AND parentId IS ?3"); - scope(exit) s.finalise(); // Ensure that the prepared statement is finalised after execution. + auto s = db.prepare("SELECT * FROM item WHERE name IS ?1 AND driveId IS ?2 AND parentId IS ?3"); + scope(exit) s.finalise(); // Ensure that the prepared statement is finalised after execution. - try { - foreach (name; pathSplitter(path)) { - s.bind(1, name); - s.bind(2, currItem.driveId); - s.bind(3, currItem.id); - auto r = s.exec(); - if (r.empty) return false; - currItem = buildItem(r); - } + try { + foreach (name; pathSplitter(path)) { + s.bind(1, name); + s.bind(2, currItem.driveId); + s.bind(3, currItem.id); + auto r = s.exec(); + if (r.empty) return false; + currItem = buildItem(r); + } - if (currItem.type == ItemType.remote) { - addLogEntry("Record selected is a Remote Object: " ~ to!string(currItem), ["debug"]); - } + if (currItem.type == ItemType.remote) { + addLogEntry("Record selected is a Remote Object: " ~ to!string(currItem), ["debug"]); + } - item = currItem; - return true; - } catch (SqliteException exception) { - // Handle the error appropriately - detailSQLErrorMessage(exception); - return false; + item = currItem; + return true; + } catch (SqliteException exception) { + // Handle the error appropriately + detailSQLErrorMessage(exception); + return false; + } } } void deleteById(const(char)[] driveId, const(char)[] id) { - auto p = db.prepare(deleteItemByIdStmt); - scope(exit) p.finalise(); // Ensure that the prepared statement is finalised after execution. - try { - p.bind(1, driveId); - p.bind(2, id); - p.exec(); - } catch (SqliteException exception) { - // Handle the error appropriately - detailSQLErrorMessage(exception); + synchronized(lock) { + auto p = db.prepare(deleteItemByIdStmt); + scope(exit) p.finalise(); // Ensure that the prepared statement is finalised after execution. + try { + p.bind(1, driveId); + p.bind(2, id); + p.exec(); + } catch (SqliteException exception) { + // Handle the error appropriately + detailSQLErrorMessage(exception); + } } } @@ -663,6 +722,8 @@ final class ItemDatabase { private Item buildItem(Statement.Result result) { assert(!result.empty, "The result must not be empty"); assert(result.front.length == 18, "The result must have 18 columns"); + assert(isValidUTCDateTime(result.front[7].dup), "The DB record mtime entry is not a valid ISO timestamp entry. Please attempt a --resync to fix the local database."); + Item item = { // column 0: driveId @@ -691,7 +752,7 @@ final class ItemDatabase { // Column 4 is type - not set here eTag: result.front[5].dup, cTag: result.front[6].dup, - mtime: SysTime.fromISOExtString(result.front[7]), + mtime: SysTime.fromISOExtString(result.front[7].dup), parentId: result.front[8].dup, quickXorHash: result.front[9].dup, sha256Hash: result.front[10].dup, @@ -727,131 +788,139 @@ final class ItemDatabase { // the path is relative to the sync directory ex: "Music/Turbo Killer.mp3" // the trailing slash is not added even if the item is a directory string computePath(const(char)[] driveId, const(char)[] id) { - assert(driveId && id); - string path; - Item item; - auto s = db.prepare("SELECT * FROM item WHERE driveId = ?1 AND id = ?2"); - auto s2 = db.prepare("SELECT driveId, id FROM item WHERE remoteDriveId = ?1 AND remoteId = ?2"); - - scope(exit) { - s.finalise(); // Ensure that the prepared statement is finalised after execution. - s2.finalise(); // Ensure that the prepared statement is finalised after execution. - } - - try { - while (true) { - s.bind(1, driveId); - s.bind(2, id); - auto r = s.exec(); - if (!r.empty) { - item = buildItem(r); - if (item.type == ItemType.remote) { - // substitute the last name with the current - ptrdiff_t idx = indexOf(path, '/'); - path = idx >= 0 ? item.name ~ path[idx .. $] : item.name; + synchronized(lock) { + assert(driveId && id); + string path; + Item item; + auto s = db.prepare("SELECT * FROM item WHERE driveId = ?1 AND id = ?2"); + auto s2 = db.prepare("SELECT driveId, id FROM item WHERE remoteDriveId = ?1 AND remoteId = ?2"); + + scope(exit) { + s.finalise(); // Ensure that the prepared statement is finalised after execution. + s2.finalise(); // Ensure that the prepared statement is finalised after execution. + } + + try { + while (true) { + s.bind(1, driveId); + s.bind(2, id); + auto r = s.exec(); + if (!r.empty) { + item = buildItem(r); + if (item.type == ItemType.remote) { + // substitute the last name with the current + ptrdiff_t idx = indexOf(path, '/'); + path = idx >= 0 ? item.name ~ path[idx .. $] : item.name; + } else { + if (path) path = item.name ~ "/" ~ path; + else path = item.name; + } + id = item.parentId; } else { - if (path) path = item.name ~ "/" ~ path; - else path = item.name; - } - id = item.parentId; - } else { - if (id == null) { - // check for remoteItem - s2.bind(1, item.driveId); - s2.bind(2, item.id); - auto r2 = s2.exec(); - if (r2.empty) { - // root reached - assert(path.length >= 4); - // remove "root/" from path string if it exists - if (path.length >= 5) { - if (canFind(path, "root/")){ - path = path[5 .. $]; + if (id == null) { + // check for remoteItem + s2.bind(1, item.driveId); + s2.bind(2, item.id); + auto r2 = s2.exec(); + if (r2.empty) { + // root reached + assert(path.length >= 4); + // remove "root/" from path string if it exists + if (path.length >= 5) { + if (canFind(path, "root/")){ + path = path[5 .. $]; + } + } else { + path = path[4 .. $]; } + // special case of computing the path of the root itself + if (path.length == 0) path = "."; + break; } else { - path = path[4 .. $]; + // remote folder + driveId = r2.front[0].dup; + id = r2.front[1].dup; } - // special case of computing the path of the root itself - if (path.length == 0) path = "."; - break; } else { - // remote folder - driveId = r2.front[0].dup; - id = r2.front[1].dup; + // broken tree + addLogEntry("The following generated a broken tree query:", ["debug"]); + addLogEntry("Drive ID: " ~ to!string(driveId), ["debug"]); + addLogEntry("Item ID: " ~ to!string(id), ["debug"]); + assert(0); } - } else { - // broken tree - addLogEntry("The following generated a broken tree query:", ["debug"]); - addLogEntry("Drive ID: " ~ to!string(driveId), ["debug"]); - addLogEntry("Item ID: " ~ to!string(id), ["debug"]); - assert(0); } } + } catch (SqliteException exception) { + // Handle the error appropriately + detailSQLErrorMessage(exception); } - } catch (SqliteException exception) { - // Handle the error appropriately - detailSQLErrorMessage(exception); + + return path; } - - return path; } Item[] selectRemoteItems() { - Item[] items; - auto stmt = db.prepare("SELECT * FROM item WHERE remoteDriveId IS NOT NULL"); - scope (exit) stmt.finalise(); // Ensure that the prepared statement is finalised after execution. + synchronized(lock) { + Item[] items; + auto stmt = db.prepare("SELECT * FROM item WHERE remoteDriveId IS NOT NULL"); + scope (exit) stmt.finalise(); // Ensure that the prepared statement is finalised after execution. - try { - auto res = stmt.exec(); - while (!res.empty) { - items ~= buildItem(res); - res.step(); + try { + auto res = stmt.exec(); + while (!res.empty) { + items ~= buildItem(res); + res.step(); + } + } catch (SqliteException exception) { + // Handle the error appropriately + detailSQLErrorMessage(exception); } - } catch (SqliteException exception) { - // Handle the error appropriately - detailSQLErrorMessage(exception); + return items; } - return items; } string getDeltaLink(const(char)[] driveId, const(char)[] id) { - // Log what we received - addLogEntry("DeltaLink Query (driveId): " ~ to!string(driveId), ["debug"]); - addLogEntry("DeltaLink Query (id): " ~ to!string(id), ["debug"]); - // assert if these are null - assert(driveId && id); - - auto stmt = db.prepare("SELECT deltaLink FROM item WHERE driveId = ?1 AND id = ?2"); - scope(exit) stmt.finalise(); // Ensure that the prepared statement is finalised after execution. + synchronized(lock) { + // Log what we received + addLogEntry("DeltaLink Query (driveId): " ~ to!string(driveId), ["debug"]); + addLogEntry("DeltaLink Query (id): " ~ to!string(id), ["debug"]); + // assert if these are null + assert(driveId && id); + + auto stmt = db.prepare("SELECT deltaLink FROM item WHERE driveId = ?1 AND id = ?2"); + scope(exit) stmt.finalise(); // Ensure that the prepared statement is finalised after execution. - try { - stmt.bind(1, driveId); - stmt.bind(2, id); - auto res = stmt.exec(); - if (res.empty) return null; - return res.front[0].dup; - } catch (SqliteException exception) { - // Handle the error appropriately - detailSQLErrorMessage(exception); - return null; + try { + stmt.bind(1, driveId); + stmt.bind(2, id); + auto res = stmt.exec(); + if (res.empty) return null; + return res.front[0].dup; + } catch (SqliteException exception) { + // Handle the error appropriately + detailSQLErrorMessage(exception); + return null; + } } } void setDeltaLink(const(char)[] driveId, const(char)[] id, const(char)[] deltaLink) { - assert(driveId && id); - assert(deltaLink); - - auto stmt = db.prepare("UPDATE item SET deltaLink = ?3 WHERE driveId = ?1 AND id = ?2"); - scope(exit) stmt.finalise(); // Ensure that the prepared statement is finalised after execution. - - try { - stmt.bind(1, driveId); - stmt.bind(2, id); - stmt.bind(3, deltaLink); - stmt.exec(); - } catch (SqliteException exception) { - // Handle the error appropriately - detailSQLErrorMessage(exception); + synchronized(lock) { + assert(driveId && id); + assert(deltaLink); + + auto stmt = db.prepare("UPDATE item SET deltaLink = ?3 WHERE driveId = ?1 AND id = ?2"); + scope(exit) stmt.finalise(); // Ensure that the prepared statement is finalised after execution. + + try { + stmt.bind(1, driveId); + stmt.bind(2, id); + stmt.bind(3, deltaLink); + stmt.exec(); + } catch (SqliteException exception) { + // Handle the error appropriately + detailSQLErrorMessage(exception); + } } } @@ -865,20 +934,22 @@ final class ItemDatabase { // to be flagged as not-in-sync, thus, we can use that flag to determine what was previously // in-sync, but now deleted on OneDrive void downgradeSyncStatusFlag(const(char)[] driveId, const(char)[] id) { - assert(driveId); + synchronized(lock) { + assert(driveId); - auto stmt = db.prepare("UPDATE item SET syncStatus = 'N' WHERE driveId = ?1 AND id = ?2"); - scope(exit) { - stmt.finalise(); // Ensure that the prepared statement is finalised after execution. - } + auto stmt = db.prepare("UPDATE item SET syncStatus = 'N' WHERE driveId = ?1 AND id = ?2"); + scope(exit) { + stmt.finalise(); // Ensure that the prepared statement is finalised after execution. + } - try { - stmt.bind(1, driveId); - stmt.bind(2, id); - stmt.exec(); - } catch (SqliteException exception) { - // Handle the error appropriately - detailSQLErrorMessage(exception); + try { + stmt.bind(1, driveId); + stmt.bind(2, id); + stmt.exec(); + } catch (SqliteException exception) { + // Handle the error appropriately + detailSQLErrorMessage(exception); + } } } @@ -890,157 +961,169 @@ final class ItemDatabase { // // Select items that have a out-of-sync flag set Item[] selectOutOfSyncItems(const(char)[] driveId) { - assert(driveId); - Item[] items; - auto stmt = db.prepare("SELECT * FROM item WHERE syncStatus = 'N' AND driveId = ?1"); - scope(exit) stmt.finalise(); // Ensure that the prepared statement is finalised after execution. - - try { - stmt.bind(1, driveId); - auto res = stmt.exec(); - while (!res.empty) { - items ~= buildItem(res); - res.step(); + synchronized(lock) { + assert(driveId); + Item[] items; + auto stmt = db.prepare("SELECT * FROM item WHERE syncStatus = 'N' AND driveId = ?1"); + scope(exit) stmt.finalise(); // Ensure that the prepared statement is finalised after execution. + + try { + stmt.bind(1, driveId); + auto res = stmt.exec(); + while (!res.empty) { + items ~= buildItem(res); + res.step(); + } + } catch (SqliteException exception) { + // Handle the error appropriately + detailSQLErrorMessage(exception); } - } catch (SqliteException exception) { - // Handle the error appropriately - detailSQLErrorMessage(exception); + return items; } - return items; } // OneDrive Business Folders are stored in the database potentially without a root | parentRoot link // Select items associated with the provided driveId Item[] selectByDriveId(const(char)[] driveId) { - assert(driveId); - Item[] items; - auto stmt = db.prepare("SELECT * FROM item WHERE driveId = ?1 AND parentId IS NULL"); - scope(exit) stmt.finalise(); // Ensure that the prepared statement is finalised after execution. + synchronized(lock) { + assert(driveId); + Item[] items; + auto stmt = db.prepare("SELECT * FROM item WHERE driveId = ?1 AND parentId IS NULL"); + scope(exit) stmt.finalise(); // Ensure that the prepared statement is finalised after execution. - try { - stmt.bind(1, driveId); - auto res = stmt.exec(); - while (!res.empty) { - items ~= buildItem(res); - res.step(); + try { + stmt.bind(1, driveId); + auto res = stmt.exec(); + while (!res.empty) { + items ~= buildItem(res); + res.step(); + } + } catch (SqliteException exception) { + // Handle the error appropriately + detailSQLErrorMessage(exception); } - } catch (SqliteException exception) { - // Handle the error appropriately - detailSQLErrorMessage(exception); + return items; } - return items; } // Perform a vacuum on the database, commit WAL / SHM to file void performVacuum() { - // Log what we are attempting to do - addLogEntry("Attempting to perform a database vacuum to optimise database"); - - try { - // Check the current DB Status - we have to be in a clean state here - db.checkStatus(); + synchronized(lock) { + // Log what we are attempting to do + addLogEntry("Attempting to perform a database vacuum to optimise database"); - // Are there any open statements that need to be closed? - if (db.count_open_statements() > 0) { - // Dump open statements - db.dump_open_statements(); // dump open statements so we know what the are + try { + // Check the current DB Status - we have to be in a clean state here + db.checkStatus(); - // SIGINT (CTRL-C), SIGTERM (kill) handling - if (exitHandlerTriggered) { - // The SQLITE_INTERRUPT result code indicates that an operation was interrupted - which if we have open statements, most likely a SIGINT scenario - throw new SqliteException(9, "Open SQL Statements due to interrupted operations"); - } else { - // Try and close open statements - db.close_open_statements(); + // Are there any open statements that need to be closed? + if (db.count_open_statements() > 0) { + // Dump open statements + db.dump_open_statements(); // dump open statements so we know what the are + + // SIGINT (CTRL-C), SIGTERM (kill) handling + if (exitHandlerTriggered) { + // The SQLITE_INTERRUPT result code indicates that an operation was interrupted - which if we have open statements, most likely a SIGINT scenario + throw new SqliteException(9, "Open SQL Statements due to interrupted operations"); + } else { + // Try and close open statements + db.close_open_statements(); + } } + + // Ensure there are no pending operations by performing a checkpoint + db.exec("PRAGMA wal_checkpoint(TRUNCATE);"); + + // Prepare and execute VACUUM statement + Statement stmt = db.prepare("VACUUM;"); + scope(exit) stmt.finalise(); // Ensure the statement is finalised when we exit + stmt.exec(); + addLogEntry("Database vacuum is complete"); + } catch (SqliteException exception) { + addLogEntry(); + addLogEntry("ERROR: Unable to perform a database vacuum: " ~ exception.msg); + addLogEntry(); } - - // Ensure there are no pending operations by performing a checkpoint - db.exec("PRAGMA wal_checkpoint(TRUNCATE);"); - - // Prepare and execute VACUUM statement - Statement stmt = db.prepare("VACUUM;"); - scope(exit) stmt.finalise(); // Ensure the statement is finalised when we exit - stmt.exec(); - addLogEntry("Database vacuum is complete"); - } catch (SqliteException exception) { - addLogEntry(); - addLogEntry("ERROR: Unable to perform a database vacuum: " ~ exception.msg); - addLogEntry(); } } // Perform a checkpoint by writing the data into to the database from the WAL file void performCheckpoint() { - // Log what we are attempting to do - addLogEntry("Attempting to perform a database checkpoint to merge temporary data", ["debug"]); - - try { - // Check the current DB Status - we have to be in a clean state here - db.checkStatus(); + synchronized(lock) { + // Log what we are attempting to do + addLogEntry("Attempting to perform a database checkpoint to merge temporary data", ["debug"]); - // Are there any open statements that need to be closed? - if (db.count_open_statements() > 0) { - // Dump open statements - db.dump_open_statements(); // dump open statements so we know what the are + try { + // Check the current DB Status - we have to be in a clean state here + db.checkStatus(); - // SIGINT (CTRL-C), SIGTERM (kill) handling - if (exitHandlerTriggered) { - // The SQLITE_INTERRUPT result code indicates that an operation was interrupted - which if we have open statements, most likely a SIGINT scenario - throw new SqliteException(9, "Open SQL Statements due to interrupted operations"); - } else { - // Try and close open statements - db.close_open_statements(); + // Are there any open statements that need to be closed? + if (db.count_open_statements() > 0) { + // Dump open statements + db.dump_open_statements(); // dump open statements so we know what the are + + // SIGINT (CTRL-C), SIGTERM (kill) handling + if (exitHandlerTriggered) { + // The SQLITE_INTERRUPT result code indicates that an operation was interrupted - which if we have open statements, most likely a SIGINT scenario + throw new SqliteException(9, "Open SQL Statements due to interrupted operations"); + } else { + // Try and close open statements + db.close_open_statements(); + } } + + // Ensure there are no pending operations by performing a checkpoint + db.exec("PRAGMA wal_checkpoint(TRUNCATE);"); + addLogEntry("Database checkpoint is complete", ["debug"]); + + } catch (SqliteException exception) { + addLogEntry(); + addLogEntry("ERROR: Unable to perform a database checkpoint: " ~ exception.msg); + addLogEntry(); } - - // Ensure there are no pending operations by performing a checkpoint - db.exec("PRAGMA wal_checkpoint(TRUNCATE);"); - addLogEntry("Database checkpoint is complete", ["debug"]); - - } catch (SqliteException exception) { - addLogEntry(); - addLogEntry("ERROR: Unable to perform a database checkpoint: " ~ exception.msg); - addLogEntry(); } } // Select distinct driveId items from database string[] selectDistinctDriveIds() { - string[] driveIdArray; - auto stmt = db.prepare("SELECT DISTINCT driveId FROM item;"); - scope(exit) stmt.finalise(); // Ensure that the prepared statement is finalised after execution. - - try { - auto res = stmt.exec(); - if (res.empty) return driveIdArray; - while (!res.empty) { - driveIdArray ~= res.front[0].dup; - res.step(); + synchronized(lock) { + string[] driveIdArray; + auto stmt = db.prepare("SELECT DISTINCT driveId FROM item;"); + scope(exit) stmt.finalise(); // Ensure that the prepared statement is finalised after execution. + + try { + auto res = stmt.exec(); + if (res.empty) return driveIdArray; + while (!res.empty) { + driveIdArray ~= res.front[0].dup; + res.step(); + } + } catch (SqliteException exception) { + // Handle the error appropriately + detailSQLErrorMessage(exception); } - } catch (SqliteException exception) { - // Handle the error appropriately - detailSQLErrorMessage(exception); + return driveIdArray; } - return driveIdArray; } // Function to get the total number of rows in a table int getTotalRowCount() { - int rowCount = 0; - auto stmt = db.prepare("SELECT COUNT(*) FROM item;"); - scope(exit) stmt.finalise(); // Ensure that the prepared statement is finalised after execution. + synchronized(lock) { + int rowCount = 0; + auto stmt = db.prepare("SELECT COUNT(*) FROM item;"); + scope(exit) stmt.finalise(); // Ensure that the prepared statement is finalised after execution. - try { - auto res = stmt.exec(); - if (!res.empty) { - rowCount = res.front[0].to!int; + try { + auto res = stmt.exec(); + if (!res.empty) { + rowCount = res.front[0].to!int; + } + } catch (SqliteException exception) { + // Handle the error appropriately + detailSQLErrorMessage(exception); } - } catch (SqliteException exception) { - // Handle the error appropriately - detailSQLErrorMessage(exception); + + return rowCount; } - - return rowCount; } } \ No newline at end of file diff --git a/src/sqlite.d b/src/sqlite.d index eb47ddd5b..78430b26d 100644 --- a/src/sqlite.d +++ b/src/sqlite.d @@ -97,7 +97,13 @@ struct Database { // Open the database file void open(const(char)[] filename) { // https://www.sqlite.org/c3ref/open.html - int rc = sqlite3_open(toStringz(filename), &pDb); + // Safest multithreaded way to open the database + int rc = sqlite3_open_v2( + toStringz(filename), /* Database filename (UTF-8) */ + &pDb, /* OUT: SQLite db handle */ + SQLITE_OPEN_READWRITE | SQLITE_OPEN_CREATE | SQLITE_OPEN_FULLMUTEX, /* Flags */ + null /* Optional: Name of the VFS module to use */ + ); if (rc != SQLITE_OK) { string errorMsg; diff --git a/src/sync.d b/src/sync.d index c44452963..8af87d7de 100644 --- a/src/sync.d +++ b/src/sync.d @@ -2488,12 +2488,33 @@ class SyncEngine { try { // get the mtime from the JSON data SysTime itemModifiedTime; + string lastModifiedTimestamp; if (isItemRemote(onedriveJSONItem)) { // remote file item - itemModifiedTime = SysTime.fromISOExtString(onedriveJSONItem["remoteItem"]["fileSystemInfo"]["lastModifiedDateTime"].str); + lastModifiedTimestamp = strip(onedriveJSONItem["remoteItem"]["fileSystemInfo"]["lastModifiedDateTime"].str); + // is lastModifiedTimestamp valid? + if (isValidUTCDateTime(lastModifiedTimestamp)) { + // string is a valid timestamp + itemModifiedTime = SysTime.fromISOExtString(lastModifiedTimestamp); + } else { + // invalid timestamp from JSON file + addLogEntry("WARNING: Invalid timestamp provided by the Microsoft OneDrive API: " ~ lastModifiedTimestamp); + // Set mtime to Clock.currTime(UTC()) given that the time in the JSON should be a UTC timestamp + itemModifiedTime = Clock.currTime(UTC()); + } } else { // not a remote item - itemModifiedTime = SysTime.fromISOExtString(onedriveJSONItem["fileSystemInfo"]["lastModifiedDateTime"].str); + lastModifiedTimestamp = strip(onedriveJSONItem["fileSystemInfo"]["lastModifiedDateTime"].str); + // is lastModifiedTimestamp valid? + if (isValidUTCDateTime(lastModifiedTimestamp)) { + // string is a valid timestamp + itemModifiedTime = SysTime.fromISOExtString(lastModifiedTimestamp); + } else { + // invalid timestamp from JSON file + addLogEntry("WARNING: Invalid timestamp provided by the Microsoft OneDrive API: " ~ lastModifiedTimestamp); + // Set mtime to Clock.currTime(UTC()) given that the time in the JSON should be a UTC timestamp + itemModifiedTime = Clock.currTime(UTC()); + } } // set the correct time on the downloaded file @@ -2648,18 +2669,28 @@ class SyncEngine { addLogEntry("Local item has the same hash value as the item online - correcting the applicable file timestamp", ["verbose"]); // Correction logic based on the configuration and the comparison of timestamps if (localModifiedTime > itemModifiedTime) { - // Local file is newer .. are we in a --download-only situation? + // Local file is newer timestamp wise, but has the same hash .. are we in a --download-only situation? if (!appConfig.getValueBool("download_only") && !dryRun) { - // The source of the out-of-date timestamp was OneDrive and this needs to be corrected to avoid always generating a hash test if timestamp is different - addLogEntry("The source of the incorrect timestamp was OneDrive online - correcting timestamp online", ["verbose"]); - // Attempt to update the online date time stamp - // We need to use the correct driveId and itemId, especially if we are updating a OneDrive Business Shared File timestamp - if (item.type == ItemType.file) { - // Not a remote file - uploadLastModifiedTime(item, item.driveId, item.id, localModifiedTime, item.eTag); + // Not --download-only .. but are we in a --resync scenario? + if (appConfig.getValueBool("resync")) { + // --resync was used + // The source of the out-of-date timestamp was the local item and needs to be corrected ... but why is it newer - indexing application potentially changing the timestamp ? + addLogEntry("The source of the incorrect timestamp was the local file - correcting timestamp locally due to --resync", ["verbose"]); + // Fix the local file timestamp + addLogEntry("Calling setTimes() for this file: " ~ path, ["debug"]); + setTimes(path, item.mtime, item.mtime); } else { - // Remote file, remote values need to be used - uploadLastModifiedTime(item, item.remoteDriveId, item.remoteId, localModifiedTime, item.eTag); + // The source of the out-of-date timestamp was OneDrive and this needs to be corrected to avoid always generating a hash test if timestamp is different + addLogEntry("The source of the incorrect timestamp was OneDrive online - correcting timestamp online", ["verbose"]); + // Attempt to update the online date time stamp + // We need to use the correct driveId and itemId, especially if we are updating a OneDrive Business Shared File timestamp + if (item.type == ItemType.file) { + // Not a remote file + uploadLastModifiedTime(item, item.driveId, item.id, localModifiedTime, item.eTag); + } else { + // Remote file, remote values need to be used + uploadLastModifiedTime(item, item.remoteDriveId, item.remoteId, localModifiedTime, item.eTag); + } } } else if (!dryRun) { // --download-only is being used ... local file needs to be corrected ... but why is it newer - indexing application potentially changing the timestamp ? @@ -4410,25 +4441,32 @@ class SyncEngine { } } catch (FileException e) { - writeln("DEBUG TO REMOVE: Modified file upload FileException Handling (Create the Upload Session)"); + addLogEntry("DEBUG TO REMOVE: Modified file upload FileException Handling (Create the Upload Session)"); displayFileSystemErrorMessage(e.msg, getFunctionName!({})); } - // Perform the upload using the session that has been created - try { - uploadResponse = performSessionFileUpload(uploadFileOneDriveApiInstance, thisFileSizeLocal, uploadSessionData, threadUploadSessionFilePath); - } catch (OneDriveException exception) { - // Function name - string thisFunctionName = getFunctionName!({}); - - // Handle all other HTTP status codes - // - 408,429,503,504 errors are handled as a retry within uploadFileOneDriveApiInstance - // Display what the error is - displayOneDriveErrorMessage(exception.msg, thisFunctionName); - - } catch (FileException e) { - writeln("DEBUG TO REMOVE: Modified file upload FileException Handling (Perform the Upload using the session)"); - displayFileSystemErrorMessage(e.msg, getFunctionName!({})); + // Do we have a valid session URL that we can use ? + if (uploadSessionData.type() == JSONType.object) { + // This is a valid JSON object + // Perform the upload using the session that has been created + try { + uploadResponse = performSessionFileUpload(uploadFileOneDriveApiInstance, thisFileSizeLocal, uploadSessionData, threadUploadSessionFilePath); + } catch (OneDriveException exception) { + // Function name + string thisFunctionName = getFunctionName!({}); + + // Handle all other HTTP status codes + // - 408,429,503,504 errors are handled as a retry within uploadFileOneDriveApiInstance + // Display what the error is + displayOneDriveErrorMessage(exception.msg, thisFunctionName); + + } catch (FileException e) { + addLogEntry("DEBUG TO REMOVE: Modified file upload FileException Handling (Perform the Upload using the session)"); + displayFileSystemErrorMessage(e.msg, getFunctionName!({})); + } + } else { + // Create session Upload URL failed + addLogEntry("Unable to upload modified file as the creation of the upload session URL failed", ["debug"]); } } } else { @@ -4522,7 +4560,7 @@ class SyncEngine { if (appConfig.accountType == "personal") { addLogEntry("ERROR: OneDrive account currently has zero space available. Please free up some space online or purchase additional capacity."); } else { // Assuming 'business' or 'sharedLibrary' - addLogEntry("WARNING: OneDrive quota information is being restricted or providing a zero value. Please fix by speaking to your OneDrive / Office 365 Administrator."); + addLogEntry("WARNING: OneDrive quota information is being restricted or providing a zero value. Please fix by speaking to your OneDrive / Office 365 Administrator." , ["verbose"]); } } } else { @@ -4531,10 +4569,10 @@ class SyncEngine { // what sort of account type is this? if (appConfig.accountType == "personal") { - addLogEntry("ERROR: OneDrive quota information is missing. Your OneDrive account potentially has zero space available. Please free up some space online."); + addLogEntry("ERROR: OneDrive quota information is missing. Your OneDrive account potentially has zero space available. Please free up some space online.", ["verbose"]); } else { // quota details not available - addLogEntry("WARNING: OneDrive quota information is being restricted. Please fix by speaking to your OneDrive / Office 365 Administrator."); + addLogEntry("WARNING: OneDrive quota information is being restricted. Please fix by speaking to your OneDrive / Office 365 Administrator.", ["verbose"]); } } } else { @@ -8381,7 +8419,22 @@ class SyncEngine { // Check the session data for expirationDateTime if ("expirationDateTime" in sessionFileData) { - auto expiration = SysTime.fromISOExtString(sessionFileData["expirationDateTime"].str); + addLogEntry("expirationDateTime: " ~ sessionFileData["expirationDateTime"].str); + SysTime expiration; + string expirationTimestamp; + expirationTimestamp = strip(sessionFileData["expirationDateTime"].str); + + // is expirationTimestamp valid? + if (isValidUTCDateTime(expirationTimestamp)) { + // string is a valid timestamp + expiration = SysTime.fromISOExtString(expirationTimestamp); + } else { + // invalid timestamp from JSON file + addLogEntry("WARNING: Invalid timestamp provided by the Microsoft OneDrive API: " ~ expirationTimestamp); + return false; + } + + // valid timestamp if (expiration < Clock.currTime()) { addLogEntry("The upload session has expired for: " ~ sessionFilePath, ["verbose"]); return false; @@ -8694,6 +8747,7 @@ class SyncEngine { // New DB Tie Item to detail the 'root' of the Shared Folder Item tieDBItem; + string lastModifiedTimestamp; tieDBItem.name = "root"; // Get the right parentReference details @@ -8718,8 +8772,23 @@ class SyncEngine { } } + // set the item type tieDBItem.type = ItemType.dir; - tieDBItem.mtime = SysTime.fromISOExtString(onedriveJSONItem["fileSystemInfo"]["lastModifiedDateTime"].str); + + // get the lastModifiedDateTime + lastModifiedTimestamp = strip(onedriveJSONItem["fileSystemInfo"]["lastModifiedDateTime"].str); + // is lastModifiedTimestamp valid? + if (isValidUTCDateTime(lastModifiedTimestamp)) { + // string is a valid timestamp + tieDBItem.mtime = SysTime.fromISOExtString(lastModifiedTimestamp); + } else { + // invalid timestamp from JSON file + addLogEntry("WARNING: Invalid timestamp provided by the Microsoft OneDrive API: " ~ lastModifiedTimestamp); + // Set mtime to SysTime(0) + tieDBItem.mtime = SysTime(0); + } + + // ensure there is no parentId tieDBItem.parentId = null; // Add this DB Tie parent record to the local database diff --git a/src/util.d b/src/util.d index 18b6a1e7a..3bfa70730 100644 --- a/src/util.d +++ b/src/util.d @@ -475,6 +475,19 @@ bool containsASCIIControlCodes(string path) { return !matchResult.empty; } +// Is the string a valid UTF-8 string? +bool isValidUTF8(string input) { + try { + auto it = input.byUTF!(char); + foreach (_; it) { + // Just iterate to check for valid UTF-8 + } + return true; + } catch (UTFException) { + return false; + } +} + // Is the path a valid UTF-16 encoded path? bool isValidUTF16(string path) { // Check for null or empty string @@ -512,6 +525,33 @@ bool isValidUTF16(string path) { return true; } +// Validate that the provided string is a valid date time stamp in UTC format +bool isValidUTCDateTime(string dateTimeString) { + // Regular expression for validating the string against UTC datetime format + auto pattern = regex(r"^\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}Z$"); + + // Validate for UTF-8 first + if (!isValidUTF8(dateTimeString)) { + addLogEntry("BAD TIMESTAMP (UTF-8 FAIL): " ~ dateTimeString); + return false; + } + + // First, check if the string matches the pattern + if (!match(dateTimeString, pattern)) { + addLogEntry("BAD TIMESTAMP (REGEX FAIL): " ~ dateTimeString); + return false; + } + + // Attempt to parse the string into a DateTime object + try { + auto dt = SysTime.fromISOExtString(dateTimeString); + return true; + } catch (TimeException) { + addLogEntry("BAD TIMESTAMP (CONVERSION FAIL): " ~ dateTimeString); + return false; + } +} + // Does the path contain any HTML URL encoded items (e.g., '%20' for space) bool containsURLEncodedItems(string path) { // Check for null or empty string @@ -1296,6 +1336,7 @@ extern(C) nothrow @nogc @system void exitScopeSignalHandler(int signo) { } } +// Return the compiler details string compilerDetails() { version(DigitalMars) enum compiler = "DMD"; else version(LDC) enum compiler = "LDC";