feat(cache): File cache uses folder scan to get file list instead of DB

src/HistoryQuery/HistoryQuery.class.js

@@ -134,7 +134,8 @@ class HistoryQuery {
       return
     }
 
-    if (!this.north.isCacheEmpty()) {
+    const isCacheEmpty = await this.north.isCacheEmpty()
+    if (!isCacheEmpty) {
       this.logger.trace(`History query "${this.historySettings.id}" not over yet: Data cache not empty for "${this.northSettings.name}".`)
       return
     }

src/engine/cache/FileCache.class.js

@@ -1,7 +1,6 @@
 const fs = require('node:fs/promises')
 const path = require('node:path')
 
-const databaseService = require('../../services/database.service')
 const BaseCache = require('./BaseCache.class')
 const { createFolder } = require('../../services/utils')
 
@@ -12,9 +11,6 @@ const ERROR_FOLDER = 'errors'
 const ARCHIVE_FOLDER = 'archive'
 const FILE_FOLDER = 'files'
 
-const FILE_DB_FILE_NAME = 'files.db'
-const FILE_ERROR_DB_FILE_NAME = 'files-error.db'
-
 /**
  * Local cache implementation to group events and store them when the communication with the North is down.
  */
@@ -47,8 +43,6 @@ class FileCache extends BaseCache {
     this.fileFolder = path.resolve(this.baseFolder, FILE_FOLDER)
     this.errorFolder = path.resolve(this.baseFolder, ERROR_FOLDER)
 
-    this.filesDatabase = null
-    this.filesErrorDatabase = null
     this.archiveTimeout = null
   }
 
@@ -57,30 +51,23 @@ class FileCache extends BaseCache {
    * @returns {Promise<void>} - The result promise
    */
   async init() {
-    const filesDatabasePath = path.resolve(this.baseFolder, FILE_DB_FILE_NAME)
-    this.logger.debug(`Use file cache database: "${filesDatabasePath}".`)
-    this.filesDatabase = databaseService.createFilesDatabase(filesDatabasePath)
-    const filesCount = databaseService.getCount(this.filesDatabase)
-    if (filesCount > 0) {
-      this.logger.debug(`${filesCount} files in cache.`)
+    await createFolder(this.fileFolder)
+    await createFolder(this.errorFolder)
+
+    const files = await fs.readdir(this.fileFolder)
+    if (files.length > 0) {
+      this.logger.debug(`${files.length} files in cache.`)
     } else {
       this.logger.debug('No files in cache.')
     }
 
-    const filesErrorDatabasePath = path.resolve(this.baseFolder, FILE_ERROR_DB_FILE_NAME)
-
-    this.logger.debug(`Initialize files error db: ${filesErrorDatabasePath}`)
-    this.filesErrorDatabase = databaseService.createFilesDatabase(filesErrorDatabasePath)
-    const errorCount = databaseService.getCount(this.filesErrorDatabase)
-    if (errorCount > 0) {
-      this.logger.warn(`${errorCount} files in error cache.`)
+    const errorFiles = await fs.readdir(this.errorFolder)
+    if (errorFiles.length > 0) {
+      this.logger.warn(`${errorFiles.length} files in error cache.`)
     } else {
       this.logger.debug('No error files in cache.')
     }
 
-    await createFolder(this.fileFolder)
-    await createFolder(this.errorFolder)
-
     if (this.archiveFiles) {
       await createFolder(this.archiveFolder)
       // refresh the archiveFolder at the beginning only if retentionDuration is different from 0
@@ -98,8 +85,6 @@ class FileCache extends BaseCache {
     if (this.archiveTimeout) {
       clearTimeout(this.archiveTimeout)
     }
-    this.filesDatabase.close()
-    this.filesErrorDatabase.close()
   }
 
   /**
@@ -117,18 +102,29 @@ class FileCache extends BaseCache {
     const cachePath = path.join(this.fileFolder, cacheFilename)
 
     await fs.copyFile(filePath, cachePath)
-
-    databaseService.saveFile(this.filesDatabase, timestamp, cachePath)
     this.logger.debug(`File "${filePath}" cached in "${cachePath}".`)
   }
 
   /**
    * Retrieve files from the Cache database and send them to the associated northHandleFileFunction function
    * This method is called when the group count is reached or when the Cache timeout is reached
-   * @returns {{path: string, timestamp: number}|null} - The file to send
+   * @returns {Promise<{path: string, timestamp: number}|null>} - The file to send
    */
-  retrieveFileFromCache() {
-    return databaseService.getFileToSend(this.filesDatabase)
+  async retrieveFileFromCache() {
+    const fileNames = await fs.readdir(this.fileFolder)
+
+    if (fileNames.length === 0) {
+      return null
+    }
+
+    const sortedFiles = fileNames
+      .map(async (fileName) => ({
+        path: `${this.fileFolder}/${fileName}`,
+        timestamp: (await fs.stat(`${this.fileFolder}/${fileName}`)).mtime.getTime(),
+      }))
+      .sort((a, b) => a.time - b.time)
+
+    return sortedFiles[0]
   }
 
   /**
@@ -138,9 +134,6 @@ class FileCache extends BaseCache {
    * @returns {Promise<void>} - The result promise
    */
   async manageErroredFiles(filePathInCache) {
-    databaseService.saveFile(this.filesErrorDatabase, new Date().getTime(), filePathInCache)
-    databaseService.deleteSentFile(this.filesDatabase, filePathInCache)
-
     const filenameInfo = path.parse(filePathInCache)
     const errorPath = path.join(this.errorFolder, filenameInfo.base)
     // Move cache file into the archive folder
@@ -159,8 +152,6 @@ class FileCache extends BaseCache {
    * @return {Promise<void>} - The result promise
    */
   async removeFileFromCache(filePathInCache, archiveFile) {
-    databaseService.deleteSentFile(this.filesDatabase, filePathInCache)
-
     if (archiveFile) {
       const filenameInfo = path.parse(filePathInCache)
       const archivePath = path.join(this.archiveFolder, filenameInfo.base)
@@ -186,8 +177,9 @@ class FileCache extends BaseCache {
    * Check if the file cache is empty or not
    * @returns {Boolean} - Cache empty or not
    */
-  isEmpty() {
-    return databaseService.getFileCountForNorthConnector(this.filesDatabase) === 0
+  async isEmpty() {
+    const files = await fs.readdir(this.fileFolder)
+    return files.length === 0
   }
 
   /**

src/north/NorthConnector.class.js

@@ -240,7 +240,7 @@ class NorthConnector {
       return
     }
 
-    const fileToSend = this.fileCache.retrieveFileFromCache()
+    const fileToSend = await this.fileCache.retrieveFileFromCache()
     if (!fileToSend) {
       this.logger.trace('No file to send in the cache database.')
       this.resetFilesTimeout(this.settings.caching.sendInterval)
@@ -364,10 +364,12 @@ class NorthConnector {
 
   /**
    * Check appropriate caches emptiness
-   * @returns {Boolean} - True if North cache is empty, false otherwise
+   * @returns {Promise<Boolean>} - True if North cache is empty, false otherwise
    */
-  isCacheEmpty() {
-    return this.valueCache.isEmpty() && this.fileCache.isEmpty()
+  async isCacheEmpty() {
+    const isValueCacheEmpty = this.valueCache.isEmpty()
+    const isFileCacheEmpty = await this.fileCache.isEmpty()
+    return isValueCacheEmpty && isFileCacheEmpty
   }
 }
 

src/north/NorthConnector.class.spec.js

@@ -339,22 +339,21 @@ describe('NorthConnector', () => {
     expect(north.isSubscribed('southId')).toBeTruthy()
   })
 
-  it('should check if North caches are empty', () => {
+  it('should check if North caches are empty', async () => {
     north.valueCache.isEmpty.mockReturnValue(true)
-    north.fileCache.isEmpty.mockReturnValue(true)
-
-    expect(north.isCacheEmpty()).toBeTruthy()
+    north.fileCache.isEmpty.mockReturnValue(Promise.resolve(true))
+    expect(await north.isCacheEmpty()).toBeTruthy()
 
     north.valueCache.isEmpty.mockReturnValue(true)
-    north.fileCache.isEmpty.mockReturnValue(false)
-    expect(north.isCacheEmpty()).toBeFalsy()
+    north.fileCache.isEmpty.mockReturnValue(Promise.resolve(false))
+    expect(await north.isCacheEmpty()).toBeFalsy()
 
     north.valueCache.isEmpty.mockReturnValue(false)
-    north.fileCache.isEmpty.mockReturnValue(true)
-    expect(north.isCacheEmpty()).toBeFalsy()
+    north.fileCache.isEmpty.mockReturnValue(Promise.resolve(true))
+    expect(await north.isCacheEmpty()).toBeFalsy()
 
     north.valueCache.isEmpty.mockReturnValue(false)
-    north.fileCache.isEmpty.mockReturnValue(false)
-    expect(north.isCacheEmpty()).toBeFalsy()
+    north.fileCache.isEmpty.mockReturnValue(Promise.resolve(false))
+    expect(await north.isCacheEmpty()).toBeFalsy()
   })
 })

src/services/database.class.spec.js

@@ -74,17 +74,6 @@ describe('Database services', () => {
     expect(expectedDatabase).toBe(mockDatabase)
   })
 
-  it('should create file database', () => {
-    const expectedDatabase = databaseService.createFilesDatabase('myDb.db')
-    expect(prepare).toHaveBeenCalledTimes(1)
-    expect(prepare).toHaveBeenCalledWith('CREATE TABLE IF NOT EXISTS cache ('
-        + 'id INTEGER PRIMARY KEY, '
-        + 'timestamp INTEGER, '
-        + 'path TEXT);')
-    expect(run).toHaveBeenCalledTimes(1)
-    expect(expectedDatabase).toBe(mockDatabase)
-  })
-
   it('should create config database', () => {
     const expectedDatabase = databaseService.createConfigDatabase('myDb.db')
     expect(prepare).toHaveBeenCalledTimes(1)
@@ -152,75 +141,6 @@ describe('Database services', () => {
     expect(result).toEqual(2)
   })
 
-  it('should save file', () => {
-    databaseService.saveFile(mockDatabase, 123, 'myFilePath')
-    expect(prepare).toHaveBeenCalledTimes(1)
-    expect(prepare).toHaveBeenCalledWith('INSERT INTO cache (timestamp, path) VALUES (?, ?)')
-
-    expect(run).toHaveBeenCalledTimes(1)
-    expect(run).toHaveBeenCalledWith(123, 'myFilePath')
-  })
-
-  it('should get file to send', () => {
-    const result = databaseService.getFileToSend(mockDatabase)
-    expect(prepare).toHaveBeenCalledTimes(1)
-    expect(prepare).toHaveBeenCalledWith('SELECT path, timestamp '
-        + 'FROM cache '
-        + 'ORDER BY timestamp '
-        + 'LIMIT 1')
-
-    expect(all).toHaveBeenCalledTimes(1)
-    expect(all).toHaveBeenCalledWith()
-    expect(result).toEqual({ path: 'myFilePath', timestamp: 123 })
-  })
-
-  it('should return null if no file to send', () => {
-    mockDatabase.prepare.mockReturnValue({ all: jest.fn(() => []) })
-
-    const result = databaseService.getFileToSend(mockDatabase)
-
-    expect(result).toEqual(null)
-  })
-
-  it('should delete sent file', () => {
-    databaseService.deleteSentFile(mockDatabase, 'myFilePath')
-    expect(prepare).toHaveBeenCalledTimes(1)
-    expect(prepare).toHaveBeenCalledWith('DELETE FROM cache WHERE path = ?')
-
-    expect(run).toHaveBeenCalledTimes(1)
-    expect(run).toHaveBeenCalledWith('myFilePath')
-  })
-
-  it('should get file count', () => {
-    const localGet = jest.fn()
-    localGet.mockReturnValue({ count: 1 })
-    mockDatabase.prepare.mockReturnValue({ get: localGet })
-
-    const result = databaseService.getFileCount(mockDatabase, 'myFilePath')
-    expect(prepare).toHaveBeenCalledTimes(1)
-    expect(prepare).toHaveBeenCalledWith('SELECT COUNT(*) AS count FROM cache WHERE path = ?')
-
-    expect(localGet).toHaveBeenCalledTimes(1)
-    expect(localGet).toHaveBeenCalledWith('myFilePath')
-
-    expect(result).toEqual(1)
-  })
-
-  it('should get file count for a North connector', () => {
-    const localGet = jest.fn()
-    localGet.mockReturnValue({ count: 11 })
-    mockDatabase.prepare.mockReturnValue({ get: localGet })
-
-    const result = databaseService.getFileCountForNorthConnector(mockDatabase)
-    expect(prepare).toHaveBeenCalledTimes(1)
-    expect(prepare).toHaveBeenCalledWith('SELECT COUNT(*) AS count FROM cache')
-
-    expect(localGet).toHaveBeenCalledTimes(1)
-    expect(localGet).toHaveBeenCalledWith()
-
-    expect(result).toEqual(11)
-  })
-
   it('should upsert config', () => {
     databaseService.upsertConfig(mockDatabase, 'configEntry', 'value')
     expect(prepare).toHaveBeenCalledTimes(1)