fix(north): fix north retrieve file when file being written

src/engine/cache/file-cache.js

@@ -111,17 +111,34 @@ class FileCache extends BaseCache {
    * @returns {Promise<{path: string, timestamp: number}|null>} - The file to send
    */
   async retrieveFileFromCache() {
-    const fileNames = await fs.readdir(this.fileFolder)
+    let fileNames = []
+    try {
+      fileNames = await fs.readdir(this.fileFolder)
+    } catch (error) {
+      this.logger.error(error)
+    }
 
     if (fileNames.length === 0) {
       return null
     }
 
     const sortedFiles = fileNames
-      .map(async (fileName) => ({
-        path: path.resolve(this.fileFolder, fileName),
-        timestamp: (await fs.stat(path.resolve(this.fileFolder, fileName))).mtime.getTime(),
-      }))
+      .map(async (fileName) => {
+        // Retrieve file state to retrieve the oldest one from the cache
+        let fileState = null
+        try {
+          // Error triggered when a file is being written (from a South)
+          fileState = await fs.stat(path.resolve(this.fileFolder, fileName))
+        } catch (error) {
+          this.logger.error(error)
+        }
+        return {
+          path: path.resolve(this.fileFolder, fileName),
+          timestamp: fileState ? fileState.mtime.getTime() : null,
+        }
+      })
+    // filter out files that are not completely written
+      .filter((file) => file.timestamp !== null)
       .sort((a, b) => a.timestamp - b.timestamp)
 
     return sortedFiles[0]

src/north/north-connector.js

@@ -1,4 +1,3 @@
-const fs = require('node:fs/promises')
 const path = require('node:path')
 
 const EncryptionService = require('../service/encryption.service')
@@ -245,22 +244,12 @@ class NorthConnector {
 
     const fileToSend = await this.fileCache.retrieveFileFromCache()
     if (!fileToSend) {
-      this.logger.trace('No file to send in the cache database.')
+      this.logger.trace('No file to send in the cache folder.')
       this.resetFilesTimeout(this.caching.sendInterval)
       return
     }
     this.logger.trace(`File to send: "${fileToSend.path}".`)
 
-    try {
-      await fs.stat(fileToSend.path)
-    } catch (error) {
-      // File in cache does not exist on filesystem
-      await this.fileCache.removeFileFromCache(fileToSend.path, false)
-      this.logger.error(`File "${fileToSend.path}" not found! The file has been removed from the cache.`)
-      this.resetFilesTimeout(this.caching.sendInterval)
-      return
-    }
-
     this.sendingFilesInProgress = true
     this.resendFilesImmediately = false
 

src/north/north-connector.spec.js

@@ -1,4 +1,3 @@
-const fs = require('node:fs/promises')
 const NorthConnector = require('./north-connector')
 
 const { defaultConfig: config } = require('../../tests/test-config')
@@ -234,24 +233,10 @@ describe('NorthConnector', () => {
     north.resetFilesTimeout = jest.fn()
     await north.retrieveFromCacheAndSendFile()
 
-    expect(north.logger.trace).toHaveBeenCalledWith('No file to send in the cache database.')
+    expect(north.logger.trace).toHaveBeenCalledWith('No file to send in the cache folder.')
     expect(north.resetFilesTimeout).toHaveBeenCalledWith(settings.caching.sendInterval)
   })
 
-  it('should not send files if it does not exist', async () => {
-    const fileToSend = { path: 'myFile' }
-    north.fileCache.retrieveFileFromCache = jest.fn(() => fileToSend)
-    north.fileCache.removeFileFromCache = jest.fn()
-    fs.stat = jest.fn().mockImplementationOnce(() => {
-      throw new Error('file does not exist')
-    })
-
-    await north.retrieveFromCacheAndSendFile()
-
-    expect(north.logger.error).toHaveBeenCalledWith(`File "${fileToSend.path}" not found! The file has been removed from the cache.`)
-    expect(north.fileCache.removeFileFromCache).toHaveBeenCalledWith(fileToSend.path, false)
-  })
-
   it('should retry to send files if it fails', async () => {
     clearTimeout(north.valuesTimeout)
     clearTimeout(north.filesTimeout)