From f0c81172b627eb7f14f6b46cbe2436c1291b0017 Mon Sep 17 00:00:00 2001
From: burgerni10 <nicolas.burger@optimistik.fr>
Date: Fri, 21 Oct 2022 21:40:23 +0200
Subject: [PATCH] fix(north): fix north retrieve file when file being written

---
 src/engine/cache/file-cache.js    | 27 ++++++++++++++++++++++-----
 src/north/north-connector.js      | 13 +------------
 src/north/north-connector.spec.js | 17 +----------------
 3 files changed, 24 insertions(+), 33 deletions(-)

diff --git a/src/engine/cache/file-cache.js b/src/engine/cache/file-cache.js
index f532eaf93a..734b85ab8e 100644
--- a/src/engine/cache/file-cache.js
+++ b/src/engine/cache/file-cache.js
@@ -111,17 +111,34 @@ class FileCache extends BaseCache {
    * @returns {Promise<{path: string, timestamp: number}|null>} - The file to send
    */
   async retrieveFileFromCache() {
-    const fileNames = await fs.readdir(this.fileFolder)
+    let fileNames = []
+    try {
+      fileNames = await fs.readdir(this.fileFolder)
+    } catch (error) {
+      this.logger.error(error)
+    }
 
     if (fileNames.length === 0) {
       return null
     }
 
     const sortedFiles = fileNames
-      .map(async (fileName) => ({
-        path: path.resolve(this.fileFolder, fileName),
-        timestamp: (await fs.stat(path.resolve(this.fileFolder, fileName))).mtime.getTime(),
-      }))
+      .map(async (fileName) => {
+        // Retrieve file state to retrieve the oldest one from the cache
+        let fileState = null
+        try {
+          // Error triggered when a file is being written (from a South)
+          fileState = await fs.stat(path.resolve(this.fileFolder, fileName))
+        } catch (error) {
+          this.logger.error(error)
+        }
+        return {
+          path: path.resolve(this.fileFolder, fileName),
+          timestamp: fileState ? fileState.mtime.getTime() : null,
+        }
+      })
+    // filter out files that are not completely written
+      .filter((file) => file.timestamp !== null)
       .sort((a, b) => a.timestamp - b.timestamp)
 
     return sortedFiles[0]
diff --git a/src/north/north-connector.js b/src/north/north-connector.js
index cab21223d0..73c14c68d9 100644
--- a/src/north/north-connector.js
+++ b/src/north/north-connector.js
@@ -1,4 +1,3 @@
-const fs = require('node:fs/promises')
 const path = require('node:path')
 
 const EncryptionService = require('../service/encryption.service')
@@ -245,22 +244,12 @@ class NorthConnector {
 
     const fileToSend = await this.fileCache.retrieveFileFromCache()
     if (!fileToSend) {
-      this.logger.trace('No file to send in the cache database.')
+      this.logger.trace('No file to send in the cache folder.')
       this.resetFilesTimeout(this.caching.sendInterval)
       return
     }
     this.logger.trace(`File to send: "${fileToSend.path}".`)
 
-    try {
-      await fs.stat(fileToSend.path)
-    } catch (error) {
-      // File in cache does not exist on filesystem
-      await this.fileCache.removeFileFromCache(fileToSend.path, false)
-      this.logger.error(`File "${fileToSend.path}" not found! The file has been removed from the cache.`)
-      this.resetFilesTimeout(this.caching.sendInterval)
-      return
-    }
-
     this.sendingFilesInProgress = true
     this.resendFilesImmediately = false
 
diff --git a/src/north/north-connector.spec.js b/src/north/north-connector.spec.js
index b84b8d745c..126c58bf69 100644
--- a/src/north/north-connector.spec.js
+++ b/src/north/north-connector.spec.js
@@ -1,4 +1,3 @@
-const fs = require('node:fs/promises')
 const NorthConnector = require('./north-connector')
 
 const { defaultConfig: config } = require('../../tests/test-config')
@@ -234,24 +233,10 @@ describe('NorthConnector', () => {
     north.resetFilesTimeout = jest.fn()
     await north.retrieveFromCacheAndSendFile()
 
-    expect(north.logger.trace).toHaveBeenCalledWith('No file to send in the cache database.')
+    expect(north.logger.trace).toHaveBeenCalledWith('No file to send in the cache folder.')
     expect(north.resetFilesTimeout).toHaveBeenCalledWith(settings.caching.sendInterval)
   })
 
-  it('should not send files if it does not exist', async () => {
-    const fileToSend = { path: 'myFile' }
-    north.fileCache.retrieveFileFromCache = jest.fn(() => fileToSend)
-    north.fileCache.removeFileFromCache = jest.fn()
-    fs.stat = jest.fn().mockImplementationOnce(() => {
-      throw new Error('file does not exist')
-    })
-
-    await north.retrieveFromCacheAndSendFile()
-
-    expect(north.logger.error).toHaveBeenCalledWith(`File "${fileToSend.path}" not found! The file has been removed from the cache.`)
-    expect(north.fileCache.removeFileFromCache).toHaveBeenCalledWith(fileToSend.path, false)
-  })
-
   it('should retry to send files if it fails', async () => {
     clearTimeout(north.valuesTimeout)
     clearTimeout(north.filesTimeout)