From f55dd00c97b0f95e4e4733a94479fffabad693d6 Mon Sep 17 00:00:00 2001 From: Sylvain Bellone Date: Fri, 30 Aug 2024 17:15:30 +0200 Subject: [PATCH 1/2] fix: write errors happening while reading delta XML in the jobReport --- .../cartridge/scripts/algolia/helper/jobHelper.js | 13 +++++++++---- .../algolia/steps/algoliaProductDeltaIndex.js | 14 ++++++-------- 2 files changed, 15 insertions(+), 12 deletions(-) diff --git a/cartridges/int_algolia/cartridge/scripts/algolia/helper/jobHelper.js b/cartridges/int_algolia/cartridge/scripts/algolia/helper/jobHelper.js index 011ed3a2..2a743605 100644 --- a/cartridges/int_algolia/cartridge/scripts/algolia/helper/jobHelper.js +++ b/cartridges/int_algolia/cartridge/scripts/algolia/helper/jobHelper.js @@ -496,17 +496,16 @@ function updateCPObjectFromXML(xmlFile, changedProducts, resourceType) { var XMLStreamReader = require('dw/io/XMLStreamReader'); var XMLStreamConstants = require('dw/io/XMLStreamConstants'); var FileReader = require('dw/io/FileReader'); - var catalogID; var resultObj = { nrProductsRead: 0, - success: false + success: false, + errorMessage: '' }; try { if (xmlFile.exists()) { var fileReader = new FileReader(xmlFile); var xmlStreamReader = new XMLStreamReader(fileReader); - var success = false; switch (resourceType) { case 'catalog': @@ -575,9 +574,15 @@ function updateCPObjectFromXML(xmlFile, changedProducts, resourceType) { } xmlStreamReader.close(); - }; + } } catch (error) { + algoliaLogger.error('Error while reading from file: ' + xmlFile.getFullPath()); + algoliaLogger.error(error); resultObj.success = false; + resultObj.errorMessage = + 'Error while reading from file: ' + xmlFile.getFullPath() + '\n' + + error.message + '\n' + + 'If your product attributes have special characters such as emojis, you must enable "Filter Invalid XML Characters during Export" in Administration > Global Preferences > Import & Export'; return resultObj; } diff --git a/cartridges/int_algolia/cartridge/scripts/algolia/steps/algoliaProductDeltaIndex.js b/cartridges/int_algolia/cartridge/scripts/algolia/steps/algoliaProductDeltaIndex.js index c90c189b..309450b5 100644 --- a/cartridges/int_algolia/cartridge/scripts/algolia/steps/algoliaProductDeltaIndex.js +++ b/cartridges/int_algolia/cartridge/scripts/algolia/steps/algoliaProductDeltaIndex.js @@ -268,13 +268,9 @@ exports.beforeStep = function(parameters, stepExecution) { if (result.success) { jobReport.processedItems += result.nrProductsRead; } else { - // abort if error reading from any of the delta export zips - let errorMessage = 'Error reading from file: ' + catalogFile.getFullPath(); - jobHelper.logError(errorMessage); - + // Mark the job in error if an error occurred while reading from any of the delta export zips jobReport.error = true; - jobReport.errorMessage = errorMessage; - jobReport.writeToCustomObject(); + jobReport.errorMessage = result.errorMessage; } }); } @@ -287,8 +283,10 @@ exports.beforeStep = function(parameters, stepExecution) { // cleanup - removing "_processing" dir fileHelper.removeFolderRecursively(l1_processingDir); - changedProductsIterator = new CPObjectIterator(changedProducts); - logger.info(jobReport.processedItems + ' updated products found. Starting indexing...'); + if (!jobReport.error) { + changedProductsIterator = new CPObjectIterator(changedProducts); + logger.info(jobReport.processedItems + ' updated products found. Starting indexing...'); + } } /** From 73de4df4276d594a2ee7917dc7ef2b6c0e2fefb7 Mon Sep 17 00:00:00 2001 From: Sylvain Bellone Date: Fri, 30 Aug 2024 18:39:07 +0200 Subject: [PATCH 2/2] move delta export files to a _failed dir in case of error in the beforeStep --- .../scripts/algolia/helper/fileHelper.js | 22 +++++++++++++++++++ .../algolia/steps/algoliaProductDeltaIndex.js | 18 +++++++-------- 2 files changed, 30 insertions(+), 10 deletions(-) diff --git a/cartridges/int_algolia/cartridge/scripts/algolia/helper/fileHelper.js b/cartridges/int_algolia/cartridge/scripts/algolia/helper/fileHelper.js index b2123e55..d053b311 100644 --- a/cartridges/int_algolia/cartridge/scripts/algolia/helper/fileHelper.js +++ b/cartridges/int_algolia/cartridge/scripts/algolia/helper/fileHelper.js @@ -119,6 +119,27 @@ function moveFile(sourceFile, targetFile) { return success; } +/** + * Moves files generated by the delta export to another location + * @param {string[]} deltaExportZipList List of delta export zip names + * @param {dw.io.File} sourceDir Directory to move from + * @param {dw.io.File} destDir Target directory + */ +function moveDeltaExportFiles(deltaExportZipList, sourceDir, destDir) { + if (empty(deltaExportZipList)) { + return; + } + deltaExportZipList.forEach(function (filename) { + let currentZipFile = new dw.io.File(sourceDir, filename); // 000001.zip, 000002.zip, etc. + let targetZipFile = new dw.io.File(destDir, currentZipFile.getName()); + currentZipFile.renameTo(targetZipFile); + + let currentMetaFile = new dw.io.File(sourceDir, filename.replace('.zip', '.meta')); // each .zip has a corresponding .meta file + let targetMetaFile = new dw.io.File(destDir, currentMetaFile.getName()); + currentMetaFile.renameTo(targetMetaFile) + }); +} + module.exports = { getFirstChildFolder: getFirstChildFolder, getDeltaExportZipList: getDeltaExportZipList, @@ -126,4 +147,5 @@ module.exports = { getAllXMLFilesInFolder: getAllXMLFilesInFolder, removeFolderRecursively: removeFolderRecursively, moveFile: moveFile, + moveDeltaExportFiles: moveDeltaExportFiles, } diff --git a/cartridges/int_algolia/cartridge/scripts/algolia/steps/algoliaProductDeltaIndex.js b/cartridges/int_algolia/cartridge/scripts/algolia/steps/algoliaProductDeltaIndex.js index 309450b5..fe3fd0e5 100644 --- a/cartridges/int_algolia/cartridge/scripts/algolia/steps/algoliaProductDeltaIndex.js +++ b/cartridges/int_algolia/cartridge/scripts/algolia/steps/algoliaProductDeltaIndex.js @@ -17,7 +17,7 @@ var fileHelper, jobHelper, modelHelper, reindexHelper, sendHelper; // logging-related variables and constants var jobReport; -var l0_deltaExportDir, l1_processingDir, l1_completedDir; +var l0_deltaExportDir, l1_processingDir, l1_completedDir, l1_failedDir; var changedProducts = [], changedProductsIterator; var deltaExportZips, siteLocales, attributesToSend; var masterAttributes = [], variantAttributes = []; @@ -233,6 +233,9 @@ exports.beforeStep = function(parameters, stepExecution) { l1_completedDir = new File(l0_deltaExportDir, '_completed'); l1_completedDir.mkdir(); // creating "_completed" folder -- does no harm if it already exists + l1_failedDir = new File(l0_deltaExportDir, '_failed'); + l1_failedDir.mkdir(); + // process each export zip one by one deltaExportZips.forEach(function(filename) { logger.info('Processing ' + filename + '...'); @@ -286,6 +289,9 @@ exports.beforeStep = function(parameters, stepExecution) { if (!jobReport.error) { changedProductsIterator = new CPObjectIterator(changedProducts); logger.info(jobReport.processedItems + ' updated products found. Starting indexing...'); + } else { + logger.info('Moving the Delta export files to the "' + l1_failedDir.getName() + '" directory...') + fileHelper.moveDeltaExportFiles(deltaExportZips, l0_deltaExportDir, l1_failedDir); } } @@ -465,15 +471,7 @@ exports.afterStep = function(success, parameters, stepExecution) { // cleanup: after the products have successfully been sent, move the delta zips from which the productIDs have successfully been extracted and the corresponding products sent to "_completed" if (!empty(deltaExportZips)) { logger.info('Moving the Delta export files to the "_completed" directory...') - deltaExportZips.forEach(function (filename) { - let currentZipFile = new File(l0_deltaExportDir, filename); // 000001.zip, 000002.zip, etc. - let targetZipFile = new File(l1_completedDir, currentZipFile.getName()); - fileHelper.moveFile(currentZipFile, targetZipFile); - - let currentMetaFile = new File(l0_deltaExportDir, filename.replace('.zip', '.meta')); // each .zip has a corresponding .meta file as well, we'll need to delete these later - let targetMetaFile = new File(l1_completedDir, currentMetaFile.getName()); - fileHelper.moveFile(currentMetaFile, targetMetaFile); - }); + fileHelper.moveDeltaExportFiles(deltaExportZips, l0_deltaExportDir, l1_completedDir); } } else { jobReport.error = true;