Skip to content

Commit

Permalink
fix(sqlite): add tests and remove async
Browse files Browse the repository at this point in the history
  • Loading branch information
burgerni10 authored and Nicolas Burger committed Aug 25, 2022
1 parent ed7c258 commit fcc924f
Show file tree
Hide file tree
Showing 8 changed files with 438 additions and 246 deletions.
2 changes: 1 addition & 1 deletion src/HistoryQuery/HistoryQueryEngine.class.js
View file
Original file line number Diff line number Diff line change
Expand Up @@ -203,7 +203,7 @@ class HistoryQueryEngine extends BaseEngine {
const databasePath = `${folder}/${historyQueryConfig.southId}.db`
try {
await fs.stat(databasePath)
const entries = await databaseService.getHistoryQuerySouthData(databasePath)
const entries = databaseService.getHistoryQuerySouthData(databasePath)
data.south = entries.map((entry) => ({
scanMode: entry.name.replace('lastCompletedAt-', ''),
lastCompletedDate: entry.value,
Expand Down
43 changes: 21 additions & 22 deletions src/engine/Cache.class.js
View file
Original file line number Diff line number Diff line change
Expand Up @@ -67,8 +67,8 @@ class Cache {
// only initialize the db if the api can handle values
if (api.canHandleValues) {
this.logger.debug(`use db: ${this.cacheFolder}/${api.id}.db for ${api.name}`)
api.database = await databaseService.createValuesDatabase(`${this.cacheFolder}/${api.id}.db`, {})
this.logger.debug(`db count: ${await databaseService.getCount(api.database)}`)
api.database = databaseService.createValuesDatabase(`${this.cacheFolder}/${api.id}.db`, {})
this.logger.debug(`db count: ${databaseService.getCount(api.database)}`)
}
this.apis[api.id] = api
if (api.config?.sendInterval) {
Expand Down Expand Up @@ -130,12 +130,12 @@ class Cache {

this.logger.debug(`Cache initialized with cacheFolder:${this.archiveFolder} and archiveFolder: ${this.archiveFolder}`)
this.logger.debug(`Use file dbs: ${this.cacheFolder}/fileCache.db and ${this.filesErrorDatabasePath}`)
this.filesDatabase = await databaseService.createFilesDatabase(`${this.cacheFolder}/fileCache.db`)
this.filesErrorDatabase = await databaseService.createFilesDatabase(this.filesErrorDatabasePath)
this.valuesErrorDatabase = await databaseService.createValueErrorsDatabase(this.valuesErrorDatabasePath)
this.logger.debug(`Files db count: ${await databaseService.getCount(this.filesDatabase)}`)
this.logger.debug(`Files error db count: ${await databaseService.getCount(this.filesErrorDatabase)}`)
this.logger.debug(`Values error db count: ${await databaseService.getCount(this.valuesErrorDatabase)}`)
this.filesDatabase = databaseService.createFilesDatabase(`${this.cacheFolder}/fileCache.db`)
this.filesErrorDatabase = databaseService.createFilesDatabase(this.filesErrorDatabasePath)
this.valuesErrorDatabase = databaseService.createValueErrorsDatabase(this.valuesErrorDatabasePath)
this.logger.debug(`Files db count: ${databaseService.getCount(this.filesDatabase)}`)
this.logger.debug(`Files error db count: ${databaseService.getCount(this.filesErrorDatabase)}`)
this.logger.debug(`Values error db count: ${databaseService.getCount(this.valuesErrorDatabase)}`)
}

/**
Expand Down Expand Up @@ -171,7 +171,7 @@ class Cache {

// if the group size is over the groupCount => we immediately send the cache
// to the North even if the timeout is not finished.
const count = await databaseService.getCount(database)
const count = databaseService.getCount(database)
if (count >= config.groupCount) {
this.logger.trace(`groupCount reached: ${count}>=${config.groupCount}`)
return api
Expand Down Expand Up @@ -225,7 +225,7 @@ class Cache {

// Cache file
this.logger.debug(`cacheFileForApi() ${cachePath} for api: ${applicationName}`)
await databaseService.saveFile(this.filesDatabase, timestamp, applicationId, cachePath)
databaseService.saveFile(this.filesDatabase, timestamp, applicationId, cachePath)
return api
}
this.logger.trace(`datasource "${this.engine.activeProtocols[id]?.dataSource.name || id}" is not subscribed to application "${applicationName}"`)
Expand Down Expand Up @@ -341,7 +341,7 @@ class Cache {
const { id, name, database, config } = application

try {
const values = await databaseService.getValuesToSend(database, config.maxSendCount)
const values = databaseService.getValuesToSend(database, config.maxSendCount)
let removed

if (values.length) {
Expand All @@ -352,10 +352,10 @@ class Cache {
if (successCountStatus === ApiHandler.STATUS.LOGIC_ERROR) {
// Add errored values into error table
this.logger.trace(`Cache:addErroredValues, add ${values.length} values to error database for ${name}`)
await databaseService.saveErroredValues(this.valuesErrorDatabase, id, values)
databaseService.saveErroredValues(this.valuesErrorDatabase, id, values)

// Remove them from the cache table
removed = await databaseService.removeSentValues(database, values)
removed = databaseService.removeSentValues(database, values)
this.logger.trace(`Cache:removeSentValues, removed: ${removed} AppId: ${application.name}`)
if (removed !== values.length) {
this.logger.debug(`Cache for ${name} can't be deleted: ${removed}/${values.length}`)
Expand All @@ -364,7 +364,7 @@ class Cache {
// If some values were successfully sent
if (successCountStatus > 0) {
const valuesSent = values.slice(0, successCountStatus)
removed = await databaseService.removeSentValues(database, valuesSent)
removed = databaseService.removeSentValues(database, valuesSent)
this.logger.trace(`Cache:removeSentValues, removed: ${removed} AppId: ${application.name}`)
if (removed !== valuesSent.length) {
this.logger.debug(`Cache for ${name} can't be deleted: ${removed}/${valuesSent.length}`)
Expand All @@ -390,7 +390,7 @@ class Cache {
this.logger.trace(`sendCallbackForFiles() for ${name}`)

try {
const fileToSend = await databaseService.getFileToSend(this.filesDatabase, id)
const fileToSend = databaseService.getFileToSend(this.filesDatabase, id)

if (!fileToSend) {
this.logger.trace('sendCallbackForFiles(): no file to send')
Expand All @@ -403,7 +403,7 @@ class Cache {
await fs.stat(fileToSend.path)
} catch (error) {
// file in cache does not exist on filesystem
await databaseService.deleteSentFile(this.filesDatabase, id, fileToSend.path)
databaseService.deleteSentFile(this.filesDatabase, id, fileToSend.path)
this.logger.error(new Error(`${fileToSend.path} not found! Removing it from db.`))
return ApiHandler.STATUS.SUCCESS
}
Expand All @@ -412,15 +412,15 @@ class Cache {
switch (status) {
case ApiHandler.STATUS.SUCCESS:
this.logger.trace(`sendCallbackForFiles(${fileToSend.path}) deleteSentFile for ${name}`)
await databaseService.deleteSentFile(this.filesDatabase, id, fileToSend.path)
databaseService.deleteSentFile(this.filesDatabase, id, fileToSend.path)
await this.handleSentFile(fileToSend.path)
break
case ApiHandler.STATUS.LOGIC_ERROR:
this.logger.error(`sendCallbackForFiles(${fileToSend.path}) move to error database for ${name}`)
await databaseService.saveFile(this.filesErrorDatabase, fileToSend.timestamp, id, fileToSend.path)
databaseService.saveFile(this.filesErrorDatabase, fileToSend.timestamp, id, fileToSend.path)

this.logger.trace(`sendCallbackForFiles(${fileToSend.path}) deleteSentFile for ${name}`)
await databaseService.deleteSentFile(this.filesDatabase, id, fileToSend.path)
databaseService.deleteSentFile(this.filesDatabase, id, fileToSend.path)
break
default:
break
Expand All @@ -439,7 +439,7 @@ class Cache {
*/
async handleSentFile(filePath) {
this.logger.trace(`handleSentFile(${filePath})`)
const count = await databaseService.getFileCount(this.filesDatabase, filePath)
const count = databaseService.getFileCount(this.filesDatabase, filePath)
if (count === 0) {
if (this.archiveMode) {
const archivedFilename = path.basename(filePath)
Expand Down Expand Up @@ -543,8 +543,7 @@ class Cache {
// Get file APIs stats
const fileApis = Object.values(this.apis).filter((api) => api.canHandleFiles)
const filesTotalCounts = fileApis.map((api) => this.cacheStats[api.name])
const filesCacheSizeActions = fileApis.map((api) => databaseService.getFileCountForApi(this.filesDatabase, api.name))
const filesCacheSizes = await Promise.all(filesCacheSizeActions)
const filesCacheSizes = fileApis.map((api) => databaseService.getFileCountForNorthConnector(this.filesDatabase, api.name))
const fileApisStats = this.generateApiCacheStat(fileApis, filesTotalCounts, filesCacheSizes, 'files')

// Merge results
Expand Down
20 changes: 10 additions & 10 deletions src/migration/migrationRules.js
View file
Original file line number Diff line number Diff line change
Expand Up @@ -1108,25 +1108,25 @@ module.exports = {
if (dataSource.protocol === 'SQL') {
logger.info(`Update lastCompletedAt key for ${dataSource.name}`)
const databasePath = `${config.engine.caching.cacheFolder}/${dataSource.id}.db`
const database = await databaseService.createConfigDatabase(databasePath)
const lastCompletedAt = await databaseService.getConfig(database, 'lastCompletedAt')
await databaseService.upsertConfig(database, `lastCompletedAt-${dataSource.scanMode}`, lastCompletedAt)
const database = databaseService.createConfigDatabase(databasePath)
const lastCompletedAt = databaseService.getConfig(database, 'lastCompletedAt')
databaseService.upsertConfig(database, `lastCompletedAt-${dataSource.scanMode}`, lastCompletedAt)
}

if (['OPCUA_HA', 'OPCHDA'].includes(dataSource.protocol)) {
const databasePath = `${config.engine.caching.cacheFolder}/${dataSource.id}.db`
const database = await databaseService.createConfigDatabase(databasePath)
const database = databaseService.createConfigDatabase(databasePath)
if (!dataSource[dataSource.protocol].scanGroups) {
dataSource[dataSource.protocol].scanGroups = []
}
const scanModes = dataSource[dataSource.protocol].scanGroups.map((scanGroup) => scanGroup.scanMode)
// eslint-disable-next-line no-restricted-syntax
for (const scanMode of scanModes) {
logger.info(`Update lastCompletedAt-${scanMode} value for ${dataSource.name}`)
const lastCompletedAtString = await databaseService.getConfig(database, `lastCompletedAt-${scanMode}`)
const lastCompletedAtString = databaseService.getConfig(database, `lastCompletedAt-${scanMode}`)
if (lastCompletedAtString) {
const lastCompletedAt = new Date(parseInt(lastCompletedAtString, 10))
await databaseService.upsertConfig(database, `lastCompletedAt-${scanMode}`, lastCompletedAt.toISOString())
databaseService.upsertConfig(database, `lastCompletedAt-${scanMode}`, lastCompletedAt.toISOString())
}
}
}
Expand Down Expand Up @@ -1203,7 +1203,7 @@ module.exports = {
// eslint-disable-next-line max-len
logger.info(`Migration of values database ${cachePath}/${application.id}.db: Renaming column name "data_source_id" into "data_source" for application ${application.name}`)
try {
await databaseMigrationService.changeColumnName(
databaseMigrationService.changeColumnName(
newApplicationPath,
'data_source_id',
'data_source',
Expand All @@ -1216,7 +1216,7 @@ module.exports = {
// eslint-disable-next-line max-len
logger.info(`Migration of file database ${fileCacheDbPath}: Changing application value from ${application.name} to ${application.id}`)
try {
await databaseMigrationService.changeColumnValue(
databaseMigrationService.changeColumnValue(
fileCacheDbPath,
'application',
application.name,
Expand All @@ -1231,7 +1231,7 @@ module.exports = {
// eslint-disable-next-line max-len
logger.info(`Migration of file error database ${fileCacheErrorDbPath}: Changing application value from ${application.name} to ${application.id}`)
try {
await databaseMigrationService.changeColumnValue(
databaseMigrationService.changeColumnValue(
fileCacheErrorDbPath,
'application',
application.name,
Expand All @@ -1246,7 +1246,7 @@ module.exports = {
// eslint-disable-next-line max-len
logger.info(`Migration of value error database ${valueCacheErrorDbPath}: Changing application value from ${application.name} to ${application.id}`)
try {
await databaseMigrationService.changeColumnValue(
databaseMigrationService.changeColumnValue(
valueCacheErrorDbPath,
'application',
application.name,
Expand Down
4 changes: 2 additions & 2 deletions src/server/controllers/logController.js
View file
Original file line number Diff line number Diff line change
Expand Up @@ -10,7 +10,7 @@ const databaseService = require('../../services/database.service')
* @param {function} ctx.ok - The context response
* @return {void}
*/
const getLogs = async (ctx) => {
const getLogs = (ctx) => {
const { engineConfig } = ctx.app.engine.configService.getConfig()
const databasePath = engineConfig.logParameters.sqliteLog.fileName
const now = Date.now()
Expand All @@ -19,7 +19,7 @@ const getLogs = async (ctx) => {
const toDate = ctx.query.toDate || new Date(now).toISOString()
const verbosity = ctx.query.verbosity?.replace(/[[\]]/g, '').split(',') || 'info'

const logs = await databaseService.getLogs(databasePath, fromDate, toDate, verbosity)
const logs = databaseService.getLogs(databasePath, fromDate, toDate, verbosity)
ctx.ok(logs)
}

Expand Down
Loading

0 comments on commit fcc924f

Please sign in to comment.