diff --git a/app/api/entities/denormalize.ts b/app/api/entities/denormalize.ts index 35cbfc1e9a..838850b5c8 100644 --- a/app/api/entities/denormalize.ts +++ b/app/api/entities/denormalize.ts @@ -1,6 +1,6 @@ /* eslint-disable max-lines */ import { WithId } from 'api/odm'; -import translationsModel from 'api/i18n/translations'; +import translationsModel, { IndexedTranslations } from 'api/i18n/translations'; import { search } from 'api/search'; import templates from 'api/templates'; import dictionariesModel from 'api/thesauri/dictionariesModel'; @@ -263,8 +263,8 @@ const denormalizeThesauriLabelInMetadata = async ( const denormalizeSelectProperty = async ( property: PropertySchema, values: MetadataObjectSchema[], - thesauriByKey: Record, - translation: unknown + thesauriByKey?: Record, + translation?: unknown ) => { const thesaurus = thesauriByKey ? thesauriByKey[property.content!] @@ -382,7 +382,7 @@ const denormalizeProperty = async ( translation, allTemplates, }: { - thesauriByKey: Record; + thesauriByKey?: Record; translation: unknown; allTemplates: TemplateSchema[]; } @@ -407,20 +407,20 @@ const denormalizeProperty = async ( async function denormalizeMetadata( metadata: MetadataSchema, language: LanguageISO6391, - templateId: string, - thesauriByKey: Record + template: TemplateSchema, + preloadedData: { + allTemplates?: TemplateSchema[]; + translation?: IndexedTranslations; + thesauriByKey?: Record; + } = {} ) { - if (!metadata) { + if (!metadata || !template) { return metadata; } - const translation = (await translationsModel.get({ locale: language }))[0]; - const allTemplates = await templates.get(); - - const template = allTemplates.find(t => t._id.toString() === templateId); - if (!template) { - return metadata; - } + const allTemplates = preloadedData.allTemplates || (await templates.get()); + const translation = + preloadedData.translation || (await translationsModel.get({ locale: language }))[0]; const denormalizedProperties: { propertyName: string; @@ -432,7 +432,7 @@ async function denormalizeMetadata( template.properties?.find(p => p.name === propertyName), metadata[propertyName], language, - { thesauriByKey, translation, allTemplates } + { thesauriByKey: preloadedData.thesauriByKey, translation, allTemplates } ), })) ); diff --git a/app/api/entities/entities.js b/app/api/entities/entities.js index 0081a0b7fd..76272d95a9 100644 --- a/app/api/entities/entities.js +++ b/app/api/entities/entities.js @@ -75,20 +75,17 @@ async function updateEntity(entity, _template, unrestricted = false) { delete toSave.permissions; if (entity.metadata) { - toSave.metadata = await denormalizeMetadata( - entity.metadata, - d.language, - template._id.toString(), - thesauriByKey - ); + toSave.metadata = await denormalizeMetadata(entity.metadata, d.language, template, { + thesauriByKey, + }); } if (entity.suggestedMetadata) { toSave.suggestedMetadata = await denormalizeMetadata( entity.suggestedMetadata, entity.language, - template._id.toString(), - thesauriByKey + template, + { thesauriByKey } ); } @@ -124,8 +121,8 @@ async function updateEntity(entity, _template, unrestricted = false) { toSave[metadataParent] = await denormalizeMetadata( toSave[metadataParent], toSave.language, - template._id.toString(), - thesauriByKey + template, + { thesauriByKey } ); } }, Promise.resolve()); @@ -187,15 +184,15 @@ async function createEntity(doc, [currentLanguage, languages], sharedId, docTemp langDoc.metadata = await denormalizeMetadata( langDoc.metadata, langDoc.language, - langDoc.template.toString(), - thesauriByKey + docTemplate, + { thesauriByKey } ); langDoc.suggestedMetadata = await denormalizeMetadata( langDoc.suggestedMetadata, langDoc.language, - langDoc.template.toString(), - thesauriByKey + docTemplate, + { thesauriByKey } ); return model.save(langDoc); @@ -464,15 +461,11 @@ export default { docTemplate = defaultTemplate; } const entity = this.sanitize(doc, docTemplate); - entity.metadata = await denormalizeMetadata( - entity.metadata, - entity.language, - entity.template.toString() - ); + entity.metadata = await denormalizeMetadata(entity.metadata, entity.language, docTemplate); entity.suggestedMetadata = await denormalizeMetadata( entity.suggestedMetadata, entity.language, - entity.template.toString() + docTemplate ); return entity; }, diff --git a/docker-compose.yml b/docker-compose.yml index f7fdfbb557..44c9ce20dc 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -24,14 +24,18 @@ services: - mongodata:/data/db ports: - '27017:27017' + ulimits: + nofile: + soft: "65536" + hard: "65536" mongoreplicaset_start_script: image: "mongo:5.0.27" restart: "no" container_name: mongoreplicaset_start_script - depends_on: + depends_on: - mongo - entrypoint: [ "bash", "-c", "sleep 1 && mongo --host mongo:27017 --eval 'rs.initiate();cfg = rs.config(); cfg.members[0].host = \"localhost:27017\";rs.reconfig(cfg, {force:true})'"] + entrypoint: [ "bash", "-c", "sleep 1 && mongo --host mongo:27017 --eval 'rs.initiate();cfg = rs.config(); cfg.members[0].host = \"localhost:27017\";rs.reconfig(cfg, {force:true})'"] redis: image: 'redis:5.0.14' diff --git a/package.json b/package.json index 0cfa9796b0..f44ad987e2 100644 --- a/package.json +++ b/package.json @@ -31,6 +31,7 @@ "dev-worker": "node --no-experimental-fetch ./scripts/run.js ../app/worker.ts", "dev-queue": "node --no-experimental-fetch ./scripts/run.js ../app/queueWorker.ts", "new-files-healthcheck": "tsx ./scripts/scripts.v2/filesHealthCheck.ts", + "denormalize-all-entities": "tsx ./scripts/scripts.v2/denormalizeAllEntities.ts", "generateAutomaticTranslationConfig": "node --no-experimental-fetch ./scripts/run.js ../scripts/scripts.v2/generateAutomaticTranslationConfig.ts", "check-translations": "node --no-experimental-fetch ./scripts/checkTranslations.mjs", "update-translations-db": "node --no-experimental-fetch scripts/run.js ../scripts/updateTranslationsDB.js", diff --git a/scripts/scripts.v2/denormalizeAllEntities.ts b/scripts/scripts.v2/denormalizeAllEntities.ts new file mode 100644 index 0000000000..e9b0a9e7ea --- /dev/null +++ b/scripts/scripts.v2/denormalizeAllEntities.ts @@ -0,0 +1,142 @@ +import { config } from 'api/config'; +import { DB } from 'api/odm'; +import { tenants } from 'api/tenants'; + +import { denormalizeMetadata } from 'api/entities/denormalize'; +import entities from 'api/entities/entities'; +import entitiesModel from 'api/entities/entitiesModel'; +import translationsModel, { IndexedTranslations } from 'api/i18n/translations'; +import { permissionsContext } from 'api/permissions/permissionsContext'; +import { search } from 'api/search'; +import templates from 'api/templates'; +import { LanguageISO6391 } from 'shared/types/commonTypes'; +import { TemplateSchema } from 'shared/types/templateType'; +import { ThesaurusSchema } from 'shared/types/thesaurusType'; +import { inspect } from 'util'; + +const { tenant, allTenants } = require('yargs') + .option('tenant', { + alias: 't', + type: 'string', + describe: 'Tenant to use', + default: 'default', + }) + .option('allTenants', { + alias: 'a', + type: 'boolean', + describe: 'All tenants', + default: false, + }).argv; + +let dbAuth = {}; + +if (process.env.DBUSER) { + dbAuth = { + auth: { authSource: 'admin' }, + user: process.env.DBUSER, + pass: process.env.DBPASS, + }; +} + +async function handleTenant(tenantName: string) { + await tenants.run(async () => { + const start = process.hrtime(); + + permissionsContext.setCommandContext(); + + const templateRelatedThesauri: { [templateId: string]: Record } = {}; + const indexedTemplates = await ( + await templates.get() + ).reduce>(async (prev, t) => { + let memo = await prev; + memo[t._id.toString()] = t; + templateRelatedThesauri[t._id.toString()] = await templates.getRelatedThesauri(t); + return memo; + }, Promise.resolve({})); + + const translationsByLanguage = ( + await translationsModel.get() + ).reduce<{ [language: string]: IndexedTranslations }>((memo, t) => { + if (!t.locale) { + throw new Error(`translation ${t._id} has no locale !`); + } + memo[t.locale] = t; + return memo; + }, {}); + + const allTemplates = Object.values(indexedTemplates); + const entityIds = await entities.getUnrestricted({}, '_id', {}); + + let entitiesProcessed = 0; + const errors: any[] = []; + + await entityIds.reduce(async (prev, entity) => { + await prev; + const _id = entity._id; + const [entityToSave] = await entities.getUnrestricted({ _id }); + try { + console.log( + JSON.stringify({ updating: `${entityToSave.title} | ${entityToSave.sharedId}` }) + ); + + if (!entityToSave.template) { + throw new Error(`Entity ${entityToSave._id} has no template !`); + } + + if (!entityToSave.language) { + throw new Error(`Entity ${entityToSave._id} has no language !`); + } + + entityToSave.metadata = await denormalizeMetadata( + entityToSave.metadata || {}, + entityToSave.language as LanguageISO6391, + indexedTemplates[entityToSave.template.toString()], + { + thesauriByKey: templateRelatedThesauri[entityToSave.template.toString()], + allTemplates, + translation: translationsByLanguage[entityToSave.language], + } + ); + await entitiesModel.save(entityToSave); + entitiesProcessed += 1; + } catch (e) { + const error = { tenant, sharedId: _id, error: inspect(e) }; + errors.push(error); + } + return Promise.resolve(); + }, Promise.resolve()); + + await search.indexEntities({}); + + const [seconds, nanoseconds] = process.hrtime(start); + const elapsedTime = seconds + nanoseconds / 1e9; + + console.log( + inspect({ + logType: 'summary', + tenant: tenantName, + entitiesFetched: entityIds.length, + correctlyProcessed: entitiesProcessed, + notProcessed: errors.length, + elapsedTime: `${elapsedTime.toFixed(3)} s`, + errors: errors, + }) + ); + }, tenantName); +} + +(async function run() { + await DB.connect(config.DBHOST, dbAuth); + await tenants.setupTenants(); + + if (!allTenants) { + await handleTenant(tenant); + } else { + await Object.keys(tenants.tenants).reduce(async (prev, tenantName) => { + await prev; + await handleTenant(tenantName); + }, Promise.resolve()); + } + await tenants.model?.closeChangeStream(); + await DB.disconnect(); +})();