From 84530d5ae1b6da08da1991ed3fbd8ac236e5eb6d Mon Sep 17 00:00:00 2001 From: Ewan Cahen Date: Wed, 31 Aug 2022 16:31:52 +0200 Subject: [PATCH 1/3] feat: create a fake data generation script --- .gitignore | 5 +- data-generator/.dockerignore | 6 + data-generator/Dockerfile | 11 + data-generator/main.js | 529 +++++++++++++++++++++++ data-generator/package-lock.json | 455 +++++++++++++++++++ data-generator/package-lock.json.license | 4 + data-generator/package.json | 17 + data-generator/package.json.license | 4 + docker-compose.yml | 14 + 9 files changed, 1043 insertions(+), 2 deletions(-) create mode 100644 data-generator/.dockerignore create mode 100644 data-generator/Dockerfile create mode 100644 data-generator/main.js create mode 100644 data-generator/package-lock.json create mode 100644 data-generator/package-lock.json.license create mode 100644 data-generator/package.json create mode 100644 data-generator/package.json.license diff --git a/.gitignore b/.gitignore index bd2456716..5188c64f6 100644 --- a/.gitignore +++ b/.gitignore @@ -1,7 +1,7 @@ # SPDX-FileCopyrightText: 2021 - 2022 Dusan Mijatovic (dv4all) +# SPDX-FileCopyrightText: 2021 - 2022 Ewan Cahen (Netherlands eScience Center) +# SPDX-FileCopyrightText: 2021 - 2022 Netherlands eScience Center # SPDX-FileCopyrightText: 2021 - 2022 dv4all -# SPDX-FileCopyrightText: 2021 Ewan Cahen (Netherlands eScience Center) -# SPDX-FileCopyrightText: 2021 Netherlands eScience Center # # SPDX-License-Identifier: Apache-2.0 @@ -12,6 +12,7 @@ **.iml # .vscode +**/launch.json node_modules diff --git a/data-generator/.dockerignore b/data-generator/.dockerignore new file mode 100644 index 000000000..45c1a9fb9 --- /dev/null +++ b/data-generator/.dockerignore @@ -0,0 +1,6 @@ +# SPDX-FileCopyrightText: 2022 Ewan Cahen (Netherlands eScience Center) +# SPDX-FileCopyrightText: 2022 Netherlands eScience Center +# +# SPDX-License-Identifier: Apache-2.0 + +node_modules diff --git a/data-generator/Dockerfile b/data-generator/Dockerfile new file mode 100644 index 000000000..35081798c --- /dev/null +++ b/data-generator/Dockerfile @@ -0,0 +1,11 @@ +# SPDX-FileCopyrightText: 2022 Ewan Cahen (Netherlands eScience Center) +# SPDX-FileCopyrightText: 2022 Netherlands eScience Center +# +# SPDX-License-Identifier: Apache-2.0 + +FROM node:18.8.0-bullseye-slim +WORKDIR /usr/app +COPY ./package.json /usr/app +RUN npm install +COPY ./main.js /usr/app +CMD npx wait-on --timeout 10000 $POSTGREST_URL && node main.js diff --git a/data-generator/main.js b/data-generator/main.js new file mode 100644 index 000000000..42c36304f --- /dev/null +++ b/data-generator/main.js @@ -0,0 +1,529 @@ +// SPDX-FileCopyrightText: 2022 Ewan Cahen (Netherlands eScience Center) +// SPDX-FileCopyrightText: 2022 Netherlands eScience Center +// +// SPDX-License-Identifier: Apache-2.0 + +import {faker} from '@faker-js/faker'; +import jwt from 'jsonwebtoken'; + +function generateMentions(amountExtra = 10) { + const dois = [ + '10.1017/S0033291718004038', + '10.1186/s12966-019-0834-1', + '10.5334/dsj-2022-010', + '10.1175/BAMS-D-19-0337.1', + '10.5194/egusphere-egu21-4805', + '10.5194/esd-12-253-2021', + '10.5194/gmd-13-4205-2020', + '10.1145/2996913.2997005', + '10.1016/j.future.2018.08.004', + '10.1515/itit-2019-0040', + ]; + + const mentionTypes = [ + 'blogPost', + 'book', + 'bookSection', + 'computerProgram', + 'conferencePaper', + 'dataset', + 'interview', + 'highlight', + 'journalArticle', + 'magazineArticle', + 'newspaperArticle', + 'presentation', + 'report', + 'thesis', + 'videoRecording', + 'webpage', + 'other', + ]; + + const result = []; + + // first use up all the DOIs, then generate random mentions without DOI + for (const doi of dois) { + result.push({ + doi: doi, + url: 'https://doi.org/' + doi, + title: faker.music.songName(), + authors: faker.name.fullName, + publisher: faker.company.name(), + publication_year: faker.mersenne.rand(2026, 2000), + page: faker.helpers.maybe(() => faker.mersenne.rand(301, 0), 0.1) ?? null, + image_url: null, + mention_type: faker.helpers.arrayElement(mentionTypes), + source: 'faker', + }); + } + + for (let index = 0; index < amountExtra; index++) { + result.push({ + doi: null, + url: faker.internet.url(), + title: faker.music.songName(), + authors: faker.name.fullName, + publisher: faker.company.name(), + publication_year: faker.mersenne.rand(2026, 2000), + page: faker.helpers.maybe(() => faker.mersenne.rand(301, 0), 0.1) ?? null, + image_url: null, + mention_type: faker.helpers.arrayElement(mentionTypes), + source: 'faker', + }); + } + + return result; +} + +function generateSofware(amount=10) { + const result = []; + + for (let index = 0; index < amount; index++) { + const brandName = faker.hacker.adjective() + ' ' + faker.hacker.noun() + ' ' + faker.helpers.replaceSymbolWithNumber('####'); + result.push({ + slug: faker.helpers.slugify(brandName), + brand_name: brandName, + concept_doi: faker.helpers.replaceSymbols('10.*****/*****'), + description: faker.lorem.paragraphs(4, '\n\n'), + get_started_url: faker.internet.url(), + is_published: !!faker.helpers.maybe(() => true, {probability: 0.8}), + short_statement: faker.commerce.productDescription() + }); + } + + return result; +} + +function generateTestimonials(ids) { + const result = []; + + for (const id of ids) { + // each software will get 0, 1 or 2 testimonials + const numberOfTestimonials = faker.mersenne.rand(3,0); + for (let index = 0; index < numberOfTestimonials; index++) { + result.push({ + software: id, + message: faker.hacker.phrase(), + source: faker.name.fullName(), + }); + } + } + + return result; +} + +function generateRepositoryUrls(ids) { + const githubUrls = [ + 'https://github.com/research-software-directory/RSD-as-a-service', + 'https://github.com/wadpac/GGIR', + 'https://github.com/ESMValGroup/ESMValTool', + 'https://github.com/ESMValGroup/ESMValCore', + 'https://github.com/benvanwerkhoven/kernel_tuner', + 'https://github.com/NLeSC/pattyvis', + ]; + + const gitlabUrls = [ + 'https://gitlab.com/inkscape/inkscape', + 'https://gitlab.com/dwt1/dotfiles', + 'https://gitlab.com/famedly/fluffychat', + 'https://gitlab.com/gitlab-org/gitlab-shell', + 'https://gitlab.com/cerfacs/batman', + 'https://gitlab.com/cyber5k/mistborn', + ]; + + const repoUrls = githubUrls.concat(gitlabUrls); + + const result = []; + + for (let index = 0; index < ids.length; index++) { + if (!!faker.helpers.maybe(() => true, {probability: 0.25})) continue; + + const repoUrl = faker.helpers.arrayElement(repoUrls); + const codePlatform = repoUrl.startsWith('https://github.com') ? 'github' : 'gitlab'; + result.push({ + software: ids[index], + url: repoUrl, + code_platform: codePlatform, + }); + } + + return result; +} + +function generateLincensesForSoftware(ids) { + const licenses = [ + 'Apache-2.0', + 'MIT', + 'GPL-2.0-or-later', + 'LGPL-2.0-or-later', + 'CC-BY-4.0', + 'CC-BY-NC-ND-3.0', + ]; + + const result = []; + + for (const id of ids) { + const nummerOfLicenses = faker.mersenne.rand(3, 0); + if (nummerOfLicenses === 0) continue; + + const licensesToAdd = faker.helpers.arrayElements(licenses, nummerOfLicenses); + for (const license of licensesToAdd) { + result.push({ + software: id, + license: license, + }); + } + } + + return result; +} + +function generateKeywordsForEntity(idsEntity, idsKeyword, nameEntity) { + const result = []; + + for (const idEntity of idsEntity) { + const nummerOfKeywords = faker.mersenne.rand(3, 0); + if (nummerOfKeywords === 0) continue; + + const keywordIdsToAdd = faker.helpers.arrayElements(idsKeyword, nummerOfKeywords); + for (const keywordId of keywordIdsToAdd) { + result.push({ + [nameEntity]: idEntity, + keyword: keywordId, + }); + } + } + + return result; +} + +function generateMentionsForEntity(idsEntity, idsMention, nameEntity) { + const result = []; + + for (const idEntity of idsEntity) { + const nummerOfMentions = faker.mersenne.rand(4, 0); + if (nummerOfMentions === 0) continue; + + const mentionIdsToAdd = faker.helpers.arrayElements(idsMention, nummerOfMentions); + for (const mentionId of mentionIdsToAdd) { + result.push({ + [nameEntity]: idEntity, + mention: mentionId, + }); + } + } + + return result; +} + +function generateResearchDomainsForProjects(idsProject, idsResearchDomain) { + const result = []; + + for (const idProject of idsProject) { + const nummerOfKeywords = faker.mersenne.rand(3, 0); + if (nummerOfKeywords === 0) continue; + + const researchDomainIdsToAdd = faker.helpers.arrayElements(idsResearchDomain, nummerOfKeywords); + for (const researchDomainId of researchDomainIdsToAdd) { + result.push({ + project: idProject, + research_domain: researchDomainId, + }); + } + } + + return result; +} + +function generateSoftwareForSoftware(ids) { + const result = []; + + for (let index = 0; index < ids.length; index++) { + const numberOfRelatedSoftware = faker.mersenne.rand(5, 0); + if (numberOfRelatedSoftware === 0) continue; + + const origin = ids[index]; + const idsWithoutOrigin = ids.filter(id => id !== origin); + const idsRelation = faker.helpers.arrayElements(idsWithoutOrigin, numberOfRelatedSoftware); + for (const relation of idsRelation) { + result.push({ + origin: origin, + relation: relation, + }) + } + } + + return result; +} + +function generateProjects(amount=10) { + const result = []; + + for (let index = 0; index < amount; index++) { + const title = faker.hacker.adjective() + ' ' + faker.hacker.noun() + ' ' + faker.helpers.replaceSymbolWithNumber('####'); + result.push({ + slug: faker.helpers.slugify(title), + title: title, + subtitle: faker.commerce.productDescription(), + date_end: faker.date.future(2), + date_start: faker.date.past(2), + description: faker.lorem.paragraphs(5, '\n\n'), + grant_id: faker.helpers.replaceSymbols('******'), + image_caption: faker.animal.cat(), + image_contain: !!faker.helpers.maybe(() => true, {probability: 0.5}), + is_published: !!faker.helpers.maybe(() => true, {probability: 0.8}), + }); + } + + return result; +} + +async function generateContributors(ids, amount=20) { + const base64Images = await downloadImagesAsBase64(faker.image.avatar, amount); + + const result = []; + + for (let index = 0; index < amount; index++) { + result.push({ + software: faker.helpers.arrayElement(ids), + is_contact_person: !!faker.helpers.maybe(() => true, {probability: 0.2}), + email_address: faker.internet.email(), + family_names: faker.name.lastName(), + given_names: faker.name.firstName(), + affiliation: faker.company.name(), + role: faker.name.jobTitle(), + orcid: faker.helpers.replaceSymbolWithNumber('####-####-####-####'), + avatar_data: base64Images[index], + avatar_mime_type: 'image/jpeg', + }); + } + + return result; +} + +async function generateTeamMembers(ids, amount=20) { + const result = await generateContributors(ids, amount); + result.forEach(contributor => { + contributor['project'] = contributor['software']; + delete contributor['software']; + }); + return result; +} + +async function generateImagesForProjects(ids) { + const base64Images = await downloadImagesAsBase64(faker.image.cats, ids.length); + + const result = []; + + for (let index = 0; index < ids.length; index++) { + result.push({ + project: ids[index], + data: base64Images[index], + mime_type: 'image/jpeg', + }); + } + + return result; +} + +function generateUrlsForProjects(ids) { + const result = []; + + for (const id of ids) { + // each project will get 0, 1 or 2 URLs + const numberOfUrls = faker.mersenne.rand(3,0); + for (let index = 0; index < numberOfUrls; index++) { + result.push({ + project: id, + title: faker.commerce.product(), + url: faker.internet.url(), + }); + } + } + + return result; +} + +function generateOrganisations(amount=10) { + const result = []; + + for (let index = 0; index < amount; index++) { + const name = faker.company.name(); + result.push({ + parent: null, + primary_maintainer: null, + slug: faker.helpers.slugify(name).toLowerCase().replaceAll(/-{2,}/g, '-'), + name: name, + ror_id: faker.helpers.replaceSymbols('https://ror.org/********'), + website: faker.internet.url(), + is_tenant: !!faker.helpers.maybe(() => true, {probability: 0.3}), + }); + } + + return result; +} + +async function generateLogosForOrganisations(ids) { + const base64Images = await downloadImagesAsBase64(faker.image.business, ids.length); + + const result = []; + + for (let index = 0; index < ids.length; index++) { + result.push({ + organisation: ids[index], + data: base64Images[index], + mime_type: 'image/jpeg', + }); + } + + return result; +} + +function generateMetaPages() { + const result = []; + + const titles = ['Terms of Service', 'Privacy Statement']; + const slugs = ['tos', 'privacy']; + for (let index = 0; index < titles.length; index++) { + result.push({ + title: titles[index], + slug: slugs[index], + description: faker.lorem.paragraphs(10, '\n\n'), + is_published: true, + position: index + 1, + }); + } + + return result; +} + +function generateRelationsForDifferingEntities(idsOrigin, idsRelation, nameOrigin, nameRelation) { + const result = []; + + for (const idOrigin of idsOrigin) { + const numberOfIdsRelation = faker.mersenne.rand(5,0); + const relationsToAdd = faker.helpers.arrayElements(idsRelation, numberOfIdsRelation); + for (const idRelation of relationsToAdd) { + result.push({ + [nameOrigin]: idOrigin, + [nameRelation]: idRelation, + }); + } + } + + return result; +} + +function generateProjectForOrganisation(idsProjects, idsOrganisations) { + const result = generateRelationsForDifferingEntities(idsProjects, idsOrganisations, 'project', 'organisation'); + + const roles = ['funding', 'hosting', 'participating']; + result.forEach(entry => { + entry['role'] = faker.helpers.arrayElement(roles); + }); + + return result; +} + +function createJWT() { + const secret = process.env.PGRST_JWT_SECRET; + return jwt.sign({ 'role': 'rsd_admin' }, secret, {expiresIn: '2m'}); +} + +const token = createJWT(); +const headers = {'Content-Type': 'application/json', 'Authorization': 'bearer ' + token, 'Prefer': 'return=representation'} +const backendUrl = process.env.POSTGREST_URL || 'http://localhost/api/v1'; + +async function postToBackend(endpoint, body) { + const response = await fetch(backendUrl + endpoint, {method: 'POST', body: JSON.stringify(body), headers: headers}); + if(!response.ok) { + console.warn('Warning: post request to ' + endpoint + ' had status code ' + response.status + ' and body ' + await response.text()); + } + return response; +} + +async function getFromBackend(endpoint) { + const response = await fetch(backendUrl + endpoint, {headers: headers}); + if(!response.ok) { + console.warn('Warning: post request to ' + endpoint + ' had status code ' + response.status + ' and body ' + await response.text()); + } + return response; +} + +async function downloadImagesAsBase64(urlGenerator, amount) { + const imagePromises = []; + const timeOuts = []; + for (let index = 0; index < amount; index++) { + const url = urlGenerator(); + imagePromises.push( + Promise.race([ + fetch(url) + .then(resp => {clearTimeout(timeOuts[index]); return resp.arrayBuffer()}) + .then(ab => Buffer.from(ab)) + .then(bf => bf.toString('base64')), + new Promise((res, rej) => timeOuts[index] = setTimeout(rej, 3000, url)) + ]) + ); + } + return await Promise.all(imagePromises); +} + +let idsMentions, idsKeywords, idsResearchDomains; +const mentionsPromise = postToBackend('/mention', generateMentions()) + .then(() => getFromBackend('/mention?select=id')) + .then(res => res.json()) + .then(jsonMentions => idsMentions = jsonMentions.map(element => element.id)); +const keywordPromise = getFromBackend('/keyword?select=id') + .then(res => res.json()) + .then(jsonKeywords => idsKeywords = jsonKeywords.map(element => element.id)); +const researchDomainsPromise = await getFromBackend('/research_domain?select=id') + .then(res => res.json()) + .then(jsonResearchDomains => idsResearchDomains = jsonResearchDomains.map(element => element.id)); + +await Promise.all([mentionsPromise, keywordPromise, researchDomainsPromise]) + .then(() => console.log('mentions, keywords, research domains done')); + +let idsSoftware, idsProjects, idsOrganisations; +const softwarePromise = postToBackend('/software', generateSofware()) + .then(resp => resp.json()) + .then(async swArray => { + idsSoftware = swArray.map(sw => sw['id']); + postToBackend('/contributor', await generateContributors(idsSoftware)); + postToBackend('/testimonial', generateTestimonials(idsSoftware)); + postToBackend('/repository_url', generateRepositoryUrls(idsSoftware)); + postToBackend('/license_for_software', generateLincensesForSoftware(idsSoftware)); + postToBackend('/keyword_for_software', generateKeywordsForEntity(idsSoftware, idsKeywords, 'software')); + postToBackend('/mention_for_software', generateMentionsForEntity(idsSoftware, idsMentions, 'software')); + postToBackend('/software_for_software', generateSoftwareForSoftware(idsSoftware)); + }); +const projectPromise = postToBackend('/project', generateProjects()) + .then(resp => resp.json()) + .then(async pjArray => { + idsProjects = pjArray.map(sw => sw['id']); + postToBackend('/team_member', await generateTeamMembers(idsProjects)); + postToBackend('/image_for_project', await generateImagesForProjects(idsProjects)); + postToBackend('/url_for_project', generateUrlsForProjects(idsProjects)); + postToBackend('/keyword_for_project', generateKeywordsForEntity(idsProjects, idsKeywords, 'project')); + postToBackend('/output_for_project', generateMentionsForEntity(idsProjects, idsMentions, 'project')); + postToBackend('/impact_for_project', generateMentionsForEntity(idsProjects, idsMentions, 'project')); + postToBackend('/research_domain_for_project', generateResearchDomainsForProjects(idsProjects, idsResearchDomains)); + postToBackend('/project_for_project', generateSoftwareForSoftware(idsProjects)); + }); +const organisationPromise = postToBackend('/organisation', generateOrganisations()) + .then(resp => resp.json()) + .then(async orgArray => { + idsOrganisations = orgArray.map(org => org['id']); + postToBackend('/logo_for_organisation', await generateLogosForOrganisations(idsOrganisations)); + }); +await postToBackend('/meta_pages', generateMetaPages()).then(() => console.log('meta pages done')); + +await Promise.all([softwarePromise, projectPromise, organisationPromise]).then(() => console.log('sw, pg, org done')); + +await postToBackend('/software_for_project', generateRelationsForDifferingEntities(idsSoftware, idsProjects, 'software', 'project')).then(() => console.log('sw-pj done')); +await postToBackend('/software_for_organisation', generateRelationsForDifferingEntities(idsSoftware, idsOrganisations, 'software', 'organisation')).then(() => console.log('sw-org done')); +await postToBackend('/project_for_organisation', generateProjectForOrganisation(idsProjects, idsOrganisations)).then(() => console.log('pj-org done')); + +console.log('Done'); +// This is unfortunately needed, because when using docker-compose, the node process might hang for a long time +process.exit(0); diff --git a/data-generator/package-lock.json b/data-generator/package-lock.json new file mode 100644 index 000000000..5776507fc --- /dev/null +++ b/data-generator/package-lock.json @@ -0,0 +1,455 @@ +{ + "name": "data-generator", + "version": "1.0.0", + "lockfileVersion": 2, + "requires": true, + "packages": { + "": { + "name": "data-generator", + "version": "1.0.0", + "license": "ISC", + "dependencies": { + "@faker-js/faker": "7.5.0", + "jsonwebtoken": "8.5.1", + "wait-on": "6.0.1" + } + }, + "node_modules/@faker-js/faker": { + "version": "7.5.0", + "resolved": "https://registry.npmjs.org/@faker-js/faker/-/faker-7.5.0.tgz", + "integrity": "sha512-8wNUCCUHvfvI0gQpDUho/3gPzABffnCn5um65F8dzQ86zz6dlt4+nmAA7PQUc8L+eH+9RgR/qzy5N/8kN0Ozdw==", + "engines": { + "node": ">=14.0.0", + "npm": ">=6.0.0" + } + }, + "node_modules/@hapi/hoek": { + "version": "9.3.0", + "resolved": "https://registry.npmjs.org/@hapi/hoek/-/hoek-9.3.0.tgz", + "integrity": "sha512-/c6rf4UJlmHlC9b5BaNvzAcFv7HZ2QHaV0D4/HNlBdvFnvQq8RI4kYdhyPCl7Xj+oWvTWQ8ujhqS53LIgAe6KQ==" + }, + "node_modules/@hapi/topo": { + "version": "5.1.0", + "resolved": "https://registry.npmjs.org/@hapi/topo/-/topo-5.1.0.tgz", + "integrity": "sha512-foQZKJig7Ob0BMAYBfcJk8d77QtOe7Wo4ox7ff1lQYoNNAb6jwcY1ncdoy2e9wQZzvNy7ODZCYJkK8kzmcAnAg==", + "dependencies": { + "@hapi/hoek": "^9.0.0" + } + }, + "node_modules/@sideway/address": { + "version": "4.1.4", + "resolved": "https://registry.npmjs.org/@sideway/address/-/address-4.1.4.tgz", + "integrity": "sha512-7vwq+rOHVWjyXxVlR76Agnvhy8I9rpzjosTESvmhNeXOXdZZB15Fl+TI9x1SiHZH5Jv2wTGduSxFDIaq0m3DUw==", + "dependencies": { + "@hapi/hoek": "^9.0.0" + } + }, + "node_modules/@sideway/formula": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/@sideway/formula/-/formula-3.0.0.tgz", + "integrity": "sha512-vHe7wZ4NOXVfkoRb8T5otiENVlT7a3IAiw7H5M2+GO+9CDgcVUUsX1zalAztCmwyOr2RUTGJdgB+ZvSVqmdHmg==" + }, + "node_modules/@sideway/pinpoint": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/@sideway/pinpoint/-/pinpoint-2.0.0.tgz", + "integrity": "sha512-RNiOoTPkptFtSVzQevY/yWtZwf/RxyVnPy/OcA9HBM3MlGDnBEYL5B41H0MTn0Uec8Hi+2qUtTfG2WWZBmMejQ==" + }, + "node_modules/axios": { + "version": "0.25.0", + "resolved": "https://registry.npmjs.org/axios/-/axios-0.25.0.tgz", + "integrity": "sha512-cD8FOb0tRH3uuEe6+evtAbgJtfxr7ly3fQjYcMcuPlgkwVS9xboaVIpcDV+cYQe+yGykgwZCs1pzjntcGa6l5g==", + "dependencies": { + "follow-redirects": "^1.14.7" + } + }, + "node_modules/buffer-equal-constant-time": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/buffer-equal-constant-time/-/buffer-equal-constant-time-1.0.1.tgz", + "integrity": "sha512-zRpUiDwd/xk6ADqPMATG8vc9VPrkck7T07OIx0gnjmJAnHnTVXNQG3vfvWNuiZIkwu9KrKdA1iJKfsfTVxE6NA==" + }, + "node_modules/ecdsa-sig-formatter": { + "version": "1.0.11", + "resolved": "https://registry.npmjs.org/ecdsa-sig-formatter/-/ecdsa-sig-formatter-1.0.11.tgz", + "integrity": "sha512-nagl3RYrbNv6kQkeJIpt6NJZy8twLB/2vtz6yN9Z4vRKHN4/QZJIEbqohALSgwKdnksuY3k5Addp5lg8sVoVcQ==", + "dependencies": { + "safe-buffer": "^5.0.1" + } + }, + "node_modules/follow-redirects": { + "version": "1.15.1", + "resolved": "https://registry.npmjs.org/follow-redirects/-/follow-redirects-1.15.1.tgz", + "integrity": "sha512-yLAMQs+k0b2m7cVxpS1VKJVvoz7SS9Td1zss3XRwXj+ZDH00RJgnuLx7E44wx02kQLrdM3aOOy+FpzS7+8OizA==", + "funding": [ + { + "type": "individual", + "url": "https://github.com/sponsors/RubenVerborgh" + } + ], + "engines": { + "node": ">=4.0" + }, + "peerDependenciesMeta": { + "debug": { + "optional": true + } + } + }, + "node_modules/joi": { + "version": "17.6.0", + "resolved": "https://registry.npmjs.org/joi/-/joi-17.6.0.tgz", + "integrity": "sha512-OX5dG6DTbcr/kbMFj0KGYxuew69HPcAE3K/sZpEV2nP6e/j/C0HV+HNiBPCASxdx5T7DMoa0s8UeHWMnb6n2zw==", + "dependencies": { + "@hapi/hoek": "^9.0.0", + "@hapi/topo": "^5.0.0", + "@sideway/address": "^4.1.3", + "@sideway/formula": "^3.0.0", + "@sideway/pinpoint": "^2.0.0" + } + }, + "node_modules/jsonwebtoken": { + "version": "8.5.1", + "resolved": "https://registry.npmjs.org/jsonwebtoken/-/jsonwebtoken-8.5.1.tgz", + "integrity": "sha512-XjwVfRS6jTMsqYs0EsuJ4LGxXV14zQybNd4L2r0UvbVnSF9Af8x7p5MzbJ90Ioz/9TI41/hTCvznF/loiSzn8w==", + "dependencies": { + "jws": "^3.2.2", + "lodash.includes": "^4.3.0", + "lodash.isboolean": "^3.0.3", + "lodash.isinteger": "^4.0.4", + "lodash.isnumber": "^3.0.3", + "lodash.isplainobject": "^4.0.6", + "lodash.isstring": "^4.0.1", + "lodash.once": "^4.0.0", + "ms": "^2.1.1", + "semver": "^5.6.0" + }, + "engines": { + "node": ">=4", + "npm": ">=1.4.28" + } + }, + "node_modules/jwa": { + "version": "1.4.1", + "resolved": "https://registry.npmjs.org/jwa/-/jwa-1.4.1.tgz", + "integrity": "sha512-qiLX/xhEEFKUAJ6FiBMbes3w9ATzyk5W7Hvzpa/SLYdxNtng+gcurvrI7TbACjIXlsJyr05/S1oUhZrc63evQA==", + "dependencies": { + "buffer-equal-constant-time": "1.0.1", + "ecdsa-sig-formatter": "1.0.11", + "safe-buffer": "^5.0.1" + } + }, + "node_modules/jws": { + "version": "3.2.2", + "resolved": "https://registry.npmjs.org/jws/-/jws-3.2.2.tgz", + "integrity": "sha512-YHlZCB6lMTllWDtSPHz/ZXTsi8S00usEV6v1tjq8tOUZzw7DpSDWVXjXDre6ed1w/pd495ODpHZYSdkRTsa0HA==", + "dependencies": { + "jwa": "^1.4.1", + "safe-buffer": "^5.0.1" + } + }, + "node_modules/lodash": { + "version": "4.17.21", + "resolved": "https://registry.npmjs.org/lodash/-/lodash-4.17.21.tgz", + "integrity": "sha512-v2kDEe57lecTulaDIuNTPy3Ry4gLGJ6Z1O3vE1krgXZNrsQ+LFTGHVxVjcXPs17LhbZVGedAJv8XZ1tvj5FvSg==" + }, + "node_modules/lodash.includes": { + "version": "4.3.0", + "resolved": "https://registry.npmjs.org/lodash.includes/-/lodash.includes-4.3.0.tgz", + "integrity": "sha512-W3Bx6mdkRTGtlJISOvVD/lbqjTlPPUDTMnlXZFnVwi9NKJ6tiAk6LVdlhZMm17VZisqhKcgzpO5Wz91PCt5b0w==" + }, + "node_modules/lodash.isboolean": { + "version": "3.0.3", + "resolved": "https://registry.npmjs.org/lodash.isboolean/-/lodash.isboolean-3.0.3.tgz", + "integrity": "sha512-Bz5mupy2SVbPHURB98VAcw+aHh4vRV5IPNhILUCsOzRmsTmSQ17jIuqopAentWoehktxGd9e/hbIXq980/1QJg==" + }, + "node_modules/lodash.isinteger": { + "version": "4.0.4", + "resolved": "https://registry.npmjs.org/lodash.isinteger/-/lodash.isinteger-4.0.4.tgz", + "integrity": "sha512-DBwtEWN2caHQ9/imiNeEA5ys1JoRtRfY3d7V9wkqtbycnAmTvRRmbHKDV4a0EYc678/dia0jrte4tjYwVBaZUA==" + }, + "node_modules/lodash.isnumber": { + "version": "3.0.3", + "resolved": "https://registry.npmjs.org/lodash.isnumber/-/lodash.isnumber-3.0.3.tgz", + "integrity": "sha512-QYqzpfwO3/CWf3XP+Z+tkQsfaLL/EnUlXWVkIk5FUPc4sBdTehEqZONuyRt2P67PXAk+NXmTBcc97zw9t1FQrw==" + }, + "node_modules/lodash.isplainobject": { + "version": "4.0.6", + "resolved": "https://registry.npmjs.org/lodash.isplainobject/-/lodash.isplainobject-4.0.6.tgz", + "integrity": "sha512-oSXzaWypCMHkPC3NvBEaPHf0KsA5mvPrOPgQWDsbg8n7orZ290M0BmC/jgRZ4vcJ6DTAhjrsSYgdsW/F+MFOBA==" + }, + "node_modules/lodash.isstring": { + "version": "4.0.1", + "resolved": "https://registry.npmjs.org/lodash.isstring/-/lodash.isstring-4.0.1.tgz", + "integrity": "sha512-0wJxfxH1wgO3GrbuP+dTTk7op+6L41QCXbGINEmD+ny/G/eCqGzxyCsh7159S+mgDDcoarnBw6PC1PS5+wUGgw==" + }, + "node_modules/lodash.once": { + "version": "4.1.1", + "resolved": "https://registry.npmjs.org/lodash.once/-/lodash.once-4.1.1.tgz", + "integrity": "sha512-Sb487aTOCr9drQVL8pIxOzVhafOjZN9UU54hiN8PU3uAiSV7lx1yYNpbNmex2PK6dSJoNTSJUUswT651yww3Mg==" + }, + "node_modules/minimist": { + "version": "1.2.6", + "resolved": "https://registry.npmjs.org/minimist/-/minimist-1.2.6.tgz", + "integrity": "sha512-Jsjnk4bw3YJqYzbdyBiNsPWHPfO++UGG749Cxs6peCu5Xg4nrena6OVxOYxrQTqww0Jmwt+Ref8rggumkTLz9Q==" + }, + "node_modules/ms": { + "version": "2.1.3", + "resolved": "https://registry.npmjs.org/ms/-/ms-2.1.3.tgz", + "integrity": "sha512-6FlzubTLZG3J2a/NVCAleEhjzq5oxgHyaCU9yYXvcLsvoVaHJq/s5xXI6/XXP6tz7R9xAOtHnSO/tXtF3WRTlA==" + }, + "node_modules/rxjs": { + "version": "7.5.6", + "resolved": "https://registry.npmjs.org/rxjs/-/rxjs-7.5.6.tgz", + "integrity": "sha512-dnyv2/YsXhnm461G+R/Pe5bWP41Nm6LBXEYWI6eiFP4fiwx6WRI/CD0zbdVAudd9xwLEF2IDcKXLHit0FYjUzw==", + "dependencies": { + "tslib": "^2.1.0" + } + }, + "node_modules/safe-buffer": { + "version": "5.2.1", + "resolved": "https://registry.npmjs.org/safe-buffer/-/safe-buffer-5.2.1.tgz", + "integrity": "sha512-rp3So07KcdmmKbGvgaNxQSJr7bGVSVk5S9Eq1F+ppbRo70+YeaDxkw5Dd8NPN+GD6bjnYm2VuPuCXmpuYvmCXQ==", + "funding": [ + { + "type": "github", + "url": "https://github.com/sponsors/feross" + }, + { + "type": "patreon", + "url": "https://www.patreon.com/feross" + }, + { + "type": "consulting", + "url": "https://feross.org/support" + } + ] + }, + "node_modules/semver": { + "version": "5.7.1", + "resolved": "https://registry.npmjs.org/semver/-/semver-5.7.1.tgz", + "integrity": "sha512-sauaDf/PZdVgrLTNYHRtpXa1iRiKcaebiKQ1BJdpQlWH2lCvexQdX55snPFyK7QzpudqbCI0qXFfOasHdyNDGQ==", + "bin": { + "semver": "bin/semver" + } + }, + "node_modules/tslib": { + "version": "2.4.0", + "resolved": "https://registry.npmjs.org/tslib/-/tslib-2.4.0.tgz", + "integrity": "sha512-d6xOpEDfsi2CZVlPQzGeux8XMwLT9hssAsaPYExaQMuYskwb+x1x7J371tWlbBdWHroy99KnVB6qIkUbs5X3UQ==" + }, + "node_modules/wait-on": { + "version": "6.0.1", + "resolved": "https://registry.npmjs.org/wait-on/-/wait-on-6.0.1.tgz", + "integrity": "sha512-zht+KASY3usTY5u2LgaNqn/Cd8MukxLGjdcZxT2ns5QzDmTFc4XoWBgC+C/na+sMRZTuVygQoMYwdcVjHnYIVw==", + "dependencies": { + "axios": "^0.25.0", + "joi": "^17.6.0", + "lodash": "^4.17.21", + "minimist": "^1.2.5", + "rxjs": "^7.5.4" + }, + "bin": { + "wait-on": "bin/wait-on" + }, + "engines": { + "node": ">=10.0.0" + } + } + }, + "dependencies": { + "@faker-js/faker": { + "version": "7.5.0", + "resolved": "https://registry.npmjs.org/@faker-js/faker/-/faker-7.5.0.tgz", + "integrity": "sha512-8wNUCCUHvfvI0gQpDUho/3gPzABffnCn5um65F8dzQ86zz6dlt4+nmAA7PQUc8L+eH+9RgR/qzy5N/8kN0Ozdw==" + }, + "@hapi/hoek": { + "version": "9.3.0", + "resolved": "https://registry.npmjs.org/@hapi/hoek/-/hoek-9.3.0.tgz", + "integrity": "sha512-/c6rf4UJlmHlC9b5BaNvzAcFv7HZ2QHaV0D4/HNlBdvFnvQq8RI4kYdhyPCl7Xj+oWvTWQ8ujhqS53LIgAe6KQ==" + }, + "@hapi/topo": { + "version": "5.1.0", + "resolved": "https://registry.npmjs.org/@hapi/topo/-/topo-5.1.0.tgz", + "integrity": "sha512-foQZKJig7Ob0BMAYBfcJk8d77QtOe7Wo4ox7ff1lQYoNNAb6jwcY1ncdoy2e9wQZzvNy7ODZCYJkK8kzmcAnAg==", + "requires": { + "@hapi/hoek": "^9.0.0" + } + }, + "@sideway/address": { + "version": "4.1.4", + "resolved": "https://registry.npmjs.org/@sideway/address/-/address-4.1.4.tgz", + "integrity": "sha512-7vwq+rOHVWjyXxVlR76Agnvhy8I9rpzjosTESvmhNeXOXdZZB15Fl+TI9x1SiHZH5Jv2wTGduSxFDIaq0m3DUw==", + "requires": { + "@hapi/hoek": "^9.0.0" + } + }, + "@sideway/formula": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/@sideway/formula/-/formula-3.0.0.tgz", + "integrity": "sha512-vHe7wZ4NOXVfkoRb8T5otiENVlT7a3IAiw7H5M2+GO+9CDgcVUUsX1zalAztCmwyOr2RUTGJdgB+ZvSVqmdHmg==" + }, + "@sideway/pinpoint": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/@sideway/pinpoint/-/pinpoint-2.0.0.tgz", + "integrity": "sha512-RNiOoTPkptFtSVzQevY/yWtZwf/RxyVnPy/OcA9HBM3MlGDnBEYL5B41H0MTn0Uec8Hi+2qUtTfG2WWZBmMejQ==" + }, + "axios": { + "version": "0.25.0", + "resolved": "https://registry.npmjs.org/axios/-/axios-0.25.0.tgz", + "integrity": "sha512-cD8FOb0tRH3uuEe6+evtAbgJtfxr7ly3fQjYcMcuPlgkwVS9xboaVIpcDV+cYQe+yGykgwZCs1pzjntcGa6l5g==", + "requires": { + "follow-redirects": "^1.14.7" + } + }, + "buffer-equal-constant-time": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/buffer-equal-constant-time/-/buffer-equal-constant-time-1.0.1.tgz", + "integrity": "sha512-zRpUiDwd/xk6ADqPMATG8vc9VPrkck7T07OIx0gnjmJAnHnTVXNQG3vfvWNuiZIkwu9KrKdA1iJKfsfTVxE6NA==" + }, + "ecdsa-sig-formatter": { + "version": "1.0.11", + "resolved": "https://registry.npmjs.org/ecdsa-sig-formatter/-/ecdsa-sig-formatter-1.0.11.tgz", + "integrity": "sha512-nagl3RYrbNv6kQkeJIpt6NJZy8twLB/2vtz6yN9Z4vRKHN4/QZJIEbqohALSgwKdnksuY3k5Addp5lg8sVoVcQ==", + "requires": { + "safe-buffer": "^5.0.1" + } + }, + "follow-redirects": { + "version": "1.15.1", + "resolved": "https://registry.npmjs.org/follow-redirects/-/follow-redirects-1.15.1.tgz", + "integrity": "sha512-yLAMQs+k0b2m7cVxpS1VKJVvoz7SS9Td1zss3XRwXj+ZDH00RJgnuLx7E44wx02kQLrdM3aOOy+FpzS7+8OizA==" + }, + "joi": { + "version": "17.6.0", + "resolved": "https://registry.npmjs.org/joi/-/joi-17.6.0.tgz", + "integrity": "sha512-OX5dG6DTbcr/kbMFj0KGYxuew69HPcAE3K/sZpEV2nP6e/j/C0HV+HNiBPCASxdx5T7DMoa0s8UeHWMnb6n2zw==", + "requires": { + "@hapi/hoek": "^9.0.0", + "@hapi/topo": "^5.0.0", + "@sideway/address": "^4.1.3", + "@sideway/formula": "^3.0.0", + "@sideway/pinpoint": "^2.0.0" + } + }, + "jsonwebtoken": { + "version": "8.5.1", + "resolved": "https://registry.npmjs.org/jsonwebtoken/-/jsonwebtoken-8.5.1.tgz", + "integrity": "sha512-XjwVfRS6jTMsqYs0EsuJ4LGxXV14zQybNd4L2r0UvbVnSF9Af8x7p5MzbJ90Ioz/9TI41/hTCvznF/loiSzn8w==", + "requires": { + "jws": "^3.2.2", + "lodash.includes": "^4.3.0", + "lodash.isboolean": "^3.0.3", + "lodash.isinteger": "^4.0.4", + "lodash.isnumber": "^3.0.3", + "lodash.isplainobject": "^4.0.6", + "lodash.isstring": "^4.0.1", + "lodash.once": "^4.0.0", + "ms": "^2.1.1", + "semver": "^5.6.0" + } + }, + "jwa": { + "version": "1.4.1", + "resolved": "https://registry.npmjs.org/jwa/-/jwa-1.4.1.tgz", + "integrity": "sha512-qiLX/xhEEFKUAJ6FiBMbes3w9ATzyk5W7Hvzpa/SLYdxNtng+gcurvrI7TbACjIXlsJyr05/S1oUhZrc63evQA==", + "requires": { + "buffer-equal-constant-time": "1.0.1", + "ecdsa-sig-formatter": "1.0.11", + "safe-buffer": "^5.0.1" + } + }, + "jws": { + "version": "3.2.2", + "resolved": "https://registry.npmjs.org/jws/-/jws-3.2.2.tgz", + "integrity": "sha512-YHlZCB6lMTllWDtSPHz/ZXTsi8S00usEV6v1tjq8tOUZzw7DpSDWVXjXDre6ed1w/pd495ODpHZYSdkRTsa0HA==", + "requires": { + "jwa": "^1.4.1", + "safe-buffer": "^5.0.1" + } + }, + "lodash": { + "version": "4.17.21", + "resolved": "https://registry.npmjs.org/lodash/-/lodash-4.17.21.tgz", + "integrity": "sha512-v2kDEe57lecTulaDIuNTPy3Ry4gLGJ6Z1O3vE1krgXZNrsQ+LFTGHVxVjcXPs17LhbZVGedAJv8XZ1tvj5FvSg==" + }, + "lodash.includes": { + "version": "4.3.0", + "resolved": "https://registry.npmjs.org/lodash.includes/-/lodash.includes-4.3.0.tgz", + "integrity": "sha512-W3Bx6mdkRTGtlJISOvVD/lbqjTlPPUDTMnlXZFnVwi9NKJ6tiAk6LVdlhZMm17VZisqhKcgzpO5Wz91PCt5b0w==" + }, + "lodash.isboolean": { + "version": "3.0.3", + "resolved": "https://registry.npmjs.org/lodash.isboolean/-/lodash.isboolean-3.0.3.tgz", + "integrity": "sha512-Bz5mupy2SVbPHURB98VAcw+aHh4vRV5IPNhILUCsOzRmsTmSQ17jIuqopAentWoehktxGd9e/hbIXq980/1QJg==" + }, + "lodash.isinteger": { + "version": "4.0.4", + "resolved": "https://registry.npmjs.org/lodash.isinteger/-/lodash.isinteger-4.0.4.tgz", + "integrity": "sha512-DBwtEWN2caHQ9/imiNeEA5ys1JoRtRfY3d7V9wkqtbycnAmTvRRmbHKDV4a0EYc678/dia0jrte4tjYwVBaZUA==" + }, + "lodash.isnumber": { + "version": "3.0.3", + "resolved": "https://registry.npmjs.org/lodash.isnumber/-/lodash.isnumber-3.0.3.tgz", + "integrity": "sha512-QYqzpfwO3/CWf3XP+Z+tkQsfaLL/EnUlXWVkIk5FUPc4sBdTehEqZONuyRt2P67PXAk+NXmTBcc97zw9t1FQrw==" + }, + "lodash.isplainobject": { + "version": "4.0.6", + "resolved": "https://registry.npmjs.org/lodash.isplainobject/-/lodash.isplainobject-4.0.6.tgz", + "integrity": "sha512-oSXzaWypCMHkPC3NvBEaPHf0KsA5mvPrOPgQWDsbg8n7orZ290M0BmC/jgRZ4vcJ6DTAhjrsSYgdsW/F+MFOBA==" + }, + "lodash.isstring": { + "version": "4.0.1", + "resolved": "https://registry.npmjs.org/lodash.isstring/-/lodash.isstring-4.0.1.tgz", + "integrity": "sha512-0wJxfxH1wgO3GrbuP+dTTk7op+6L41QCXbGINEmD+ny/G/eCqGzxyCsh7159S+mgDDcoarnBw6PC1PS5+wUGgw==" + }, + "lodash.once": { + "version": "4.1.1", + "resolved": "https://registry.npmjs.org/lodash.once/-/lodash.once-4.1.1.tgz", + "integrity": "sha512-Sb487aTOCr9drQVL8pIxOzVhafOjZN9UU54hiN8PU3uAiSV7lx1yYNpbNmex2PK6dSJoNTSJUUswT651yww3Mg==" + }, + "minimist": { + "version": "1.2.6", + "resolved": "https://registry.npmjs.org/minimist/-/minimist-1.2.6.tgz", + "integrity": "sha512-Jsjnk4bw3YJqYzbdyBiNsPWHPfO++UGG749Cxs6peCu5Xg4nrena6OVxOYxrQTqww0Jmwt+Ref8rggumkTLz9Q==" + }, + "ms": { + "version": "2.1.3", + "resolved": "https://registry.npmjs.org/ms/-/ms-2.1.3.tgz", + "integrity": "sha512-6FlzubTLZG3J2a/NVCAleEhjzq5oxgHyaCU9yYXvcLsvoVaHJq/s5xXI6/XXP6tz7R9xAOtHnSO/tXtF3WRTlA==" + }, + "rxjs": { + "version": "7.5.6", + "resolved": "https://registry.npmjs.org/rxjs/-/rxjs-7.5.6.tgz", + "integrity": "sha512-dnyv2/YsXhnm461G+R/Pe5bWP41Nm6LBXEYWI6eiFP4fiwx6WRI/CD0zbdVAudd9xwLEF2IDcKXLHit0FYjUzw==", + "requires": { + "tslib": "^2.1.0" + } + }, + "safe-buffer": { + "version": "5.2.1", + "resolved": "https://registry.npmjs.org/safe-buffer/-/safe-buffer-5.2.1.tgz", + "integrity": "sha512-rp3So07KcdmmKbGvgaNxQSJr7bGVSVk5S9Eq1F+ppbRo70+YeaDxkw5Dd8NPN+GD6bjnYm2VuPuCXmpuYvmCXQ==" + }, + "semver": { + "version": "5.7.1", + "resolved": "https://registry.npmjs.org/semver/-/semver-5.7.1.tgz", + "integrity": "sha512-sauaDf/PZdVgrLTNYHRtpXa1iRiKcaebiKQ1BJdpQlWH2lCvexQdX55snPFyK7QzpudqbCI0qXFfOasHdyNDGQ==" + }, + "tslib": { + "version": "2.4.0", + "resolved": "https://registry.npmjs.org/tslib/-/tslib-2.4.0.tgz", + "integrity": "sha512-d6xOpEDfsi2CZVlPQzGeux8XMwLT9hssAsaPYExaQMuYskwb+x1x7J371tWlbBdWHroy99KnVB6qIkUbs5X3UQ==" + }, + "wait-on": { + "version": "6.0.1", + "resolved": "https://registry.npmjs.org/wait-on/-/wait-on-6.0.1.tgz", + "integrity": "sha512-zht+KASY3usTY5u2LgaNqn/Cd8MukxLGjdcZxT2ns5QzDmTFc4XoWBgC+C/na+sMRZTuVygQoMYwdcVjHnYIVw==", + "requires": { + "axios": "^0.25.0", + "joi": "^17.6.0", + "lodash": "^4.17.21", + "minimist": "^1.2.5", + "rxjs": "^7.5.4" + } + } + } +} diff --git a/data-generator/package-lock.json.license b/data-generator/package-lock.json.license new file mode 100644 index 000000000..5653abdb7 --- /dev/null +++ b/data-generator/package-lock.json.license @@ -0,0 +1,4 @@ +SPDX-FileCopyrightText: 2022 Ewan Cahen (Netherlands eScience Center) +SPDX-FileCopyrightText: 2022 Netherlands eScience Center + +SPDX-License-Identifier: Apache-2.0 diff --git a/data-generator/package.json b/data-generator/package.json new file mode 100644 index 000000000..fab504935 --- /dev/null +++ b/data-generator/package.json @@ -0,0 +1,17 @@ +{ + "dependencies": { + "@faker-js/faker": "7.5.0", + "jsonwebtoken": "8.5.1", + "wait-on": "6.0.1" + }, + "name": "data-generator", + "version": "1.0.0", + "main": "test.js", + "scripts": { + "test": "echo \"Error: no test specified\" && exit 1" + }, + "author": "", + "license": "ISC", + "description": "", + "type": "module" +} diff --git a/data-generator/package.json.license b/data-generator/package.json.license new file mode 100644 index 000000000..5653abdb7 --- /dev/null +++ b/data-generator/package.json.license @@ -0,0 +1,4 @@ +SPDX-FileCopyrightText: 2022 Ewan Cahen (Netherlands eScience Center) +SPDX-FileCopyrightText: 2022 Netherlands eScience Center + +SPDX-License-Identifier: Apache-2.0 diff --git a/docker-compose.yml b/docker-compose.yml index 430127204..2cc947a46 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -48,6 +48,20 @@ services: networks: - net + data-generation: + build: ./data-generator + image: rsd/generation:1.0.0 + environment: + # it needs to be here to use values from .env file + - PGRST_JWT_SECRET + - POSTGREST_URL + depends_on: + - backend + networks: + - net + deploy: + replicas: 0 + auth: build: ./authentication image: rsd/auth:1.0.0 From 7c649321866cec9f0e8126973e30a57389069f1a Mon Sep 17 00:00:00 2001 From: Ewan Cahen Date: Thu, 8 Sep 2022 13:09:26 +0200 Subject: [PATCH 2/3] fix: generate more data by default, dont't exit on failure to download an image but print warning instead --- data-generator/main.js | 27 ++++++++++++++++++--------- 1 file changed, 18 insertions(+), 9 deletions(-) diff --git a/data-generator/main.js b/data-generator/main.js index 42c36304f..a9910ea9a 100644 --- a/data-generator/main.js +++ b/data-generator/main.js @@ -76,11 +76,13 @@ function generateMentions(amountExtra = 10) { return result; } -function generateSofware(amount=10) { +function generateSofware(amount=50) { const result = []; for (let index = 0; index < amount; index++) { - const brandName = faker.hacker.adjective() + ' ' + faker.hacker.noun() + ' ' + faker.helpers.replaceSymbolWithNumber('####'); + const brandName = faker.helpers.unique(() => + faker.hacker.adjective() + ' ' + faker.hacker.noun() + ' ' + faker.helpers.replaceSymbolWithNumber('####') + ); result.push({ slug: faker.helpers.slugify(brandName), brand_name: brandName, @@ -257,11 +259,13 @@ function generateSoftwareForSoftware(ids) { return result; } -function generateProjects(amount=10) { +function generateProjects(amount=50) { const result = []; for (let index = 0; index < amount; index++) { - const title = faker.hacker.adjective() + ' ' + faker.hacker.noun() + ' ' + faker.helpers.replaceSymbolWithNumber('####'); + const title = faker.helpers.unique(() => + faker.hacker.adjective() + ' ' + faker.hacker.noun() + ' ' + faker.helpers.replaceSymbolWithNumber('####') + ); result.push({ slug: faker.helpers.slugify(title), title: title, @@ -279,7 +283,7 @@ function generateProjects(amount=10) { return result; } -async function generateContributors(ids, amount=20) { +async function generateContributors(ids, amount=100) { const base64Images = await downloadImagesAsBase64(faker.image.avatar, amount); const result = []; @@ -302,7 +306,7 @@ async function generateContributors(ids, amount=20) { return result; } -async function generateTeamMembers(ids, amount=20) { +async function generateTeamMembers(ids, amount=100) { const result = await generateContributors(ids, amount); result.forEach(contributor => { contributor['project'] = contributor['software']; @@ -317,6 +321,7 @@ async function generateImagesForProjects(ids) { const result = []; for (let index = 0; index < ids.length; index++) { + if (base64Images[index] === null) continue; result.push({ project: ids[index], data: base64Images[index], @@ -345,11 +350,13 @@ function generateUrlsForProjects(ids) { return result; } -function generateOrganisations(amount=10) { +function generateOrganisations(amount=50) { const result = []; for (let index = 0; index < amount; index++) { - const name = faker.company.name(); + const name = faker.helpers.unique(() => + faker.company.name() + ); result.push({ parent: null, primary_maintainer: null, @@ -370,6 +377,7 @@ async function generateLogosForOrganisations(ids) { const result = []; for (let index = 0; index < ids.length; index++) { + if (base64Images === null) continue; result.push({ organisation: ids[index], data: base64Images[index], @@ -462,7 +470,8 @@ async function downloadImagesAsBase64(urlGenerator, amount) { .then(resp => {clearTimeout(timeOuts[index]); return resp.arrayBuffer()}) .then(ab => Buffer.from(ab)) .then(bf => bf.toString('base64')), - new Promise((res, rej) => timeOuts[index] = setTimeout(rej, 3000, url)) + new Promise((res, rej) => timeOuts[index] = setTimeout(res, 3000)) + .then(() => {console.warn('Timeout for ' + url + ', skipping'); return null;}) ]) ); } From a4c0c3dcce9c0d6af63acd1c82b247a0c1885d22 Mon Sep 17 00:00:00 2001 From: Ewan Cahen Date: Fri, 9 Sep 2022 10:11:11 +0200 Subject: [PATCH 3/3] chore: add software or project to software or project title --- data-generator/main.js | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/data-generator/main.js b/data-generator/main.js index a9910ea9a..6066aa3fd 100644 --- a/data-generator/main.js +++ b/data-generator/main.js @@ -81,7 +81,7 @@ function generateSofware(amount=50) { for (let index = 0; index < amount; index++) { const brandName = faker.helpers.unique(() => - faker.hacker.adjective() + ' ' + faker.hacker.noun() + ' ' + faker.helpers.replaceSymbolWithNumber('####') + 'software: ' + faker.hacker.adjective() + ' ' + faker.hacker.noun() + ' ' + faker.helpers.replaceSymbolWithNumber('####') ); result.push({ slug: faker.helpers.slugify(brandName), @@ -264,7 +264,7 @@ function generateProjects(amount=50) { for (let index = 0; index < amount; index++) { const title = faker.helpers.unique(() => - faker.hacker.adjective() + ' ' + faker.hacker.noun() + ' ' + faker.helpers.replaceSymbolWithNumber('####') + 'project: ' + faker.hacker.adjective() + ' ' + faker.hacker.noun() + ' ' + faker.helpers.replaceSymbolWithNumber('####') ); result.push({ slug: faker.helpers.slugify(title),