From 2f6822e5a6518dc99700e1a9e27fad9fcca9a6bb Mon Sep 17 00:00:00 2001 From: Qing Tomlinson Date: Fri, 10 Feb 2023 07:55:30 -0800 Subject: [PATCH 1/4] Add TrimmedMongoDefinitionStore To store definition without files as the document --- providers/index.js | 3 +- .../stores/abstractMongoDefinitionStore.js | 221 ++++++++++++++++++ providers/stores/mongo.js | 189 ++------------- providers/stores/mongoConfig.js | 28 ++- .../stores/trimmedMongoDefinitionStore.js | 43 ++++ test/providers/store/mongoDefinition.js | 19 +- .../store/trimmedMongoDefinitionStore.js | 202 ++++++++++++++++ 7 files changed, 521 insertions(+), 184 deletions(-) create mode 100644 providers/stores/abstractMongoDefinitionStore.js create mode 100644 providers/stores/trimmedMongoDefinitionStore.js create mode 100644 test/providers/store/trimmedMongoDefinitionStore.js diff --git a/providers/index.js b/providers/index.js index acb6e89b6..885f69a90 100644 --- a/providers/index.js +++ b/providers/index.js @@ -8,7 +8,8 @@ module.exports = { definition: { azure: require('../providers/stores/azblobConfig').definition, file: require('../providers/stores/fileConfig').definition, - mongo: require('../providers/stores/mongoConfig'), + mongo: require('../providers/stores/mongoConfig').definitionPaged, + mongotrimmed: require('../providers/stores/mongoConfig').definitionTrimmed, dispatch: require('../providers/stores/dispatchConfig') }, attachment: { diff --git a/providers/stores/abstractMongoDefinitionStore.js b/providers/stores/abstractMongoDefinitionStore.js new file mode 100644 index 000000000..20c05afef --- /dev/null +++ b/providers/stores/abstractMongoDefinitionStore.js @@ -0,0 +1,221 @@ +// (c) Copyright 2023, SAP SE and ClearlyDefined contributors. Licensed under the MIT license. +// SPDX-License-Identifier: MIT + +const MongoClient = require('mongodb').MongoClient +const promiseRetry = require('promise-retry') +const EntityCoordinates = require('../../lib/entityCoordinates') +const logger = require('../logging/logger') +const { get } = require('lodash') +const base64 = require('base-64') + +const sortOptions = { + type: ['coordinates.type'], + provider: ['coordinates.provider'], + name: ['coordinates.name', 'coordinates.revision'], + namespace: ['coordinates.namespace', 'coordinates.name', 'coordinates.revision'], + revision: ['coordinates.revision'], + license: ['licensed.declared'], + releaseDate: ['described.releaseDate'], + licensedScore: ['licensed.score.total'], + describedScore: ['described.score.total'], + effectiveScore: ['scores.effective'], + toolScore: ['scores.tool'] +} + +const valueTransformers = { + 'licensed.score.total': value => value && parseInt(value), + 'described.score.total': value => value && parseInt(value), + 'scores.effective': value => value && parseInt(value), + 'scores.tool': value => value && parseInt(value) +} + +const SEPARATOR = '&' + +class AbstractMongoDefinitionStore { + constructor(options) { + this.logger = options.logger || logger() + this.options = options + } + + initialize() { + return promiseRetry(async retry => { + try { + this.client = await MongoClient.connect(this.options.connectionString, { useNewUrlParser: true }) + this.db = this.client.db(this.options.dbName) + this.collection = this.db.collection(this.options.collectionName) + } catch (error) { + retry(error) + } + }) + } + + async close() { + await this.client.close() + } + + /** + * List all of the matching components for the given coordinates. + * Accepts partial coordinates. + * + * @param {EntityCoordinates} coordinates + * @returns A list of matching coordinates i.e. [ 'npm/npmjs/-/JSONStream/1.3.3' ] + */ + // eslint-disable-next-line no-unused-vars + async list(coordinates) { + throw new Error('Unsupported Operation') + } + + /** + * Get and return the object at the given coordinates. + * + * @param {Coordinates} coordinates - The coordinates of the object to get + * @returns The loaded object + */ + // eslint-disable-next-line no-unused-vars + async get(coordinates) { + throw new Error('Unsupported Operation') + } + + /** + * Query and return the objects based on the query + * + * @param {object} query - The filters and sorts for the request + * @returns The data and continuationToken if there is more results + */ + async find(query, continuationToken = '', pageSize = 100, projection) { + const sort = this._buildSort(query) + const combinedFilters = this._buildQueryWithPaging(query, continuationToken, sort) + this.logger.debug(`filter: ${JSON.stringify(combinedFilters)}\nsort: ${JSON.stringify(sort)}`) + const cursor = await this.collection.find(combinedFilters, { + projection, + sort, + limit: pageSize + }) + const data = await cursor.toArray() + continuationToken = this._getContinuationToken(pageSize, data, sort) + return { data, continuationToken } + } + + // eslint-disable-next-line no-unused-vars + async store(definition) { + throw new Error('Unsupported Operation') + } + + // eslint-disable-next-line no-unused-vars + async delete(coordinates) { + throw new Error('Unsupported Operation') + } + + getId(coordinates) { + if (!coordinates) return '' + return EntityCoordinates.fromObject(coordinates) + .toString() + .toLowerCase() + } + + buildQuery(parameters) { + const filter = { } + if (parameters.type) filter['coordinates.type'] = parameters.type + if (parameters.provider) filter['coordinates.provider'] = parameters.provider + if (parameters.namespace) filter['coordinates.namespace'] = parameters.namespace + if (parameters.name) filter['coordinates.name'] = parameters.name + if (parameters.type === null) filter['coordinates.type'] = null + if (parameters.provider === null) filter['coordinates.provider'] = null + if (parameters.name === null) filter['coordinates.name'] = null + if (parameters.namespace === null) filter['coordinates.namespace'] = null + if (parameters.license) filter['licensed.declared'] = parameters.license + if (parameters.releasedAfter) filter['described.releaseDate'] = { $gt: parameters.releasedAfter } + if (parameters.releasedBefore) filter['described.releaseDate'] = { $lt: parameters.releasedBefore } + if (parameters.minEffectiveScore) filter['scores.effective'] = { $gt: parseInt(parameters.minEffectiveScore) } + if (parameters.maxEffectiveScore) filter['scores.effective'] = { $lt: parseInt(parameters.maxEffectiveScore) } + if (parameters.minToolScore) filter['scores.tool'] = { $gt: parseInt(parameters.minToolScore) } + if (parameters.maxToolScore) filter['scores.tool'] = { $lt: parseInt(parameters.maxToolScore) } + if (parameters.minLicensedScore) filter['licensed.score.total'] = { $gt: parseInt(parameters.minLicensedScore) } + if (parameters.maxLicensedScore) filter['licensed.score.total'] = { $lt: parseInt(parameters.maxLicensedScore) } + if (parameters.minDescribedScore) filter['described.score.total'] = { $gt: parseInt(parameters.minDescribedScore) } + if (parameters.maxDescribedScore) filter['described.score.total'] = { $lt: parseInt(parameters.maxDescribedScore) } + return filter + } + + _buildSort(parameters) { + const sort = sortOptions[parameters.sort] || [] + const clause = {} + sort.forEach(item => clause[item] = parameters.sortDesc ? -1 : 1) + //Always sort ascending on partitionKey for continuation token + const coordinateKey = this.getCoordinatesKey() + clause[coordinateKey] = 1 + return clause + } + + _buildQueryWithPaging(query, continuationToken, sort) { + const filter = this.buildQuery(query) + const paginationFilter = this._buildPaginationQuery(continuationToken, sort) + return paginationFilter ? { $and: [filter, paginationFilter] } : filter + } + + _buildPaginationQuery(continuationToken, sort) { + if (!continuationToken.length) return + const queryExpressions = this._buildQueryExpressions(continuationToken, sort) + return queryExpressions.length <= 1 ? + queryExpressions [0] : + { $or: [ ...queryExpressions ] } + } + + _buildQueryExpressions(continuationToken, sort) { + const lastValues = base64.decode(continuationToken) + const sortValues = lastValues.split(SEPARATOR).map(value => value.length ? value : null) + + const queryExpressions = [] + const sortConditions = Object.entries(sort) + for (let nSorts = 1; nSorts <= sortConditions.length; nSorts++) { + const subList = sortConditions.slice(0, nSorts) + queryExpressions.push(this._buildQueryExpression(subList, sortValues)) + } + return queryExpressions + } + + _buildQueryExpression(sortConditions, sortValues) { + return sortConditions.reduce((filter, [sortField, sortDirection], index) => { + const transform = valueTransformers[sortField] + let sortValue = sortValues[index] + sortValue = transform ? transform(sortValue) : sortValue + const isLast = index === sortConditions.length - 1 + const filterForSort = this._buildQueryForSort(isLast, sortField, sortValue, sortDirection) + return { ...filter, ...filterForSort} + }, {}) + } + + _buildQueryForSort(isTieBreaker, sortField, sortValue, sortDirection) { + let operator = '$eq' + if (isTieBreaker) { + if (sortDirection === 1) { + operator = sortValue === null ? '$ne' : '$gt' + } else { + operator = '$lt' + } + } + const filter = { [sortField]: { [operator]: sortValue } } + + //Less than non null value should include null as well + if (operator === '$lt' && sortValue) { + return { + $or: [ + filter, + { [sortField]: null } + ] + } + } + return filter + } + + _getContinuationToken(pageSize, data, sortClause) { + if (data.length !== pageSize) return '' + const lastItem = data[data.length - 1] + const lastValues = Object.keys(sortClause) + .map(key => get(lastItem, key)) + .join(SEPARATOR) + return base64.encode(lastValues) + } + +} +module.exports = AbstractMongoDefinitionStore diff --git a/providers/stores/mongo.js b/providers/stores/mongo.js index 7b86ac24c..fa611ea36 100644 --- a/providers/stores/mongo.js +++ b/providers/stores/mongo.js @@ -1,53 +1,10 @@ // Copyright (c) Microsoft Corporation and others. Licensed under the MIT license. // SPDX-License-Identifier: MIT -const MongoClient = require('mongodb').MongoClient -const promiseRetry = require('promise-retry') -const EntityCoordinates = require('../../lib/entityCoordinates') -const logger = require('../logging/logger') const { clone, get, range } = require('lodash') -const base64 = require('base-64') +const AbstractMongoDefinitionStore = require('./abstractMongoDefinitionStore') -const sortOptions = { - type: ['coordinates.type'], - provider: ['coordinates.provider'], - name: ['coordinates.name', 'coordinates.revision'], - namespace: ['coordinates.namespace', 'coordinates.name', 'coordinates.revision'], - revision: ['coordinates.revision'], - license: ['licensed.declared'], - releaseDate: ['described.releaseDate'], - licensedScore: ['licensed.score.total'], - describedScore: ['described.score.total'], - effectiveScore: ['scores.effective'], - toolScore: ['scores.tool'] -} - -const valueTransformers = { - 'licensed.score.total': value => value && parseInt(value), - 'described.score.total': value => value && parseInt(value), - 'scores.effective': value => value && parseInt(value), - 'scores.tool': value => value && parseInt(value) -} - -const SEPARATOR = '&' - -class MongoStore { - constructor(options) { - this.logger = options.logger || logger() - this.options = options - } - - initialize() { - return promiseRetry(async retry => { - try { - this.client = await MongoClient.connect(this.options.connectionString, { useNewUrlParser: true }) - this.db = this.client.db(this.options.dbName) - this.collection = this.db.collection(this.options.collectionName) - } catch (error) { - retry(error) - } - }) - } +class MongoStore extends AbstractMongoDefinitionStore { /** * List all of the matching components for the given coordinates. @@ -58,7 +15,7 @@ class MongoStore { */ async list(coordinates) { const list = await this.collection.find( - { '_mongo.partitionKey': new RegExp(`^${this._getId(coordinates)}`), '_mongo.page': 1 }, + { '_mongo.partitionKey': new RegExp(`^${this.getId(coordinates)}`), '_mongo.page': 1 }, { projection: { _id: 1 } } ) return (await list.toArray()).map(entry => entry._id) @@ -72,7 +29,7 @@ class MongoStore { */ async get(coordinates) { const cursor = await this.collection.find( - { '_mongo.partitionKey': this._getId(coordinates) }, + { '_mongo.partitionKey': this.getId(coordinates) }, { projection: { _id: 0, _mongo: 0 }, sort: { '_mongo.page': 1 } } ) let definition @@ -90,25 +47,17 @@ class MongoStore { * @returns The data and continuationToken if there is more results */ async find(query, continuationToken = '', pageSize = 100) { - const sort = this._buildSort(query) - const combinedFilters = this._buildQueryWithPaging(query, continuationToken, sort) - this.logger.debug(`filter: ${JSON.stringify(combinedFilters)}\nsort: ${JSON.stringify(sort)}`) - const cursor = await this.collection.find(combinedFilters, { - projection: { _id: 0, files: 0 }, - sort: sort, - limit: pageSize - }) - const data = await cursor.toArray() - continuationToken = this._getContinuationToken(pageSize, data, sort) - data.forEach(def => { + const projection = { _id: 0, files: 0 } + const result = await super.find(query, continuationToken, pageSize, projection) + result.data.forEach(def => { delete def._mongo }) - return { data, continuationToken } + return result } async store(definition) { const pageSize = 1000 - definition._id = this._getId(definition.coordinates) + definition._id = this.getId(definition.coordinates) await this.collection.deleteMany({ '_mongo.partitionKey': definition._id }) const pages = Math.ceil((get(definition, 'files.length') || 1) / pageSize) const result = await this.collection.insertMany( @@ -136,122 +85,18 @@ class MongoStore { } async delete(coordinates) { - await this.collection.deleteMany({ '_mongo.partitionKey': this._getId(coordinates) }) + await this.collection.deleteMany({ '_mongo.partitionKey': this.getId(coordinates) }) return null } - - _getId(coordinates) { - if (!coordinates) return '' - return EntityCoordinates.fromObject(coordinates) - .toString() - .toLowerCase() - } - - _buildQuery(parameters) { - const filter = { '_mongo.page': 1 } // only get page 1 of each definition - if (parameters.type) filter['coordinates.type'] = parameters.type - if (parameters.provider) filter['coordinates.provider'] = parameters.provider - if (parameters.namespace) filter['coordinates.namespace'] = parameters.namespace - if (parameters.name) filter['coordinates.name'] = parameters.name - if (parameters.type === null) filter['coordinates.type'] = null - if (parameters.provider === null) filter['coordinates.provider'] = null - if (parameters.name === null) filter['coordinates.name'] = null - if (parameters.namespace === null) filter['coordinates.namespace'] = null - if (parameters.license) filter['licensed.declared'] = parameters.license - if (parameters.releasedAfter) filter['described.releaseDate'] = { $gt: parameters.releasedAfter } - if (parameters.releasedBefore) filter['described.releaseDate'] = { $lt: parameters.releasedBefore } - if (parameters.minEffectiveScore) filter['scores.effective'] = { $gt: parseInt(parameters.minEffectiveScore) } - if (parameters.maxEffectiveScore) filter['scores.effective'] = { $lt: parseInt(parameters.maxEffectiveScore) } - if (parameters.minToolScore) filter['scores.tool'] = { $gt: parseInt(parameters.minToolScore) } - if (parameters.maxToolScore) filter['scores.tool'] = { $lt: parseInt(parameters.maxToolScore) } - if (parameters.minLicensedScore) filter['licensed.score.total'] = { $gt: parseInt(parameters.minLicensedScore) } - if (parameters.maxLicensedScore) filter['licensed.score.total'] = { $lt: parseInt(parameters.maxLicensedScore) } - if (parameters.minDescribedScore) filter['described.score.total'] = { $gt: parseInt(parameters.minDescribedScore) } - if (parameters.maxDescribedScore) filter['described.score.total'] = { $lt: parseInt(parameters.maxDescribedScore) } - return filter - } - - _buildSort(parameters) { - const sort = sortOptions[parameters.sort] || [] - const clause = {} - sort.forEach(item => clause[item] = parameters.sortDesc ? -1 : 1) - //Always sort ascending on partitionKey for continuation token - clause['_mongo.partitionKey'] = 1 - return clause - } - - _buildQueryWithPaging(query, continuationToken, sort) { - const filter = this._buildQuery(query) - const paginationFilter = this._buildPaginationQuery(continuationToken, sort) - return paginationFilter ? { $and: [filter, paginationFilter] } : filter - } - - _buildPaginationQuery(continuationToken, sort) { - if (!continuationToken.length) return - const queryExpressions = this._buildQueryExpressions(continuationToken, sort) - return queryExpressions.length <= 1 ? - queryExpressions [0] : - { $or: [ ...queryExpressions ] } + + getCoordinatesKey() { + return '_mongo.partitionKey' } - - _buildQueryExpressions(continuationToken, sort) { - const lastValues = base64.decode(continuationToken) - const sortValues = lastValues.split(SEPARATOR).map(value => value.length ? value : null) - - const queryExpressions = [] - const sortConditions = Object.entries(sort) - for (let nSorts = 1; nSorts <= sortConditions.length; nSorts++) { - const subList = sortConditions.slice(0, nSorts) - queryExpressions.push(this._buildQueryExpression(subList, sortValues)) - } - return queryExpressions + + buildQuery(parameters) { + const filter = super.buildQuery(parameters) + return { ...filter, '_mongo.page': 1 } // only get page 1 of each definition } - _buildQueryExpression(sortConditions, sortValues) { - return sortConditions.reduce((filter, [sortField, sortDirection], index) => { - const transform = valueTransformers[sortField] - let sortValue = sortValues[index] - sortValue = transform ? transform(sortValue) : sortValue - const isLast = index === sortConditions.length - 1 - const filterForSort = this._buildQueryForSort(isLast, sortField, sortValue, sortDirection) - return { ...filter, ...filterForSort} - }, {}) - } - - _buildQueryForSort(isTieBreaker, sortField, sortValue, sortDirection) { - let operator = '$eq' - if (isTieBreaker) { - if (sortDirection === 1) { - operator = sortValue === null ? '$ne' : '$gt' - } else { - operator = '$lt' - } - } - const filter = { [sortField]: { [operator]: sortValue } } - - //Less than non null value should include null as well - if (operator === '$lt' && sortValue) { - return { - $or: [ - filter, - { [sortField]: null } - ] - } - } - return filter - } - - _getContinuationToken(pageSize, data, sortClause) { - if (data.length !== pageSize) return '' - const lastItem = data[data.length - 1] - const lastValues = Object.keys(sortClause) - .map(key => get(lastItem, key)) - .join(SEPARATOR) - return base64.encode(lastValues) - } - - async close() { - await this.client.close() - } } module.exports = options => new MongoStore(options) diff --git a/providers/stores/mongoConfig.js b/providers/stores/mongoConfig.js index 4d01cf8e2..4db0cfa28 100644 --- a/providers/stores/mongoConfig.js +++ b/providers/stores/mongoConfig.js @@ -2,15 +2,25 @@ // SPDX-License-Identifier: MIT const config = require('painless-config') const mongo = require('./mongo') +const TrimmedMongoDefinitionStore = require('./trimmedMongoDefinitionStore') -function definition(options) { - return mongo( - options || { - connectionString: config.get('DEFINITION_MONGO_CONNECTION_STRING'), - dbName: config.get('DEFINITION_MONGO_DB_NAME') || 'clearlydefined', - collectionName: config.get('DEFINITION_MONGO_COLLECTION_NAME') || 'definitions' - } - ) +const dbOptions = { + connectionString: config.get('DEFINITION_MONGO_CONNECTION_STRING'), + dbName: config.get('DEFINITION_MONGO_DB_NAME') || 'clearlydefined' } -module.exports = definition +function definitionPaged(options) { + return mongo(options || { + ...dbOptions, + collectionName: config.get('DEFINITION_MONGO_COLLECTION_NAME') || 'definitions' + }) +} + +function definitionTrimmed(options) { + return TrimmedMongoDefinitionStore(options || { + ...dbOptions, + collectionName: config.get('TRIMMED_DEFINITION_MONGO_COLLECTION_NAME') || 'definitions-trimmed' + }) +} + +module.exports = { definitionPaged, definitionTrimmed } diff --git a/providers/stores/trimmedMongoDefinitionStore.js b/providers/stores/trimmedMongoDefinitionStore.js new file mode 100644 index 000000000..8320d8d5e --- /dev/null +++ b/providers/stores/trimmedMongoDefinitionStore.js @@ -0,0 +1,43 @@ +// (c) Copyright 2023, SAP SE and ClearlyDefined contributors. Licensed under the MIT license. +// SPDX-License-Identifier: MIT + +const AbstractMongoDefinitionStore = require('./abstractMongoDefinitionStore') +const { clone } = require('lodash') + +class TrimmedMongoDefinitionStore extends AbstractMongoDefinitionStore { + // eslint-disable-next-line no-unused-vars + async list(coordinates) { + //This store does not support list for coordinates + return null + } + + // eslint-disable-next-line no-unused-vars + async get(coordinates) { + //This store does not support get definition + return null + } + + async find(query, continuationToken = '', pageSize) { + const result = await super.find(query, continuationToken, pageSize) + result.data.forEach((def) => delete def._id) + return result + } + + async store(definition) { + const definitionDoc = clone(definition) + definitionDoc._id = this.getId(definition.coordinates) + delete definitionDoc.files + return await this.collection.replaceOne({ _id: definitionDoc._id }, definitionDoc, { upsert: true }) + } + + async delete(coordinates) { + await this.collection.deleteOne({ _id: this.getId(coordinates) }) + return null + } + + getCoordinatesKey() { + return '_id' + } +} + +module.exports = options => new TrimmedMongoDefinitionStore(options) diff --git a/test/providers/store/mongoDefinition.js b/test/providers/store/mongoDefinition.js index 7a7d42e79..74a431828 100644 --- a/test/providers/store/mongoDefinition.js +++ b/test/providers/store/mongoDefinition.js @@ -125,7 +125,7 @@ describe('Mongo Definition store', () => { [{ maxToolScore: 50 }, { '_mongo.page': 1, 'scores.tool': { $lt: 50 } }] ]) data.forEach((expected, input) => { - expect(store._buildQuery(input)).to.deep.equal(expected) + expect(store.buildQuery(input)).to.deep.equal(expected) }) }) @@ -192,6 +192,20 @@ describe('Mongo Definition store', () => { ]) expect(token).to.eq('') }) + + it('should call find with right arguments', async () => { + const store = createStore() + store.collection.find = sinon.fake.resolves({ toArray: () => Promise.resolve([])}) + await store.find({ type: 'npm' }) + const filter = { 'coordinates.type': 'npm','_mongo.page': 1 } + const opts = { + 'projection': { '_id': 0, 'files': 0 }, + 'sort': { '_mongo.partitionKey': 1 }, + 'limit': 100 } + const findArgs = store.collection.find.firstCall.args + expect(findArgs[0]).to.be.deep.equal(filter) + expect(findArgs[1]).to.be.deep.equal(opts) + }) }) function createDefinition(coordinates) { @@ -224,7 +238,8 @@ function createStore(data) { insertMany: sinon.stub(), deleteMany: sinon.stub() } - const store = Store({}) + const opts = { logger: { debug: () => {} }} + const store = Store(opts) store.collection = collectionStub return store } diff --git a/test/providers/store/trimmedMongoDefinitionStore.js b/test/providers/store/trimmedMongoDefinitionStore.js new file mode 100644 index 000000000..a498ac5f0 --- /dev/null +++ b/test/providers/store/trimmedMongoDefinitionStore.js @@ -0,0 +1,202 @@ +// (c) Copyright 2023, SAP SE and ClearlyDefined contributors. Licensed under the MIT license. +// SPDX-License-Identifier: MIT + +const { expect } = require('chai') +const TrimmedMongoDefinitionStore = require('../../../providers/stores/trimmedMongoDefinitionStore') +const { MongoMemoryServer } = require('mongodb-memory-server') +const fsPromise = require('fs/promises') +const path = require('path') +const { uniq } = require('lodash') +const EntityCoordinates = require('../../../lib/entityCoordinates') +const sinon = require('sinon') +const PagedMongoDefinitionStore = require('../../../providers/stores/mongo') + +const dbOptions = { + dbName: 'clearlydefined', + collectionName: 'definitions-trimmed', + logger: { + debug: () => {}, + }, +} + +describe('Trimmed Mongo Definition store', () => { + const mongoServer = new MongoMemoryServer() + let mongoStore + + before('setup database', async () => { + await mongoServer.start() + const uri = await mongoServer.getUri() + const options = { + ...dbOptions, + connectionString: uri, + } + mongoStore = TrimmedMongoDefinitionStore(options) + await mongoStore.initialize() + }) + + after('cleanup database', async () => { + await mongoStore.close() + await mongoServer.stop() + }) + + beforeEach('setup collection', async () => { + mongoStore.collection = mongoStore.db.collection(mongoStore.options.collectionName) + await setupStore(mongoStore) + }) + + afterEach('cleanup collection', async () => { + await mongoStore.collection.drop() + }) + + it('should return falsy for get', async function () { + const coordinate = EntityCoordinates.fromString('maven/mavencentral/io.jenetics/jenetics/7.1.1') + const defs = await mongoStore.get(coordinate) + expect(defs).to.be.not.ok + }) + + it('should return falsy for list', async function () { + const coordinate = EntityCoordinates.fromString('maven/mavencentral/io.jenetics/jenetics') + const defs = await mongoStore.list(coordinate) + expect(defs).to.be.not.ok + }) + + it('should call replaceOne with the right arguments', async () => { + mongoStore.collection.replaceOne = sinon.fake.resolves() + const definition = createDefinition('npm/npmjs/-/foo/1.0') + await mongoStore.store(definition) + expect(mongoStore.collection.replaceOne.callCount).to.eq(1) + const args = mongoStore.collection.replaceOne.args[0] + expect(args[0]['_id']).to.eq('npm/npmjs/-/foo/1.0') + expect(args[1].files).to.be.not.ok + }) + + it('should store the definition', async () => { + const definition = createDefinition('npm/npmjs/-/foo/1.0') + await mongoStore.store(definition) + const defs = await mongoStore.find({ name: 'foo' }, '', 5) + expect(defs.data.length).to.be.eq(1) + const coordinates = EntityCoordinates.fromObject(defs.data[0].coordinates) + expect(coordinates.toString()).to.be.eq('npm/npmjs/-/foo/1.0') + }) + + it('should call deleteOne with the right arguments', async () => { + mongoStore.collection.deleteOne = sinon.fake.resolves() + await mongoStore.delete(EntityCoordinates.fromString('npm/npmjs/-/foo/1.0')) + expect(mongoStore.collection.deleteOne.callCount).to.eq(1) + expect(mongoStore.collection.deleteOne.args[0][0]['_id']).to.eq('npm/npmjs/-/foo/1.0') + }) + + it('should delete the definition', async () => { + let defs = await mongoStore.find({ name: 'jenetics' }, '', 5) + expect(defs.data.length).to.be.eq(1) + await mongoStore.delete(EntityCoordinates.fromString('maven/mavencentral/io.jenetics/jenetics/7.1.1')) + defs = await mongoStore.find({ name: 'jenetics' }, '', 5) + expect(defs.data.length).to.be.eq(0) + }) + + it('should call find with right arguments', async () => { + mongoStore.collection.find = sinon.fake.resolves({ toArray: () => Promise.resolve([]) }) + await mongoStore.find({ type: 'npm' }) + + const filter = { 'coordinates.type': 'npm' } + const opts = { + projection: undefined, + sort: { _id: 1 }, + limit: 100, + } + const findArgs = mongoStore.collection.find.firstCall.args + expect(findArgs[0]).to.be.deep.equal(filter) + expect(findArgs[1]).to.be.deep.equal(opts) + }) + + it('should find one page of records', async () => { + const expected = [ + 'git/github/microsoft/redie/194269b5b7010ad6f8dc4ef608c88128615031ca', + 'maven/mavencentral/com.azure/azure-storage-blob/12.20.0', + 'maven/mavencentral/io.jenetics/jenetics/7.1.1', + 'maven/mavencentral/io.quarkiverse.cxf/quarkus-cxf/1.5.4', + 'maven/mavencentral/org.apache.httpcomponents/httpcore/4.0-alpha5' + ] + const defs = await mongoStore.find({}, '', 5) + expect(defs.continuationToken).to.be.ok + expect(defs.data.length).to.be.equal(5) + expect(defs.data[0]._id).to.be.not.ok + const coordinates = verifyUniqueCoordinates(defs.data) + verifyExpectedCoordinates(coordinates, expected) + }) + + it('should find all with continuous token', async () => { + const query = {} + const defs1 = await mongoStore.find(query, '', 7) + expect(defs1.continuationToken).to.be.ok + const defs2 = await mongoStore.find(query, defs1.continuationToken, 7) + expect(defs2.continuationToken).to.be.not.ok + const allDefs = [ ...defs1.data, ...defs2.data] + const coordinates = verifyUniqueCoordinates(allDefs) + expect(coordinates.length).to.be.equal(12) + }) + + describe('trimmed vs. paged definition', () => { + let pagedMongoStore + + beforeEach('setup database', async () => { + const uri = await mongoServer.getUri() + const options = { + ...dbOptions, + connectionString: uri, + collectionName: 'definitions-paged', + } + pagedMongoStore = PagedMongoDefinitionStore(options) + await pagedMongoStore.initialize() + }) + + afterEach('cleanup database', async () => { + await pagedMongoStore.close() + }) + + it('should find definition same as paged definition', async () => { + const definition = createDefinition('npm/npmjs/-/foo/1.0') + const query = { name: 'foo' } + await pagedMongoStore.store(definition) + const expected = await pagedMongoStore.find(query, '', 2) + + await mongoStore.store(definition) + const actual = await mongoStore.find(query, '', 2) + expect(actual).to.be.deep.equal(expected) + }) + }) +}) + +async function setupStore(mongoStore) { + const fileName = path.join(__dirname, '../../fixtures/store/definitions-paged-no-files') + const content = await fsPromise.readFile(fileName) + const defDump = JSON.parse(content.toString()).map((def) => { + delete def._mongo + return def + }) + await mongoStore.collection.insertMany(defDump) +} + +function verifyExpectedCoordinates(allCoordinates, expected) { + const firstCoordinates = allCoordinates.slice(0, expected.length) + expect(firstCoordinates).to.be.deep.equal(expected) +} + +function verifyUniqueCoordinates(defs) { + const allCoordinates = defs.map((e) => EntityCoordinates.fromObject(e.coordinates).toString()) + const uniqTokens = uniq(allCoordinates) + expect(uniqTokens.length).to.be.equal(allCoordinates.length) + return allCoordinates +} + +function createDefinition(coordinates) { + coordinates = EntityCoordinates.fromString(coordinates) + return { + coordinates, + described: {}, + licensed: {}, + scores: {}, + _meta: {}, + files: [{ path: '1' }, { path: '2' }] + } +} \ No newline at end of file From a3d11b5e8846ae6f0f8f936478546c10965c9d51 Mon Sep 17 00:00:00 2001 From: Qing Tomlinson Date: Mon, 13 Feb 2023 16:10:19 -0800 Subject: [PATCH 2/4] Support perform in parallel in dispatchDefinitionStore --- providers/stores/dispatchDefinitionStore.js | 26 ++++++--- .../store/dispatchDefinitionStore.js | 53 +++++++++++++++++++ 2 files changed, 71 insertions(+), 8 deletions(-) create mode 100644 test/providers/store/dispatchDefinitionStore.js diff --git a/providers/stores/dispatchDefinitionStore.js b/providers/stores/dispatchDefinitionStore.js index ad92f310f..878a833a6 100644 --- a/providers/stores/dispatchDefinitionStore.js +++ b/providers/stores/dispatchDefinitionStore.js @@ -6,34 +6,34 @@ const logger = require('../logging/logger') class DispatchDefinitionStore { constructor(options) { this.stores = options.stores - this.logger = logger() + this.logger = options.logger || logger() } initialize() { - return this._perform(store => store.initialize()) + return this._performInParallel(store => store.initialize()) } get(coordinates) { - return this._perform(store => store.get(coordinates), true) + return this._performInSequence(store => store.get(coordinates)) } list(coordinates) { - return this._perform(store => store.list(coordinates), true) + return this._performInSequence(store => store.list(coordinates)) } store(definition) { - return this._perform(store => store.store(definition)) + return this._performInParallel(store => store.store(definition)) } delete(coordinates) { - return this._perform(store => store.delete(coordinates)) + return this._performInParallel(store => store.delete(coordinates)) } find(query, continuationToken = '') { - return this._perform(store => store.find(query, continuationToken), true) + return this._performInSequence(store => store.find(query, continuationToken)) } - async _perform(operation, first = false) { + async _performInSequence(operation, first = true) { let result = null for (let i = 0; i < this.stores.length; i++) { const store = this.stores[i] @@ -47,6 +47,16 @@ class DispatchDefinitionStore { } return result } + + async _performInParallel(operation) { + const opPromises = this.stores.map(store => operation(store)) + const results = await Promise.allSettled(opPromises) + results + .filter(result => result.status === 'rejected') + .forEach(result => this.logger.error('DispatchDefinitionStore failure', result.reason)) + const fulfilled = results.find(result => result.status === 'fulfilled') + return fulfilled?.value + } } module.exports = options => new DispatchDefinitionStore(options) diff --git a/test/providers/store/dispatchDefinitionStore.js b/test/providers/store/dispatchDefinitionStore.js new file mode 100644 index 000000000..ba128b2fe --- /dev/null +++ b/test/providers/store/dispatchDefinitionStore.js @@ -0,0 +1,53 @@ +// (c) Copyright 2023, SAP SE and ClearlyDefined contributors. Licensed under the MIT license. +// SPDX-License-Identifier: MIT + +const sinon = require('sinon') +const { expect } = require('chai') +const DispatchDefinitionStore = require('../../../providers/stores/dispatchDefinitionStore') + +describe('Dispatch Definition store', () => { + let dispatchDefinitionStore, store1, store2, logger + + beforeEach(() => { + store1 = createStore() + store2 = createStore() + logger = { error: sinon.stub() } + dispatchDefinitionStore = DispatchDefinitionStore({ + stores: [store1, store2], + logger + }) + }) + + it('should perform in sequence for get', async () => { + store1.get.resolves(1) + store2.get.resolves(2) + const result = await dispatchDefinitionStore.get('test') + expect(result).to.be.equal(1) + expect(store1.get.callCount).to.be.equal(1) + expect(store2.get.callCount).to.be.equal(0) + }) + + it('should initialize in parallel', async () => { + store1.initialize.resolves() + store2.initialize.resolves() + await dispatchDefinitionStore.initialize() + expect(store1.initialize.callCount).to.be.equal(1) + expect(store2.initialize.callCount).to.be.equal(1) + }) + + it('should perform in parallel and handle exception', async () => { + store1.initialize.resolves() + store2.initialize.rejects('store2 failed') + await dispatchDefinitionStore.initialize() + expect(store1.initialize.callCount).to.be.equal(1) + expect(store2.initialize.callCount).to.be.equal(1) + expect(logger.error.callCount).to.be.equal(1) + }) +}) + +function createStore() { + return { + get: sinon.stub(), + initialize: sinon.stub() + } +} From ffb4de2923ab0ba6b5dac0510da6e6ad45b053e7 Mon Sep 17 00:00:00 2001 From: Qing Tomlinson Date: Wed, 22 Feb 2023 10:17:57 -0800 Subject: [PATCH 3/4] Reorganize unit tests --- .../store/trimmedMongoDefinitionStore.js | 146 +++++++++--------- 1 file changed, 76 insertions(+), 70 deletions(-) diff --git a/test/providers/store/trimmedMongoDefinitionStore.js b/test/providers/store/trimmedMongoDefinitionStore.js index a498ac5f0..6c14e6e52 100644 --- a/test/providers/store/trimmedMongoDefinitionStore.js +++ b/test/providers/store/trimmedMongoDefinitionStore.js @@ -60,82 +60,88 @@ describe('Trimmed Mongo Definition store', () => { expect(defs).to.be.not.ok }) - it('should call replaceOne with the right arguments', async () => { - mongoStore.collection.replaceOne = sinon.fake.resolves() - const definition = createDefinition('npm/npmjs/-/foo/1.0') - await mongoStore.store(definition) - expect(mongoStore.collection.replaceOne.callCount).to.eq(1) - const args = mongoStore.collection.replaceOne.args[0] - expect(args[0]['_id']).to.eq('npm/npmjs/-/foo/1.0') - expect(args[1].files).to.be.not.ok - }) + describe('store', () => { + it('should call replaceOne with the right arguments', async () => { + mongoStore.collection.replaceOne = sinon.fake.resolves() + const definition = createDefinition('npm/npmjs/-/foo/1.0') + await mongoStore.store(definition) + expect(mongoStore.collection.replaceOne.callCount).to.eq(1) + const args = mongoStore.collection.replaceOne.args[0] + expect(args[0]['_id']).to.eq('npm/npmjs/-/foo/1.0') + expect(args[1].files).to.be.not.ok + }) - it('should store the definition', async () => { - const definition = createDefinition('npm/npmjs/-/foo/1.0') - await mongoStore.store(definition) - const defs = await mongoStore.find({ name: 'foo' }, '', 5) - expect(defs.data.length).to.be.eq(1) - const coordinates = EntityCoordinates.fromObject(defs.data[0].coordinates) - expect(coordinates.toString()).to.be.eq('npm/npmjs/-/foo/1.0') + it('should store the definition', async () => { + const definition = createDefinition('npm/npmjs/-/foo/1.0') + await mongoStore.store(definition) + const defs = await mongoStore.find({ name: 'foo' }, '') + expect(defs.data.length).to.be.eq(1) + const coordinates = EntityCoordinates.fromObject(defs.data[0].coordinates) + expect(coordinates.toString()).to.be.eq('npm/npmjs/-/foo/1.0') + }) }) - it('should call deleteOne with the right arguments', async () => { - mongoStore.collection.deleteOne = sinon.fake.resolves() - await mongoStore.delete(EntityCoordinates.fromString('npm/npmjs/-/foo/1.0')) - expect(mongoStore.collection.deleteOne.callCount).to.eq(1) - expect(mongoStore.collection.deleteOne.args[0][0]['_id']).to.eq('npm/npmjs/-/foo/1.0') - }) + describe('delete', () => { + it('should call deleteOne with the right arguments', async () => { + mongoStore.collection.deleteOne = sinon.fake.resolves() + await mongoStore.delete(EntityCoordinates.fromString('npm/npmjs/-/foo/1.0')) + expect(mongoStore.collection.deleteOne.callCount).to.eq(1) + expect(mongoStore.collection.deleteOne.args[0][0]['_id']).to.eq('npm/npmjs/-/foo/1.0') + }) - it('should delete the definition', async () => { - let defs = await mongoStore.find({ name: 'jenetics' }, '', 5) - expect(defs.data.length).to.be.eq(1) - await mongoStore.delete(EntityCoordinates.fromString('maven/mavencentral/io.jenetics/jenetics/7.1.1')) - defs = await mongoStore.find({ name: 'jenetics' }, '', 5) - expect(defs.data.length).to.be.eq(0) + it('should delete the definition', async () => { + let defs = await mongoStore.find({ name: 'jenetics' }, '') + expect(defs.data.length).to.be.eq(1) + await mongoStore.delete(EntityCoordinates.fromString('maven/mavencentral/io.jenetics/jenetics/7.1.1')) + defs = await mongoStore.find({ name: 'jenetics' }, '') + expect(defs.data.length).to.be.eq(0) + }) }) + + describe('find', () => { + it('should call find with right arguments', async () => { + mongoStore.collection.find = sinon.fake.resolves({ toArray: () => Promise.resolve([]) }) + await mongoStore.find({ type: 'npm' }) + + const filter = { 'coordinates.type': 'npm' } + const opts = { + projection: undefined, + sort: { _id: 1 }, + limit: 100, + } + const findArgs = mongoStore.collection.find.firstCall.args + expect(findArgs[0]).to.be.deep.equal(filter) + expect(findArgs[1]).to.be.deep.equal(opts) + }) + + it('should find one page of records', async () => { + const expected = [ + 'git/github/microsoft/redie/194269b5b7010ad6f8dc4ef608c88128615031ca', + 'maven/mavencentral/com.azure/azure-storage-blob/12.20.0', + 'maven/mavencentral/io.jenetics/jenetics/7.1.1', + 'maven/mavencentral/io.quarkiverse.cxf/quarkus-cxf/1.5.4', + 'maven/mavencentral/org.apache.httpcomponents/httpcore/4.0-alpha5' + ] + const defs = await mongoStore.find({}, '', 5) + expect(defs.continuationToken).to.be.ok + expect(defs.data.length).to.be.equal(5) + expect(defs.data[0]._id).to.be.not.ok + const coordinates = verifyUniqueCoordinates(defs.data) + verifyExpectedCoordinates(coordinates, expected) + }) - it('should call find with right arguments', async () => { - mongoStore.collection.find = sinon.fake.resolves({ toArray: () => Promise.resolve([]) }) - await mongoStore.find({ type: 'npm' }) - - const filter = { 'coordinates.type': 'npm' } - const opts = { - projection: undefined, - sort: { _id: 1 }, - limit: 100, - } - const findArgs = mongoStore.collection.find.firstCall.args - expect(findArgs[0]).to.be.deep.equal(filter) - expect(findArgs[1]).to.be.deep.equal(opts) - }) - - it('should find one page of records', async () => { - const expected = [ - 'git/github/microsoft/redie/194269b5b7010ad6f8dc4ef608c88128615031ca', - 'maven/mavencentral/com.azure/azure-storage-blob/12.20.0', - 'maven/mavencentral/io.jenetics/jenetics/7.1.1', - 'maven/mavencentral/io.quarkiverse.cxf/quarkus-cxf/1.5.4', - 'maven/mavencentral/org.apache.httpcomponents/httpcore/4.0-alpha5' - ] - const defs = await mongoStore.find({}, '', 5) - expect(defs.continuationToken).to.be.ok - expect(defs.data.length).to.be.equal(5) - expect(defs.data[0]._id).to.be.not.ok - const coordinates = verifyUniqueCoordinates(defs.data) - verifyExpectedCoordinates(coordinates, expected) + it('should find all with continuous token', async () => { + const query = {} + const defs1 = await mongoStore.find(query, '', 7) + expect(defs1.continuationToken).to.be.ok + const defs2 = await mongoStore.find(query, defs1.continuationToken, 7) + expect(defs2.continuationToken).to.be.not.ok + const allDefs = [ ...defs1.data, ...defs2.data] + const coordinates = verifyUniqueCoordinates(allDefs) + expect(coordinates.length).to.be.equal(12) + }) }) - it('should find all with continuous token', async () => { - const query = {} - const defs1 = await mongoStore.find(query, '', 7) - expect(defs1.continuationToken).to.be.ok - const defs2 = await mongoStore.find(query, defs1.continuationToken, 7) - expect(defs2.continuationToken).to.be.not.ok - const allDefs = [ ...defs1.data, ...defs2.data] - const coordinates = verifyUniqueCoordinates(allDefs) - expect(coordinates.length).to.be.equal(12) - }) - describe('trimmed vs. paged definition', () => { let pagedMongoStore @@ -158,10 +164,10 @@ describe('Trimmed Mongo Definition store', () => { const definition = createDefinition('npm/npmjs/-/foo/1.0') const query = { name: 'foo' } await pagedMongoStore.store(definition) - const expected = await pagedMongoStore.find(query, '', 2) + const expected = await pagedMongoStore.find(query, '') await mongoStore.store(definition) - const actual = await mongoStore.find(query, '', 2) + const actual = await mongoStore.find(query, '') expect(actual).to.be.deep.equal(expected) }) }) From 9e7a58ca37f8e3ec0f3b05d1174fa791af6e8c7b Mon Sep 17 00:00:00 2001 From: Qing Tomlinson Date: Thu, 23 Feb 2023 12:36:57 -0800 Subject: [PATCH 4/4] Renaming and add pagination test Refactor mongo definition search pagination test to be reused for both paged and trimmed mongo definition store tests --- providers/index.js | 2 +- .../store/mongoDefinitionPagination.js | 395 +++++++++--------- .../store/pagedDefinitionPagination.js | 26 ++ .../store/trimmedDefinitionPagination.js | 26 ++ 4 files changed, 242 insertions(+), 207 deletions(-) create mode 100644 test/providers/store/pagedDefinitionPagination.js create mode 100644 test/providers/store/trimmedDefinitionPagination.js diff --git a/providers/index.js b/providers/index.js index 885f69a90..4471581fd 100644 --- a/providers/index.js +++ b/providers/index.js @@ -9,7 +9,7 @@ module.exports = { azure: require('../providers/stores/azblobConfig').definition, file: require('../providers/stores/fileConfig').definition, mongo: require('../providers/stores/mongoConfig').definitionPaged, - mongotrimmed: require('../providers/stores/mongoConfig').definitionTrimmed, + mongoTrimmed: require('../providers/stores/mongoConfig').definitionTrimmed, dispatch: require('../providers/stores/dispatchConfig') }, attachment: { diff --git a/test/providers/store/mongoDefinitionPagination.js b/test/providers/store/mongoDefinitionPagination.js index bb3beb6af..5b28b1318 100644 --- a/test/providers/store/mongoDefinitionPagination.js +++ b/test/providers/store/mongoDefinitionPagination.js @@ -2,7 +2,6 @@ // SPDX-License-Identifier: MIT const { expect } = require('chai') -const Store = require('../../../providers/stores/mongo') const EntityCoordinates = require('../../../lib/entityCoordinates') const { MongoMemoryServer } = require('mongodb-memory-server') const fsPromise = require('fs/promises') @@ -11,226 +10,208 @@ const { uniq } = require('lodash') const dbOptions = { dbName: 'clearlydefined', - collectionName: 'definitions-paged', logger: { debug: () => {} } } -describe('Mongo Definition store: search pagination', () => { - const mongoServer = new MongoMemoryServer() - let mongoStore +const shouldPaginateSearchCorrectly = function() { - before('setup database', async () => { - await mongoServer.start() - const uri = await mongoServer.getUri() - const options = { - ...dbOptions, - connectionString: uri - } - mongoStore = Store(options) - await mongoStore.initialize() - - await setupStore(mongoStore) - }) - - after('cleanup database', async () => { - await mongoStore.collection.drop() - await mongoStore.close() - await mongoServer.stop() - }) - - it('should fetch records without sort continuously', async function() { - //filter: {'_mongo.page': 1} - //sort: {'_mongo.partitionKey': 1} - const expected = [ - 'git/github/microsoft/redie/194269b5b7010ad6f8dc4ef608c88128615031ca', - 'maven/mavencentral/com.azure/azure-storage-blob/12.20.0', - 'maven/mavencentral/io.jenetics/jenetics/7.1.1', - 'maven/mavencentral/io.quarkiverse.cxf/quarkus-cxf/1.5.4', - 'maven/mavencentral/org.apache.httpcomponents/httpcore/4.0-alpha5' - ] - const query = {} - const defs = await fetchAll(mongoStore, query, 5) - expect(defs.length).to.be.equal(12) - const coordinates = verifyUniqueCoordinates(defs) - verifyExpectedCoordinates(coordinates, expected) - }) - - it('should sort ascending on releaseDate and find 1 page of records', async function() { - //filter: {'_mongo.page': 1} - //sort: {'described.releaseDate': 1,'_mongo.partitionKey': 1} - const expected = ['maven/mavencentral/org.apache.httpcomponents/httpcore/4.0-alpha5'] - const query = { - sort: 'releaseDate', - sortDesc: false - } - const defs = await fetchUpToNTimes(mongoStore, query, 1) - expect(defs.length).to.be.equal(1) - const coordinates = verifyUniqueCoordinates(defs) - verifyExpectedCoordinates(coordinates, expected) - }) - - it('should sort ascending on releaseDate and handle null and non null values in continuation', async function() { - //filter: {'_mongo.page': 1} - //sort: {'described.releaseDate': 1, '_mongo.partitionKey': 1} - const expected = [ - 'maven/mavencentral/org.apache.httpcomponents/httpcore/4.0-alpha5', - 'maven/mavencentral/org.flywaydb/flyway-maven-plugin/5.0.7', - 'npm/npmjs/@sinclair/typebox/0.24.45', - 'maven/mavencentral/org.apache.httpcomponents/httpcore/4.0-beta2', - 'pypi/pypi/-/backports.ssl_match_hostname/3.5.0.1'] + describe('Mongo Definition Store: search pagination', function() { + const mongoServer = new MongoMemoryServer() + let mongoStore + + before('setup database', async function() { + await mongoServer.start() + const uri = await mongoServer.getUri() + const options = { + ...dbOptions, + connectionString: uri + } + const defs = await loadDefinitions() + mongoStore = await this.createStore(options, defs) + }) - const query = { - sort: 'releaseDate', - sortDesc: false - } - const defs = await fetchAll(mongoStore, query) - expect(defs[0].described.releaseDate).not.to.be.ok - expect(defs[3].described.releaseDate).to.be.ok + after('cleanup database', async function() { + await mongoStore.collection.drop() + await mongoStore.close() + await mongoServer.stop() + }) - expect(defs.length).to.be.equal(12) - const coordinates = verifyUniqueCoordinates(defs) - verifyExpectedCoordinates(coordinates, expected) - }) + it('should fetch records without sort continuously', async function() { + const expected = [ + 'git/github/microsoft/redie/194269b5b7010ad6f8dc4ef608c88128615031ca', + 'maven/mavencentral/com.azure/azure-storage-blob/12.20.0', + 'maven/mavencentral/io.jenetics/jenetics/7.1.1', + 'maven/mavencentral/io.quarkiverse.cxf/quarkus-cxf/1.5.4', + 'maven/mavencentral/org.apache.httpcomponents/httpcore/4.0-alpha5' + ] + const query = {} + const defs = await fetchAll(mongoStore, query, 5) + expect(defs.length).to.be.equal(12) + const coordinates = verifyUniqueCoordinates(defs) + verifyExpectedCoordinates(coordinates, expected) + }) - it('should sort descending on releaseDate and handle null and non null values in continuation', async function() { - const query = { - sort: 'releaseDate', - sortDesc: true - } - const defs = await fetchAll(mongoStore, query) - expect(defs.length).to.be.equal(12) - verifyUniqueCoordinates(defs) - }) - - it('should sort ascending on license and handle null and non null values in continuation ', async function() { - //filter: {'_mongo.page': 1} - //sort: {'licensed.declared': 1, '_mongo.partitionKey': 1} - const expected = [ - 'npm/npmjs/@sinclair/typebox/0.24.45', - 'maven/mavencentral/io.jenetics/jenetics/7.1.1', - 'maven/mavencentral/io.quarkiverse.cxf/quarkus-cxf/1.5.4', - 'maven/mavencentral/org.apache.httpcomponents/httpcore/4.0-alpha5', - 'maven/mavencentral/org.apache.httpcomponents/httpcore/4.0-beta2'] - - const query = { - sort: 'license', - sortDesc: false - } - const defs = await fetchAll(mongoStore, query) - expect(defs[0].described.releaseDate).not.to.be.ok - expect(defs[1].described.releaseDate).to.be.ok - expect(defs.length).to.be.equal(12) - const coordinates = verifyUniqueCoordinates(defs) - verifyExpectedCoordinates(coordinates, expected) - }) - - it('should sort descending on license and handle null and non null values in continuation ', async function() { - const query = { - sort: 'license', - sortDesc: true - } - const defs = await fetchAll(mongoStore, query) - expect(defs.length).to.be.equal(12) - verifyUniqueCoordinates(defs) - }) - - it('should filter and sort ascending on multiple keys and handle null and non null namespace in continuation', async function() { - //filter: {'licensed.declared': 'MIT', '_mongo.page': 1} - //sort: {'coordinates.namespace': 1, 'coordinates.name':1, 'coordinates.revision': 1, '_mongo.partitionKey': 1} - const expected = [ - 'npm/npmjs/-/angular/1.6.9', - 'npm/npmjs/-/redie/0.3.0', - 'maven/mavencentral/com.azure/azure-storage-blob/12.20.0', - 'git/github/microsoft/redie/194269b5b7010ad6f8dc4ef608c88128615031ca'] - - const query = { - license: 'MIT', - sort: 'namespace', - sortDesc: false - } - const defs = await fetchAll(mongoStore, query) - expect(defs[0].coordinates.namespace).not.to.be.ok - expect(defs[2].coordinates.namespace).to.be.ok - expect(defs.length).to.be.equal(4) - const coordinates = verifyUniqueCoordinates(defs) - verifyExpectedCoordinates(coordinates, expected) - }) - - it('should filter and sort descending on multiple keys in continuation', async function() { - //filter: {'licensed.declared': 'MIT', '_mongo.page': 1} - //sort: {'coordinates.namespace': -1, 'coordinates.name':-1, 'coordinates.revision': -1, '_mongo.partitionKey': 1} - const expected = [ - 'git/github/microsoft/redie/194269b5b7010ad6f8dc4ef608c88128615031ca', - 'maven/mavencentral/com.azure/azure-storage-blob/12.20.0', - 'npm/npmjs/-/redie/0.3.0', - 'npm/npmjs/-/angular/1.6.9'] - - const query = { - license: 'MIT', - sort: 'namespace', - sortDesc: true - } - const defs = await fetchAll(mongoStore, query) - expect(defs.length).to.be.equal(4) - const coordinates = verifyUniqueCoordinates(defs) - verifyExpectedCoordinates(coordinates, expected) - }) - - it('should filter and sort on numerical scores and fetch continuously', async function() { - //filter: {'licensed.declared': 'MIT', '_mongo.page': 1} - //sort: {'scores.tool': 1, '_mongo.partitionKey': 1} - const expected = [ - 'maven/mavencentral/com.azure/azure-storage-blob/12.20.0', - 'npm/npmjs/-/angular/1.6.9', - 'git/github/microsoft/redie/194269b5b7010ad6f8dc4ef608c88128615031ca', - 'npm/npmjs/-/redie/0.3.0'] + it('should sort ascending on releaseDate and find 1 page of records', async function() { + const expected = ['maven/mavencentral/org.apache.httpcomponents/httpcore/4.0-alpha5'] + const query = { + sort: 'releaseDate', + sortDesc: false + } + const defs = await fetchUpToNTimes(mongoStore, query, 1) + expect(defs.length).to.be.equal(1) + const coordinates = verifyUniqueCoordinates(defs) + verifyExpectedCoordinates(coordinates, expected) + }) - const query = { - license: 'MIT', - sort: 'toolScore', - sortDesc: false - } - const defs = await fetchAll(mongoStore, query) - expect(defs.length).to.be.equal(4) - expect(defs[0].scores.tool).to.be.equal(80) - expect(defs[1].scores.tool).to.be.equal(84) - expect(defs[2].scores.tool).to.be.equal(90) - expect(defs[3].scores.tool).to.be.equal(94) - const coordinates = verifyUniqueCoordinates(defs) - verifyExpectedCoordinates(coordinates, expected) - }) - - it('should filter and sort descending on numerical scores and fetch continuously', async function() { - //filter: {'licensed.declared': 'MIT', '_mongo.page': 1} - //sort: {'scores.tool': -1, '_mongo.partitionKey': 1} - const expected = [ - 'npm/npmjs/-/redie/0.3.0', - 'git/github/microsoft/redie/194269b5b7010ad6f8dc4ef608c88128615031ca', - 'npm/npmjs/-/angular/1.6.9', - 'maven/mavencentral/com.azure/azure-storage-blob/12.20.0' - ] + it('should sort ascending on releaseDate and handle null and non null values in continuation', async function() { + const expected = [ + 'maven/mavencentral/org.apache.httpcomponents/httpcore/4.0-alpha5', + 'maven/mavencentral/org.flywaydb/flyway-maven-plugin/5.0.7', + 'npm/npmjs/@sinclair/typebox/0.24.45', + 'maven/mavencentral/org.apache.httpcomponents/httpcore/4.0-beta2', + 'pypi/pypi/-/backports.ssl_match_hostname/3.5.0.1'] + + const query = { + sort: 'releaseDate', + sortDesc: false + } + const defs = await fetchAll(mongoStore, query) + expect(defs[0].described.releaseDate).not.to.be.ok + expect(defs[3].described.releaseDate).to.be.ok + + expect(defs.length).to.be.equal(12) + const coordinates = verifyUniqueCoordinates(defs) + verifyExpectedCoordinates(coordinates, expected) + }) + + it('should sort descending on releaseDate and handle null and non null values in continuation', async function() { + const query = { + sort: 'releaseDate', + sortDesc: true + } + const defs = await fetchAll(mongoStore, query) + expect(defs.length).to.be.equal(12) + verifyUniqueCoordinates(defs) + }) + + it('should sort ascending on license and handle null and non null values in continuation ', async function() { + const expected = [ + 'npm/npmjs/@sinclair/typebox/0.24.45', + 'maven/mavencentral/io.jenetics/jenetics/7.1.1', + 'maven/mavencentral/io.quarkiverse.cxf/quarkus-cxf/1.5.4', + 'maven/mavencentral/org.apache.httpcomponents/httpcore/4.0-alpha5', + 'maven/mavencentral/org.apache.httpcomponents/httpcore/4.0-beta2'] + + const query = { + sort: 'license', + sortDesc: false + } + const defs = await fetchAll(mongoStore, query) + expect(defs[0].described.releaseDate).not.to.be.ok + expect(defs[1].described.releaseDate).to.be.ok + expect(defs.length).to.be.equal(12) + const coordinates = verifyUniqueCoordinates(defs) + verifyExpectedCoordinates(coordinates, expected) + }) + + it('should sort descending on license and handle null and non null values in continuation ', async function() { + const query = { + sort: 'license', + sortDesc: true + } + const defs = await fetchAll(mongoStore, query) + expect(defs.length).to.be.equal(12) + verifyUniqueCoordinates(defs) + }) + + it('should filter and sort ascending on multiple keys and handle null and non null namespace in continuation', async function() { + const expected = [ + 'npm/npmjs/-/angular/1.6.9', + 'npm/npmjs/-/redie/0.3.0', + 'maven/mavencentral/com.azure/azure-storage-blob/12.20.0', + 'git/github/microsoft/redie/194269b5b7010ad6f8dc4ef608c88128615031ca'] + + const query = { + license: 'MIT', + sort: 'namespace', + sortDesc: false + } + const defs = await fetchAll(mongoStore, query) + expect(defs[0].coordinates.namespace).not.to.be.ok + expect(defs[2].coordinates.namespace).to.be.ok + expect(defs.length).to.be.equal(4) + const coordinates = verifyUniqueCoordinates(defs) + verifyExpectedCoordinates(coordinates, expected) + }) + + it('should filter and sort descending on multiple keys in continuation', async function() { + const expected = [ + 'git/github/microsoft/redie/194269b5b7010ad6f8dc4ef608c88128615031ca', + 'maven/mavencentral/com.azure/azure-storage-blob/12.20.0', + 'npm/npmjs/-/redie/0.3.0', + 'npm/npmjs/-/angular/1.6.9'] + + const query = { + license: 'MIT', + sort: 'namespace', + sortDesc: true + } + const defs = await fetchAll(mongoStore, query) + expect(defs.length).to.be.equal(4) + const coordinates = verifyUniqueCoordinates(defs) + verifyExpectedCoordinates(coordinates, expected) + }) + + it('should filter and sort on numerical scores and fetch continuously', async function() { + const expected = [ + 'maven/mavencentral/com.azure/azure-storage-blob/12.20.0', + 'npm/npmjs/-/angular/1.6.9', + 'git/github/microsoft/redie/194269b5b7010ad6f8dc4ef608c88128615031ca', + 'npm/npmjs/-/redie/0.3.0'] + + const query = { + license: 'MIT', + sort: 'toolScore', + sortDesc: false + } + const defs = await fetchAll(mongoStore, query) + expect(defs.length).to.be.equal(4) + expect(defs[0].scores.tool).to.be.equal(80) + expect(defs[1].scores.tool).to.be.equal(84) + expect(defs[2].scores.tool).to.be.equal(90) + expect(defs[3].scores.tool).to.be.equal(94) + const coordinates = verifyUniqueCoordinates(defs) + verifyExpectedCoordinates(coordinates, expected) + }) + + it('should filter and sort descending on numerical scores and fetch continuously', async function() { + const expected = [ + 'npm/npmjs/-/redie/0.3.0', + 'git/github/microsoft/redie/194269b5b7010ad6f8dc4ef608c88128615031ca', + 'npm/npmjs/-/angular/1.6.9', + 'maven/mavencentral/com.azure/azure-storage-blob/12.20.0' + ] + + const query = { + license: 'MIT', + sort: 'toolScore', + sortDesc: true + } + const defs = await fetchAll(mongoStore, query) + expect(defs.length).to.be.equal(4) + const coordinates = verifyUniqueCoordinates(defs) + verifyExpectedCoordinates(coordinates, expected) + }) - const query = { - license: 'MIT', - sort: 'toolScore', - sortDesc: true - } - const defs = await fetchAll(mongoStore, query) - expect(defs.length).to.be.equal(4) - const coordinates = verifyUniqueCoordinates(defs) - verifyExpectedCoordinates(coordinates, expected) }) +} -}) - -async function setupStore(mongoStore) { +async function loadDefinitions() { const fileName = path.join(__dirname, '../../fixtures/store/definitions-paged-no-files') const content = await fsPromise.readFile(fileName) - const defDump = JSON.parse(content.toString()) - await mongoStore.collection.createIndex({ '_mongo.partitionKey': 1 }) - await mongoStore.collection.insertMany(defDump) + return JSON.parse(content.toString()) } function verifyExpectedCoordinates(allCoordinates, expected) { @@ -300,3 +281,5 @@ class ContinousFetch { return fetchedCounter } } + +module.exports = shouldPaginateSearchCorrectly \ No newline at end of file diff --git a/test/providers/store/pagedDefinitionPagination.js b/test/providers/store/pagedDefinitionPagination.js new file mode 100644 index 000000000..f0c409d79 --- /dev/null +++ b/test/providers/store/pagedDefinitionPagination.js @@ -0,0 +1,26 @@ +// (c) Copyright 2023, SAP SE and ClearlyDefined contributors. Licensed under the MIT license. +// SPDX-License-Identifier: MIT + +const Store = require('../../../providers/stores/mongo') +const shouldPaginateSearchCorrectly = require('./mongoDefinitionPagination') + +const dbOptions = { + collectionName: 'definitions-paged', +} + +describe('Mongo Definition Store: Paged', function() { + + before('setup store factory', async function() { + this.createStore = createStore + }) + + shouldPaginateSearchCorrectly() +}) + +async function createStore(options, defs) { + const mongoStore = Store({ ...options, ...dbOptions }) + await mongoStore.initialize() + await mongoStore.collection.createIndex({ '_mongo.partitionKey': 1 }) + await mongoStore.collection.insertMany(defs) + return mongoStore +} diff --git a/test/providers/store/trimmedDefinitionPagination.js b/test/providers/store/trimmedDefinitionPagination.js new file mode 100644 index 000000000..95af5d9cf --- /dev/null +++ b/test/providers/store/trimmedDefinitionPagination.js @@ -0,0 +1,26 @@ +// (c) Copyright 2023, SAP SE and ClearlyDefined contributors. Licensed under the MIT license. +// SPDX-License-Identifier: MIT + +const Store = require('../../../providers/stores/trimmedMongoDefinitionStore') +const shouldPaginateSearchCorrectly = require('./mongoDefinitionPagination') + +const dbOptions = { + collectionName: 'definitions-trimmed' +} + +describe('Mongo Definition Store: Trimmed', function() { + + before('setup store factory', async function() { + this.createStore = createStore + }) + + shouldPaginateSearchCorrectly() +}) + +async function createStore(options, defs) { + const mongoStore = Store({ ...options, ...dbOptions }) + await mongoStore.initialize() + defs.forEach(def => delete def._mongo) + await mongoStore.collection.insertMany(defs) + return mongoStore +}