diff --git a/providers/index.js b/providers/index.js index 885f69a90..4471581fd 100644 --- a/providers/index.js +++ b/providers/index.js @@ -9,7 +9,7 @@ module.exports = { azure: require('../providers/stores/azblobConfig').definition, file: require('../providers/stores/fileConfig').definition, mongo: require('../providers/stores/mongoConfig').definitionPaged, - mongotrimmed: require('../providers/stores/mongoConfig').definitionTrimmed, + mongoTrimmed: require('../providers/stores/mongoConfig').definitionTrimmed, dispatch: require('../providers/stores/dispatchConfig') }, attachment: { diff --git a/test/providers/store/mongoDefinitionPagination.js b/test/providers/store/mongoDefinitionPagination.js index bb3beb6af..5c81ab02a 100644 --- a/test/providers/store/mongoDefinitionPagination.js +++ b/test/providers/store/mongoDefinitionPagination.js @@ -2,235 +2,181 @@ // SPDX-License-Identifier: MIT const { expect } = require('chai') -const Store = require('../../../providers/stores/mongo') const EntityCoordinates = require('../../../lib/entityCoordinates') -const { MongoMemoryServer } = require('mongodb-memory-server') -const fsPromise = require('fs/promises') -const path = require('path') const { uniq } = require('lodash') -const dbOptions = { - dbName: 'clearlydefined', - collectionName: 'definitions-paged', - logger: { - debug: () => {} - } -} - -describe('Mongo Definition store: search pagination', () => { - const mongoServer = new MongoMemoryServer() - let mongoStore - - before('setup database', async () => { - await mongoServer.start() - const uri = await mongoServer.getUri() - const options = { - ...dbOptions, - connectionString: uri - } - mongoStore = Store(options) - await mongoStore.initialize() - - await setupStore(mongoStore) - }) - - after('cleanup database', async () => { - await mongoStore.collection.drop() - await mongoStore.close() - await mongoServer.stop() - }) - - it('should fetch records without sort continuously', async function() { - //filter: {'_mongo.page': 1} - //sort: {'_mongo.partitionKey': 1} - const expected = [ - 'git/github/microsoft/redie/194269b5b7010ad6f8dc4ef608c88128615031ca', - 'maven/mavencentral/com.azure/azure-storage-blob/12.20.0', - 'maven/mavencentral/io.jenetics/jenetics/7.1.1', - 'maven/mavencentral/io.quarkiverse.cxf/quarkus-cxf/1.5.4', - 'maven/mavencentral/org.apache.httpcomponents/httpcore/4.0-alpha5' - ] - const query = {} - const defs = await fetchAll(mongoStore, query, 5) - expect(defs.length).to.be.equal(12) - const coordinates = verifyUniqueCoordinates(defs) - verifyExpectedCoordinates(coordinates, expected) - }) - - it('should sort ascending on releaseDate and find 1 page of records', async function() { - //filter: {'_mongo.page': 1} - //sort: {'described.releaseDate': 1,'_mongo.partitionKey': 1} - const expected = ['maven/mavencentral/org.apache.httpcomponents/httpcore/4.0-alpha5'] - const query = { - sort: 'releaseDate', - sortDesc: false - } - const defs = await fetchUpToNTimes(mongoStore, query, 1) - expect(defs.length).to.be.equal(1) - const coordinates = verifyUniqueCoordinates(defs) - verifyExpectedCoordinates(coordinates, expected) - }) - - it('should sort ascending on releaseDate and handle null and non null values in continuation', async function() { - //filter: {'_mongo.page': 1} - //sort: {'described.releaseDate': 1, '_mongo.partitionKey': 1} - const expected = [ - 'maven/mavencentral/org.apache.httpcomponents/httpcore/4.0-alpha5', - 'maven/mavencentral/org.flywaydb/flyway-maven-plugin/5.0.7', - 'npm/npmjs/@sinclair/typebox/0.24.45', - 'maven/mavencentral/org.apache.httpcomponents/httpcore/4.0-beta2', - 'pypi/pypi/-/backports.ssl_match_hostname/3.5.0.1'] - - const query = { - sort: 'releaseDate', - sortDesc: false - } - const defs = await fetchAll(mongoStore, query) - expect(defs[0].described.releaseDate).not.to.be.ok - expect(defs[3].described.releaseDate).to.be.ok - - expect(defs.length).to.be.equal(12) - const coordinates = verifyUniqueCoordinates(defs) - verifyExpectedCoordinates(coordinates, expected) - }) +const shouldPaginateSearchCorrectly = function() { + + describe('Mongo Definition Store: search pagination', function() { + + it('should fetch records without sort continuously', async function() { + const expected = [ + 'git/github/microsoft/redie/194269b5b7010ad6f8dc4ef608c88128615031ca', + 'maven/mavencentral/com.azure/azure-storage-blob/12.20.0', + 'maven/mavencentral/io.jenetics/jenetics/7.1.1', + 'maven/mavencentral/io.quarkiverse.cxf/quarkus-cxf/1.5.4', + 'maven/mavencentral/org.apache.httpcomponents/httpcore/4.0-alpha5' + ] + const query = {} + const defs = await fetchAll(this.mongoStore, query, 5) + expect(defs.length).to.be.equal(12) + const coordinates = verifyUniqueCoordinates(defs) + verifyExpectedCoordinates(coordinates, expected) + }) - it('should sort descending on releaseDate and handle null and non null values in continuation', async function() { - const query = { - sort: 'releaseDate', - sortDesc: true - } - const defs = await fetchAll(mongoStore, query) - expect(defs.length).to.be.equal(12) - verifyUniqueCoordinates(defs) - }) - - it('should sort ascending on license and handle null and non null values in continuation ', async function() { - //filter: {'_mongo.page': 1} - //sort: {'licensed.declared': 1, '_mongo.partitionKey': 1} - const expected = [ - 'npm/npmjs/@sinclair/typebox/0.24.45', - 'maven/mavencentral/io.jenetics/jenetics/7.1.1', - 'maven/mavencentral/io.quarkiverse.cxf/quarkus-cxf/1.5.4', - 'maven/mavencentral/org.apache.httpcomponents/httpcore/4.0-alpha5', - 'maven/mavencentral/org.apache.httpcomponents/httpcore/4.0-beta2'] - - const query = { - sort: 'license', - sortDesc: false - } - const defs = await fetchAll(mongoStore, query) - expect(defs[0].described.releaseDate).not.to.be.ok - expect(defs[1].described.releaseDate).to.be.ok - expect(defs.length).to.be.equal(12) - const coordinates = verifyUniqueCoordinates(defs) - verifyExpectedCoordinates(coordinates, expected) - }) - - it('should sort descending on license and handle null and non null values in continuation ', async function() { - const query = { - sort: 'license', - sortDesc: true - } - const defs = await fetchAll(mongoStore, query) - expect(defs.length).to.be.equal(12) - verifyUniqueCoordinates(defs) - }) - - it('should filter and sort ascending on multiple keys and handle null and non null namespace in continuation', async function() { - //filter: {'licensed.declared': 'MIT', '_mongo.page': 1} - //sort: {'coordinates.namespace': 1, 'coordinates.name':1, 'coordinates.revision': 1, '_mongo.partitionKey': 1} - const expected = [ - 'npm/npmjs/-/angular/1.6.9', - 'npm/npmjs/-/redie/0.3.0', - 'maven/mavencentral/com.azure/azure-storage-blob/12.20.0', - 'git/github/microsoft/redie/194269b5b7010ad6f8dc4ef608c88128615031ca'] - - const query = { - license: 'MIT', - sort: 'namespace', - sortDesc: false - } - const defs = await fetchAll(mongoStore, query) - expect(defs[0].coordinates.namespace).not.to.be.ok - expect(defs[2].coordinates.namespace).to.be.ok - expect(defs.length).to.be.equal(4) - const coordinates = verifyUniqueCoordinates(defs) - verifyExpectedCoordinates(coordinates, expected) - }) - - it('should filter and sort descending on multiple keys in continuation', async function() { - //filter: {'licensed.declared': 'MIT', '_mongo.page': 1} - //sort: {'coordinates.namespace': -1, 'coordinates.name':-1, 'coordinates.revision': -1, '_mongo.partitionKey': 1} - const expected = [ - 'git/github/microsoft/redie/194269b5b7010ad6f8dc4ef608c88128615031ca', - 'maven/mavencentral/com.azure/azure-storage-blob/12.20.0', - 'npm/npmjs/-/redie/0.3.0', - 'npm/npmjs/-/angular/1.6.9'] - - const query = { - license: 'MIT', - sort: 'namespace', - sortDesc: true - } - const defs = await fetchAll(mongoStore, query) - expect(defs.length).to.be.equal(4) - const coordinates = verifyUniqueCoordinates(defs) - verifyExpectedCoordinates(coordinates, expected) - }) - - it('should filter and sort on numerical scores and fetch continuously', async function() { - //filter: {'licensed.declared': 'MIT', '_mongo.page': 1} - //sort: {'scores.tool': 1, '_mongo.partitionKey': 1} - const expected = [ - 'maven/mavencentral/com.azure/azure-storage-blob/12.20.0', - 'npm/npmjs/-/angular/1.6.9', - 'git/github/microsoft/redie/194269b5b7010ad6f8dc4ef608c88128615031ca', - 'npm/npmjs/-/redie/0.3.0'] + it('should sort ascending on releaseDate and find 1 page of records', async function() { + const expected = ['maven/mavencentral/org.apache.httpcomponents/httpcore/4.0-alpha5'] + const query = { + sort: 'releaseDate', + sortDesc: false + } + const defs = await fetchUpToNTimes(this.mongoStore, query, 1) + expect(defs.length).to.be.equal(1) + const coordinates = verifyUniqueCoordinates(defs) + verifyExpectedCoordinates(coordinates, expected) + }) - const query = { - license: 'MIT', - sort: 'toolScore', - sortDesc: false - } - const defs = await fetchAll(mongoStore, query) - expect(defs.length).to.be.equal(4) - expect(defs[0].scores.tool).to.be.equal(80) - expect(defs[1].scores.tool).to.be.equal(84) - expect(defs[2].scores.tool).to.be.equal(90) - expect(defs[3].scores.tool).to.be.equal(94) - const coordinates = verifyUniqueCoordinates(defs) - verifyExpectedCoordinates(coordinates, expected) - }) - - it('should filter and sort descending on numerical scores and fetch continuously', async function() { - //filter: {'licensed.declared': 'MIT', '_mongo.page': 1} - //sort: {'scores.tool': -1, '_mongo.partitionKey': 1} - const expected = [ - 'npm/npmjs/-/redie/0.3.0', - 'git/github/microsoft/redie/194269b5b7010ad6f8dc4ef608c88128615031ca', - 'npm/npmjs/-/angular/1.6.9', - 'maven/mavencentral/com.azure/azure-storage-blob/12.20.0' - ] + it('should sort ascending on releaseDate and handle null and non null values in continuation', async function() { + const expected = [ + 'maven/mavencentral/org.apache.httpcomponents/httpcore/4.0-alpha5', + 'maven/mavencentral/org.flywaydb/flyway-maven-plugin/5.0.7', + 'npm/npmjs/@sinclair/typebox/0.24.45', + 'maven/mavencentral/org.apache.httpcomponents/httpcore/4.0-beta2', + 'pypi/pypi/-/backports.ssl_match_hostname/3.5.0.1'] + + const query = { + sort: 'releaseDate', + sortDesc: false + } + const defs = await fetchAll(this.mongoStore, query) + expect(defs[0].described.releaseDate).not.to.be.ok + expect(defs[3].described.releaseDate).to.be.ok + + expect(defs.length).to.be.equal(12) + const coordinates = verifyUniqueCoordinates(defs) + verifyExpectedCoordinates(coordinates, expected) + }) + + it('should sort descending on releaseDate and handle null and non null values in continuation', async function() { + const query = { + sort: 'releaseDate', + sortDesc: true + } + const defs = await fetchAll(this.mongoStore, query) + expect(defs.length).to.be.equal(12) + verifyUniqueCoordinates(defs) + }) + + it('should sort ascending on license and handle null and non null values in continuation ', async function() { + const expected = [ + 'npm/npmjs/@sinclair/typebox/0.24.45', + 'maven/mavencentral/io.jenetics/jenetics/7.1.1', + 'maven/mavencentral/io.quarkiverse.cxf/quarkus-cxf/1.5.4', + 'maven/mavencentral/org.apache.httpcomponents/httpcore/4.0-alpha5', + 'maven/mavencentral/org.apache.httpcomponents/httpcore/4.0-beta2'] + + const query = { + sort: 'license', + sortDesc: false + } + const defs = await fetchAll(this.mongoStore, query) + expect(defs[0].described.releaseDate).not.to.be.ok + expect(defs[1].described.releaseDate).to.be.ok + expect(defs.length).to.be.equal(12) + const coordinates = verifyUniqueCoordinates(defs) + verifyExpectedCoordinates(coordinates, expected) + }) + + it('should sort descending on license and handle null and non null values in continuation ', async function() { + const query = { + sort: 'license', + sortDesc: true + } + const defs = await fetchAll(this.mongoStore, query) + expect(defs.length).to.be.equal(12) + verifyUniqueCoordinates(defs) + }) + + it('should filter and sort ascending on multiple keys and handle null and non null namespace in continuation', async function() { + const expected = [ + 'npm/npmjs/-/angular/1.6.9', + 'npm/npmjs/-/redie/0.3.0', + 'maven/mavencentral/com.azure/azure-storage-blob/12.20.0', + 'git/github/microsoft/redie/194269b5b7010ad6f8dc4ef608c88128615031ca'] + + const query = { + license: 'MIT', + sort: 'namespace', + sortDesc: false + } + const defs = await fetchAll(this.mongoStore, query) + expect(defs[0].coordinates.namespace).not.to.be.ok + expect(defs[2].coordinates.namespace).to.be.ok + expect(defs.length).to.be.equal(4) + const coordinates = verifyUniqueCoordinates(defs) + verifyExpectedCoordinates(coordinates, expected) + }) + + it('should filter and sort descending on multiple keys in continuation', async function() { + const expected = [ + 'git/github/microsoft/redie/194269b5b7010ad6f8dc4ef608c88128615031ca', + 'maven/mavencentral/com.azure/azure-storage-blob/12.20.0', + 'npm/npmjs/-/redie/0.3.0', + 'npm/npmjs/-/angular/1.6.9'] + + const query = { + license: 'MIT', + sort: 'namespace', + sortDesc: true + } + const defs = await fetchAll(this.mongoStore, query) + expect(defs.length).to.be.equal(4) + const coordinates = verifyUniqueCoordinates(defs) + verifyExpectedCoordinates(coordinates, expected) + }) + + it('should filter and sort on numerical scores and fetch continuously', async function() { + const expected = [ + 'maven/mavencentral/com.azure/azure-storage-blob/12.20.0', + 'npm/npmjs/-/angular/1.6.9', + 'git/github/microsoft/redie/194269b5b7010ad6f8dc4ef608c88128615031ca', + 'npm/npmjs/-/redie/0.3.0'] + + const query = { + license: 'MIT', + sort: 'toolScore', + sortDesc: false + } + const defs = await fetchAll(this.mongoStore, query) + expect(defs.length).to.be.equal(4) + expect(defs[0].scores.tool).to.be.equal(80) + expect(defs[1].scores.tool).to.be.equal(84) + expect(defs[2].scores.tool).to.be.equal(90) + expect(defs[3].scores.tool).to.be.equal(94) + const coordinates = verifyUniqueCoordinates(defs) + verifyExpectedCoordinates(coordinates, expected) + }) + + it('should filter and sort descending on numerical scores and fetch continuously', async function() { + const expected = [ + 'npm/npmjs/-/redie/0.3.0', + 'git/github/microsoft/redie/194269b5b7010ad6f8dc4ef608c88128615031ca', + 'npm/npmjs/-/angular/1.6.9', + 'maven/mavencentral/com.azure/azure-storage-blob/12.20.0' + ] + + const query = { + license: 'MIT', + sort: 'toolScore', + sortDesc: true + } + const defs = await fetchAll(this.mongoStore, query) + expect(defs.length).to.be.equal(4) + const coordinates = verifyUniqueCoordinates(defs) + verifyExpectedCoordinates(coordinates, expected) + }) - const query = { - license: 'MIT', - sort: 'toolScore', - sortDesc: true - } - const defs = await fetchAll(mongoStore, query) - expect(defs.length).to.be.equal(4) - const coordinates = verifyUniqueCoordinates(defs) - verifyExpectedCoordinates(coordinates, expected) }) - -}) - -async function setupStore(mongoStore) { - const fileName = path.join(__dirname, '../../fixtures/store/definitions-paged-no-files') - const content = await fsPromise.readFile(fileName) - const defDump = JSON.parse(content.toString()) - await mongoStore.collection.createIndex({ '_mongo.partitionKey': 1 }) - await mongoStore.collection.insertMany(defDump) } function verifyExpectedCoordinates(allCoordinates, expected) { @@ -300,3 +246,5 @@ class ContinousFetch { return fetchedCounter } } + +module.exports = shouldPaginateSearchCorrectly \ No newline at end of file diff --git a/test/providers/store/pagedDefinitionPagination.js b/test/providers/store/pagedDefinitionPagination.js new file mode 100644 index 000000000..1e8d9f433 --- /dev/null +++ b/test/providers/store/pagedDefinitionPagination.js @@ -0,0 +1,55 @@ +// (c) Copyright 2023, SAP SE and ClearlyDefined contributors. Licensed under the MIT license. +// SPDX-License-Identifier: MIT + +const Store = require('../../../providers/stores/mongo') +const shouldPaginateSearchCorrectly = require('./mongoDefinitionPagination') +const { MongoMemoryServer } = require('mongodb-memory-server') +const fsPromise = require('fs/promises') +const path = require('path') + +const dbOptions = { + dbName: 'clearlydefined', + collectionName: 'definitions-paged', + logger: { + debug: () => {} + } +} + +describe('Mongo Definition Store: Paged', function() { + const mongoServer = new MongoMemoryServer() + let mongoStore + + before('setup database', async function() { + await mongoServer.start() + const uri = await mongoServer.getUri() + const options = { ...dbOptions, connectionString: uri } + + mongoStore = Store(options) + await mongoStore.initialize() + await setupStore(mongoStore) + }) + + after('cleanup database', async function() { + await mongoStore.collection.drop() + await mongoStore.close() + await mongoServer.stop() + }) + + beforeEach(function() { + this.mongoStore = mongoStore + }) + + shouldPaginateSearchCorrectly() +}) + +async function loadDefinitions() { + const fileName = path.join(__dirname, '../../fixtures/store/definitions-paged-no-files') + const content = await fsPromise.readFile(fileName) + return JSON.parse(content.toString()) +} + +async function setupStore(mongoStore) { + const defs = await loadDefinitions() + await mongoStore.collection.createIndex({ '_mongo.partitionKey': 1 }) + await mongoStore.collection.insertMany(defs) +} diff --git a/test/providers/store/trimmedDefinitionPagination.js b/test/providers/store/trimmedDefinitionPagination.js new file mode 100644 index 000000000..ad742d4b3 --- /dev/null +++ b/test/providers/store/trimmedDefinitionPagination.js @@ -0,0 +1,55 @@ +// (c) Copyright 2023, SAP SE and ClearlyDefined contributors. Licensed under the MIT license. +// SPDX-License-Identifier: MIT + +const Store = require('../../../providers/stores/trimmedMongoDefinitionStore') +const shouldPaginateSearchCorrectly = require('./mongoDefinitionPagination') +const { MongoMemoryServer } = require('mongodb-memory-server') +const fsPromise = require('fs/promises') +const path = require('path') + +const dbOptions = { + dbName: 'clearlydefined', + collectionName: 'definitions-trimmed', + logger: { + debug: () => {} + } +} + +describe('Mongo Definition Store: Trimmed', function() { + const mongoServer = new MongoMemoryServer() + let mongoStore + + before('setup database', async function() { + await mongoServer.start() + const uri = await mongoServer.getUri() + const options = { ...dbOptions, connectionString: uri } + + mongoStore = Store(options) + await mongoStore.initialize() + await setupStore(mongoStore) + }) + + after('cleanup database', async function() { + await mongoStore.collection.drop() + await mongoStore.close() + await mongoServer.stop() + }) + + beforeEach(function() { + this.mongoStore = mongoStore + }) + + shouldPaginateSearchCorrectly() +}) + +async function loadDefinitions() { + const fileName = path.join(__dirname, '../../fixtures/store/definitions-paged-no-files') + const content = await fsPromise.readFile(fileName) + return JSON.parse(content.toString()) +} + +async function setupStore(mongoStore) { + const defs = await loadDefinitions() + defs.forEach(def => delete def._mongo) + await mongoStore.collection.insertMany(defs) +}