Expose collection metadata through the collection API (#1123)

CHANGELOG.md

@@ -2,6 +2,18 @@
 
 All changes that impact users of this module are documented in this file, in the [Common Changelog](https://common-changelog.org) format with some additional specifications defined in the CONTRIBUTING file. This codebase adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html).
 
+## Unreleased [major]
+
+> Development of this release was supported by the [French Ministry for Foreign Affairs](https://www.diplomatie.gouv.fr/fr/politique-etrangere-de-la-france/diplomatie-numerique/) through its ministerial [State Startups incubator](https://beta.gouv.fr/startups/open-terms-archive.html) under the aegis of the Ambassador for Digital Affairs.
+
+### Added
+
+- Expose collection metadata through the collection API; requires a [metadata file](https://docs.opentermsarchive.org/collections/metadata/) at the root of your collection folder
+
+### Changed
+
+- **Breaking:** Replace `@opentermsarchive/engine.services.declarationsPath` with `@opentermsarchive/engine.collectionPath`; ensure your declarations are located in `./declarations` in your collection folder
+
 ## 3.0.0 - 2024-12-03
 
 _Full changeset and discussions: [#1122](https://github.com/OpenTermsArchive/engine/pull/1122)._

config/ci.json

@@ -1,7 +1,5 @@
 {
   "@opentermsarchive/engine": {
-    "services": {
-      "declarationsPath": "./demo-declarations/declarations"
-    }
+    "collectionPath": "./demo-declarations/"
   }
 }

config/default.json

@@ -1,9 +1,7 @@
 {
   "@opentermsarchive/engine": {
     "trackingSchedule": "30 */12 * * *",
-    "services": {
-      "declarationsPath": "./declarations"
-    },
+    "collectionPath": "./",
     "recorder": {
       "versions": {
         "storage": {

config/test.json

@@ -1,8 +1,6 @@
 {
   "@opentermsarchive/engine": {
-    "services": {
-      "declarationsPath": "./test/services"
-    },
+    "collectionPath": "./test/test-declarations",
     "recorder": {
       "versions": {
         "storage": {

package.json

@@ -82,6 +82,7 @@
     "https-proxy-agent": "^5.0.0",
     "iconv-lite": "^0.6.3",
     "joplin-turndown-plugin-gfm": "^1.0.12",
+    "js-yaml": "^4.1.0",
     "jsdom": "^18.1.0",
     "json-source-map": "^0.6.1",
     "lodash": "^4.17.21",

scripts/declarations/lint/index.mocha.js

@@ -16,8 +16,8 @@ const ESLINT_CONFIG_PATH = path.join(ROOT_PATH, '.eslintrc.yaml');
 const eslint = new ESLint({ overrideConfigFile: ESLINT_CONFIG_PATH, fix: false });
 const eslintWithFix = new ESLint({ overrideConfigFile: ESLINT_CONFIG_PATH, fix: true });
 
-const declarationsPath = path.resolve(process.cwd(), config.get('@opentermsarchive/engine.services.declarationsPath'));
-const instancePath = path.resolve(declarationsPath, '../');
+const instancePath = path.resolve(process.cwd(), config.get('@opentermsarchive/engine.collectionPath'));
+const declarationsPath = path.resolve(instancePath, services.DECLARATIONS_PATH);
 
 export default async options => {
   let servicesToValidate = options.services || [];

scripts/declarations/validate/index.mocha.js

@@ -19,8 +19,8 @@ const fs = fsApi.promises;
 const MIN_DOC_LENGTH = 100;
 const SLOW_DOCUMENT_THRESHOLD = 10 * 1000; // number of milliseconds after which a document fetch is considered slow
 
-const declarationsPath = path.resolve(process.cwd(), config.get('@opentermsarchive/engine.services.declarationsPath'));
-const instancePath = path.resolve(declarationsPath, '../');
+const instancePath = path.resolve(process.cwd(), config.get('@opentermsarchive/engine.collectionPath'));
+const declarationsPath = path.resolve(instancePath, services.DECLARATIONS_PATH);
 
 export default async options => {
   const schemaOnly = options.schemaOnly || false;

src/archivist/services/index.js

@@ -1,4 +1,4 @@
-import fsApi from 'fs';
+import fs from 'fs/promises';
 import path from 'path';
 import { pathToFileURL } from 'url';
 
@@ -8,8 +8,8 @@ import Service from './service.js';
 import SourceDocument from './sourceDocument.js';
 import Terms from './terms.js';
 
-const fs = fsApi.promises;
-const declarationsPath = path.resolve(process.cwd(), config.get('@opentermsarchive/engine.services.declarationsPath'));
+export const DECLARATIONS_PATH = './declarations';
+const declarationsPath = path.resolve(process.cwd(), config.get('@opentermsarchive/engine.collectionPath'), DECLARATIONS_PATH);
 
 export async function load(servicesIdsToLoad = []) {
   let servicesIds = await getDeclaredServicesIds();

src/archivist/services/service.js

@@ -70,6 +70,6 @@ export default class Service {
   }
 
   static getNumberOfTerms(services, servicesIds, termsTypes) {
-    return servicesIds.reduce((acc, serviceId) => acc + services[serviceId].getNumberOfTerms(termsTypes), 0);
+    return (servicesIds || Object.keys(services)).reduce((acc, serviceId) => acc + services[serviceId].getNumberOfTerms(termsTypes), 0);
   }
 }

src/collection-api/routes/index.js

@@ -1,11 +1,17 @@
+import path from 'path';
+
+import config from 'config';
 import express from 'express';
 import helmet from 'helmet';
 
+import * as Services from '../../archivist/services/index.js';
+
 import docsRouter from './docs.js';
+import metadataRouter from './metadata.js';
 import servicesRouter from './services.js';
 import versionsRouter from './versions.js';
 
-export default function apiRouter(basePath) {
+export default async function apiRouter(basePath) {
   const router = express.Router();
 
   const defaultDirectives = helmet.contentSecurityPolicy.getDefaultDirectives();
@@ -27,7 +33,11 @@ export default function apiRouter(basePath) {
     res.json({ message: 'Welcome to an instance of the Open Terms Archive API. Documentation is available at /docs. Learn more on Open Terms Archive on https://opentermsarchive.org.' });
   });
 
-  router.use(servicesRouter);
+  const collectionPath = path.resolve(process.cwd(), config.get('@opentermsarchive/engine.collectionPath'));
+  const services = await Services.load();
+
+  router.use(await metadataRouter(collectionPath, services));
+  router.use(servicesRouter(services));
   router.use(versionsRouter);
 
   return router;

src/collection-api/routes/metadata.js

@@ -0,0 +1,165 @@
+import fs from 'fs/promises';
+import path from 'path';
+
+import express from 'express';
+import yaml from 'js-yaml';
+
+import Service from '../../archivist/services/service.js';
+
+const METADATA_FILENAME = 'metadata.yml';
+const PACKAGE_JSON_PATH = '../../../package.json';
+
+/**
+ * @swagger
+ * tags:
+ *   name: Metadata
+ *   description: Collection metadata API
+ * components:
+ *   schemas:
+ *     Metadata:
+ *       type: object
+ *       description: Collection metadata
+ *       properties:
+ *         id:
+ *           type: string
+ *           description: Unique identifier of the collection
+ *         name:
+ *           type: string
+ *           description: Display name of the collection
+ *         tagline:
+ *           type: string
+ *           description: Short description of the collection
+ *         description:
+ *           type: string
+ *           nullable: true
+ *           description: Detailed description of the collection
+ *         totalTerms:
+ *           type: integer
+ *           description: Total number of terms tracked in the collection
+ *         totalServices:
+ *           type: integer
+ *           description: Total number of services tracked in the collection
+ *         engineVersion:
+ *           type: string
+ *           description: Version of the Open Terms Archive engine in SemVer format (MAJOR.MINOR.PATCH)
+ *         dataset:
+ *           type: string
+ *           format: uri
+ *           description: URL to the dataset releases
+ *         declarations:
+ *           type: string
+ *           format: uri
+ *           description: URL to the declarations repository
+ *         versions:
+ *           type: string
+ *           format: uri
+ *           description: URL to the versions repository
+ *         snapshots:
+ *           type: string
+ *           format: uri
+ *           description: URL to the snapshots repository
+ *         logo:
+ *           type: string
+ *           format: uri
+ *           nullable: true
+ *           description: URL to the collection logo
+ *         languages:
+ *           type: array
+ *           items:
+ *             type: string
+ *           description: List of ISO 639 language codes representing languages allowed by the collection
+ *         jurisdictions:
+ *           type: array
+ *           items:
+ *             type: string
+ *           description: List of ISO 3166-2 country codes representing jurisdictions covered by the collection
+ *         trackingPeriods:
+ *           type: array
+ *           items:
+ *             type: object
+ *             properties:
+ *               startDate:
+ *                 type: string
+ *                 format: date
+ *                 description: The date when tracking started for this period
+ *               schedule:
+ *                 type: string
+ *                 description: A cron expression defining when terms are tracked (e.g. "0 0 * * *" for daily at midnight)
+ *               serverLocation:
+ *                 type: string
+ *                 description: The geographic location of the server used for tracking
+ *               endDate:
+ *                 type: string
+ *                 format: date
+ *                 description: The date when tracking ended for this period
+ *         governance:
+ *           type: object
+ *           properties:
+ *             hosts:
+ *               type: array
+ *               items:
+ *                 $ref: '#/components/schemas/Organization'
+ *             administrators:
+ *               type: array
+ *               items:
+ *                 $ref: '#/components/schemas/Organization'
+ *             curators:
+ *               type: array
+ *               items:
+ *                 $ref: '#/components/schemas/Organization'
+ *             maintainers:
+ *               type: array
+ *               items:
+ *                 $ref: '#/components/schemas/Organization'
+ *             sponsors:
+ *               type: array
+ *               items:
+ *                 $ref: '#/components/schemas/Organization'
+ *     Organization:
+ *       type: object
+ *       properties:
+ *         name:
+ *           type: string
+ *           description: Name of the organization
+ *         url:
+ *           type: string
+ *           format: uri
+ *           description: URL to the organization's website
+ *         logo:
+ *           type: string
+ *           format: uri
+ *           description: URL to the organization's logo
+ */
+export default async function metadataRouter(collectionPath, services) {
+  const router = express.Router();
+
+  const STATIC_METADATA = yaml.load(await fs.readFile(path.join(collectionPath, METADATA_FILENAME), 'utf8'));
+  const { version: engineVersion } = JSON.parse(await fs.readFile(new URL(PACKAGE_JSON_PATH, import.meta.url)));
+
+  /**
+   * @swagger
+   * /metadata:
+   *   get:
+   *     summary: Get collection metadata
+   *     tags: [Metadata]
+   *     produces:
+   *       - application/json
+   *     responses:
+   *       200:
+   *         description: Collection metadata
+   */
+  router.get('/metadata', (req, res) => {
+    const dynamicMetadata = {
+      totalServices: Object.keys(services).length,
+      totalTerms: Service.getNumberOfTerms(services),
+      engineVersion,
+    };
+
+    res.json({
+      ...STATIC_METADATA,
+      ...dynamicMetadata,
+    });
+  });
+
+  return router;
+}

src/collection-api/routes/metadata.test.js

@@ -0,0 +1,89 @@
+import fs from 'fs/promises';
+
+import { expect } from 'chai';
+import config from 'config';
+import request from 'supertest';
+
+import app from '../server.js';
+
+const basePath = config.get('@opentermsarchive/engine.collection-api.basePath');
+const { version: engineVersion } = JSON.parse(await fs.readFile(new URL('../../../package.json', import.meta.url)));
+
+const EXPECTED_RESPONSE = {
+  totalServices: 7,
+  totalTerms: 8,
+  id: 'test',
+  name: 'test',
+  tagline: 'Test collection',
+  description: 'This is a test collection used for testing purposes.',
+  dataset: 'https://github.com/OpenTermsArchive/test-versions/releases',
+  declarations: 'https://github.com/OpenTermsArchive/test-declarations',
+  versions: 'https://github.com/OpenTermsArchive/test-versions',
+  snapshots: 'https://github.com/OpenTermsArchive/test-snapshots',
+  donations: null,
+  logo: 'https://opentermsarchive.org/images/logo/logo-open-terms-archive-black.png',
+  languages: [
+    'en',
+  ],
+  jurisdictions: [
+    'EU',
+  ],
+  governance: {
+    hosts: [
+      { name: 'Localhost' },
+    ],
+    administrators: [
+      {
+        name: 'Open Terms Archive',
+        url: 'https://opentermsarchive.org/',
+        logo: 'https://opentermsarchive.org/images/logo/logo-open-terms-archive-black.png',
+      },
+    ],
+    curators: [
+      {
+        name: 'Open Terms Archive',
+        url: 'https://opentermsarchive.org/',
+        logo: 'https://opentermsarchive.org/images/logo/logo-open-terms-archive-black.png',
+      },
+    ],
+    maintainers: [
+      {
+        name: 'Open Terms Archive',
+        url: 'https://opentermsarchive.org/',
+        logo: 'https://opentermsarchive.org/images/logo/logo-open-terms-archive-black.png',
+      },
+    ],
+    sponsors: [
+      {
+        name: 'Open Terms Archive',
+        url: 'https://opentermsarchive.org/',
+        logo: 'https://opentermsarchive.org/images/logo/logo-open-terms-archive-black.png',
+      },
+    ],
+  },
+};
+
+describe('Metadata API', () => {
+  describe('GET /metadata', () => {
+    let response;
+
+    before(async () => {
+      response = await request(app).get(`${basePath}/v1/metadata`);
+    });
+
+    it('responds with 200 status code', () => {
+      expect(response.status).to.equal(200);
+    });
+
+    it('responds with Content-Type application/json', () => {
+      expect(response.type).to.equal('application/json');
+    });
+
+    it('returns expected metadata object', () => {
+      expect(response.body).to.deep.equal({
+        ...EXPECTED_RESPONSE,
+        engineVersion,
+      });
+    });
+  });
+});