Enable additional sources for federated API

.github/workflows/changelog.yml

@@ -15,4 +15,4 @@ on:
 
 jobs:
   check_changelog:
-    uses: OpenTermsArchive/engine/.github/workflows/changelog.yml@main
+    uses: OpenTermsArchive/engine/.github/workflows/changelog.yml@v0.37.1

CHANGELOG.md

@@ -2,7 +2,15 @@
 
 All changes that impact users of this module are documented in this file, in the [Common Changelog](https://common-changelog.org) format with some additional specifications defined in the [engine CONTRIBUTING file](https://github.com/OpenTermsArchive/engine/blob/main/CONTRIBUTING.md#changelog). This codebase adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html).
 
-## Unreleased
+## Unreleased [minor]
+
+_Full changeset and discussions: [#4](https://github.com/OpenTermsArchive/engine/pull/4)._
+
+> Development of this release was [supported](https://nlnet.nl/project/TOSDR-OTA/) by the [NGI0 Entrust Fund](https://nlnet.nl/entrust), a fund established by [NLnet](https://nlnet.nl/) with financial support from the European Commission's [Next Generation Internet](https://www.ngi.eu) programme, under the aegis of DG CNECT under grant agreement N°101069594.
+
+### Changed
+
+- **Breaking:** Replaced `collectionsUrl` config entry with `collections` key, now supporting both URLs and directly specified collections. Refer to the [configuration documentation](https://docs.opentermsarchive.org/api/federated/#configuring) for details.
 
 ## 0.1.1 - 2024-02-15
 

config/default.json

@@ -1,6 +1,8 @@
 {
   "port": 3333,
-  "collectionsUrl": "https://opentermsarchive.org/collections.json",
+  "collections": [
+    "https://opentermsarchive.org/collections.json"
+  ],
   "logger": {
     "smtp": {
       "host": "smtp-relay.sendinblue.com",

src/controllers/collections.js

@@ -1,5 +1,7 @@
+import config from 'config';
+
 import { fetchCollections } from '../services/collections.js';
 
 export const getCollections = async (req, res) => {
-  res.json(await fetchCollections());
+  res.json(await fetchCollections(config.get('collections')));
 };

src/controllers/services.js

@@ -1,10 +1,12 @@
+import config from 'config';
+
 import { fetchCollections } from '../services/collections.js';
 import { fetchServices, isServiceIDValid } from '../services/services.js';
 
 export const getServices = async (req, res) => {
   const { name: requestedName, termsType: requestedTermsType } = req.query;
 
-  const collections = await fetchCollections();
+  const collections = await fetchCollections(config.get('collections'));
 
   const results = [];
   const failures = [];
@@ -51,7 +53,7 @@ export const getService = async (req, res) => {
     return res.status(400).json();
   }
 
-  const collections = await fetchCollections();
+  const collections = await fetchCollections(config.get('collections'));
 
   const results = [];
   const failures = [];

src/routes/collections.test.js

@@ -1,24 +1,25 @@
 import { expect } from 'chai';
-import config from 'config';
 import nock from 'nock';
 import request from 'supertest';
 
 import app, { BASE_PATH } from '../index.js';
 
-const COLLECTIONS_RESULT = {
-  'Collection 1': {
+const COLLECTIONS_RESULT = [
+  {
+    name: 'Collection 1',
     id: 'collection-1',
     endpoint: 'http://collection-1.example/api/v1',
   },
-  'Collection 2': {
+  {
+    name: 'Collection 2',
     id: 'collection-2',
     endpoint: 'https://2.collection.example/api/v1',
   },
-};
+];
 
-describe('Collections routes', () => {
+describe('Routes: Collections', () => {
   before(() => {
-    nock(config.get('collectionsUrl')).persist().get('').reply(200, COLLECTIONS_RESULT);
+    nock('https://opentermsarchive.org/collections.json').persist().get('').reply(200, COLLECTIONS_RESULT);
   });
 
   after(() => {

src/routes/docs.test.js

@@ -3,7 +3,7 @@ import request from 'supertest';
 
 import app, { BASE_PATH } from '../index.js';
 
-describe('Docs API', () => {
+describe('Routes: Docs', () => {
   describe('GET /docs', () => {
     let response;
 

src/routes/services.test.js

@@ -1,20 +1,21 @@
 import { expect } from 'chai';
-import config from 'config';
 import nock from 'nock';
 import request from 'supertest';
 
 import app, { BASE_PATH } from '../index.js';
 
-export const COLLECTIONS_RESULT = {
-  'Collection 1': {
+export const COLLECTIONS_RESULT = [
+  {
+    name: 'Collection 1',
     id: 'collection-1',
     endpoint: 'http://collection-1.example/api/v1',
   },
-  'Collection 2': {
+  {
+    name: 'Collection 2',
     id: 'collection-2',
     endpoint: 'https://2.collection.example/api/v1',
   },
-};
+];
 
 const COLLECTION_1_SERVICES_RESULT = [
   {
@@ -66,11 +67,11 @@ const COLLECTION_2_SERVICES_RESULT = [
   },
 ];
 
-describe('Services routes', () => {
+describe('Routes: Services', () => {
   const serviceWithUrlEncodedChineseCharactersName = '%E6%8A%96%E9%9F%B3%E7%9F%AD%E8%A7%86%E9%A2%91';
 
   before(() => {
-    nock(config.get('collectionsUrl')).persist().get('').reply(200, COLLECTIONS_RESULT);
+    nock('https://opentermsarchive.org/collections.json').persist().get('').reply(200, COLLECTIONS_RESULT);
     nock('http://collection-1.example').persist().get('/api/v1/services').reply(200, COLLECTION_1_SERVICES_RESULT);
     nock('https://2.collection.example').persist().get('/api/v1/services').reply(200, COLLECTION_2_SERVICES_RESULT);
   });
@@ -267,7 +268,7 @@ describe('Services routes', () => {
     context('when an error occurs in one of the underlying collections', () => {
       before(async () => {
         nock.cleanAll();
-        nock(config.get('collectionsUrl')).persist().get('').reply(200, COLLECTIONS_RESULT);
+        nock('https://opentermsarchive.org/collections.json').persist().get('').reply(200, COLLECTIONS_RESULT);
         nock('http://collection-1.example').persist().get('/api/v1/services').reply(200, COLLECTION_1_SERVICES_RESULT);
         nock('https://2.collection.example').get('/api/v1/services').replyWithError({
           message: 'something went wrong',

src/services/collections.js

@@ -1,19 +1,32 @@
-import config from 'config';
-
 import fetch from '../utils/fetch.js';
 
-export const fetchCollections = async () => {
-  const collections = await fetch(config.get('collectionsUrl'));
+export const fetchCollections = async collectionsConfig => {
+  let collections = [];
+
+  for (const item of collectionsConfig) {
+    if (typeof item === 'string') {
+      /* eslint-disable no-await-in-loop */
+      const fetchedCollections = await fetch(item); // Use `await` to ensure sequential fetching of collections and preserve the order in the resulting collections array
+
+      /* eslint-enable no-await-in-loop */
+
+      collections = collections.concat(fetchedCollections.filter(collection => collection.id && collection.name && collection.endpoint));
+    }
 
-  return Object.keys(collections).reduce((result, collectionName) => {
-    if (collections[collectionName].endpoint) {
-      result.push({
-        name: collectionName,
-        id: collections[collectionName].id,
-        endpoint: collections[collectionName].endpoint,
-      });
+    if (typeof item === 'object' && item.id && item.name && item.endpoint) {
+      collections.push(item);
     }
+  }
 
-    return result;
-  }, []);
+  return removeDuplicatesKeepLatest(collections);
 };
+
+export function removeDuplicatesKeepLatest(collections) {
+  const uniqueCollections = {};
+
+  collections.forEach(collection => {
+    uniqueCollections[collection.id] = collection;
+  });
+
+  return Object.values(uniqueCollections);
+}

src/services/collections.test.js

@@ -0,0 +1,41 @@
+import { expect } from 'chai';
+import nock from 'nock';
+
+import { fetchCollections, removeDuplicatesKeepLatest } from './collections.js';
+
+describe('Services: Collections', () => {
+  const COLLECTION_1 = { id: 'collection1', name: 'Collection 1', endpoint: 'http://domain1.example/endpoint' };
+  const COLLECTION_2 = { id: 'collection2', name: 'Collection 2', endpoint: 'http://domain1.example/endpoint' };
+  const COLLECTION_3 = { id: 'collection3', name: 'Collection 3', endpoint: 'http://domain1.example/endpoint' };
+  const COLLECTION_1_OVERRIDEN = { id: COLLECTION_1.id, name: 'Override Collection 1', endpoint: 'http://domain2.example/endpoint' };
+  const COLLECTION_2_OVERRIDEN = { id: COLLECTION_2.id, name: 'Override Collection 2', endpoint: 'http://domain2.example/endpoint' };
+
+  describe('removeDuplicatesKeepLatest', () => {
+    it('removes duplicates based on their id and keep the latest defined', () => {
+      const collections = [ COLLECTION_1, COLLECTION_2, COLLECTION_2_OVERRIDEN, COLLECTION_3 ];
+      const uniqueCollections = removeDuplicatesKeepLatest(collections);
+
+      expect(uniqueCollections).to.deep.equal([ COLLECTION_1, COLLECTION_2_OVERRIDEN, COLLECTION_3 ]);
+    });
+  });
+
+  describe('fetchCollections', () => {
+    it('fetches collections from URLs, includes directly given collections, and removes duplicates', async () => {
+      nock('http://domain1.example')
+        .get('/collections.json')
+        .reply(200, [ COLLECTION_1, COLLECTION_2 ]);
+
+      nock('http://domain2.example')
+        .get('/collections.json')
+        .reply(200, [ COLLECTION_3, COLLECTION_2_OVERRIDEN ]);
+
+      const config = [
+        'http://domain1.example/collections.json',
+        COLLECTION_1_OVERRIDEN,
+        'http://domain2.example/collections.json',
+      ];
+
+      expect(await fetchCollections(config)).to.deep.equal([ COLLECTION_1_OVERRIDEN, COLLECTION_2_OVERRIDEN, COLLECTION_3 ]);
+    });
+  });
+});

src/services/services.test.js

@@ -2,54 +2,56 @@ import { expect } from 'chai';
 
 import { isServiceIDValid } from './services.js';
 
-describe('isServiceIDValid', () => {
-  const validServiceIDs = {
-    'with only alphanumeric': 'service',
-    'with dot': 'servi.ce',
-    'with exclamation mark': 'service!',
-    'with question mark': 'service?',
-    'with dash': 'service-',
-    'with semi-colon': 'service;',
-    'with comma': 'service,',
-    'with underscore': 'service_',
-    'with plus sign': 'service+',
-    'with minus sign': 'service-',
-    'with single quote': 'service\'',
-    'with double quote': 'service"',
-  };
+describe('Services: Services', () => {
+  describe('isServiceIDValid', () => {
+    const validServiceIDs = {
+      'with only alphanumeric': 'service',
+      'with dot': 'servi.ce',
+      'with exclamation mark': 'service!',
+      'with question mark': 'service?',
+      'with dash': 'service-',
+      'with semi-colon': 'service;',
+      'with comma': 'service,',
+      'with underscore': 'service_',
+      'with plus sign': 'service+',
+      'with minus sign': 'service-',
+      'with single quote': 'service\'',
+      'with double quote': 'service"',
+    };
 
-  const invalidServiceIDs = {
-    'with ideograms': 'service抖',
-    'with cyrillic': 'serviceД',
-    'with accented letter': 'sérvice',
-    'with ligature': 'særvice',
-    'with colon': 'servi:ce',
-    'with forward slash': 'servi/ce',
-    'with backslash': 'servi\\ce',
-    'with empty string': '',
-    'with null value': null,
-    'with undefined value': undefined,
-  };
+    const invalidServiceIDs = {
+      'with ideograms': 'service抖',
+      'with cyrillic': 'serviceД',
+      'with accented letter': 'sérvice',
+      'with ligature': 'særvice',
+      'with colon': 'servi:ce',
+      'with forward slash': 'servi/ce',
+      'with backslash': 'servi\\ce',
+      'with empty string': '',
+      'with null value': null,
+      'with undefined value': undefined,
+    };
 
-  context('for valid service IDs', () => {
-    Object.entries(validServiceIDs).forEach(([ description, serviceID ]) => {
-      context(`${description}`, () => {
-        it('returns true', () => {
-          const result = isServiceIDValid(serviceID);
+    context('for valid service IDs', () => {
+      Object.entries(validServiceIDs).forEach(([ description, serviceID ]) => {
+        context(`${description}`, () => {
+          it('returns true', () => {
+            const result = isServiceIDValid(serviceID);
 
-          expect(result).to.be.true;
+            expect(result).to.be.true;
+          });
         });
       });
     });
-  });
 
-  context('for invalid service IDs', () => {
-    Object.entries(invalidServiceIDs).forEach(([ description, serviceID ]) => {
-      context(`${description}`, () => {
-        it('returns false', () => {
-          const result = isServiceIDValid(serviceID);
+    context('for invalid service IDs', () => {
+      Object.entries(invalidServiceIDs).forEach(([ description, serviceID ]) => {
+        context(`${description}`, () => {
+          it('returns false', () => {
+            const result = isServiceIDValid(serviceID);
 
-          expect(result).to.be.false;
+            expect(result).to.be.false;
+          });
         });
       });
     });

src/utils/fetch.test.js

@@ -3,7 +3,7 @@
 
 import fetch, { HTTPResponseError, DEFAULT_TIMEOUT } from './fetch.js';
 
-describe('Fetch', () => {
+describe('Utils: Fetch', () => {
   describe('Response parsing', () => {
     it('parses response as JSON', async () => {
       const scope = nock('https://example.com').get('/').reply(200, { data: 'success' });
@@ -46,7 +46,7 @@
   });
 
   describe('Timeout', () => {
     it('aborts requests that exceed the timeout', async function () {
       this.timeout(DEFAULT_TIMEOUT + 1000);
 
       const scope = nock('https://example.com').get('/').delayConnection(DEFAULT_TIMEOUT + 100).reply(200, { data: 'success' }); // Simulate a delay of 3000ms