Load collections at startup

package.json

@@ -18,7 +18,7 @@
   "scripts": {
     "lint": "eslint .",
     "start": "node -r dotenv/config src/index.js",
-    "test": "cross-env NODE_ENV=test mocha --recursive \"./src/**/*.test.js\" --exit"
+    "test": "cross-env NODE_ENV=test mocha --require test/helper.js --recursive \"./src/**/*.test.js\" --exit"
   },
   "dependencies": {
     "commander": "^9.4.1",

src/controllers/collections.js

@@ -1,7 +1,3 @@
-import config from 'config';
-
-import { fetchCollections } from '../services/collections.js';
-
 export const getCollections = async (req, res) => {
-  res.json(await fetchCollections(config.get('@opentermsarchive/federation-api.collections')));
+  res.json(req.app.locals.collections);
 };

src/controllers/services.js

@@ -1,12 +1,9 @@
-import config from 'config';
-
-import { fetchCollections } from '../services/collections.js';
 import { fetchServices, isServiceIDValid } from '../services/services.js';
 
 export const getServices = async (req, res) => {
   const { name: requestedName, termsType: requestedTermsType } = req.query;
 
-  const collections = await fetchCollections(config.get('@opentermsarchive/federation-api.collections'));
+  const { collections } = req.app.locals;
 
   const results = [];
   const failures = [];
@@ -53,7 +50,7 @@ export const getService = async (req, res) => {
     return res.status(400).json();
   }
 
-  const collections = await fetchCollections(config.get('@opentermsarchive/federation-api.collections'));
+  const { collections } = req.app.locals;
 
   const results = [];
   const failures = [];

src/index.js

@@ -5,6 +5,7 @@ import 'express-async-errors';
 import errorsMiddleware from './middlewares/errors.js';
 import loggerMiddleware from './middlewares/logger.js';
 import apiRouter from './routes/index.js';
+import { fetchCollections } from './services/collections.js';
 import logger from './utils/logger.js';
 
 const app = express();
@@ -20,6 +21,18 @@ app.use(errorsMiddleware);
 
 const PORT = config.get('@opentermsarchive/federation-api.port');
 
+const collections = await fetchCollections(config.get('@opentermsarchive/federation-api.collections')).catch(error => {
+  logger.error(error);
+  process.exit(1);
+});
+
+if (collections.length == 0) {
+  logger.error('Without valid collections declared, the process will exit as this API cannot serve any requests');
+  process.exit(2);
+}
+
+app.locals.collections = collections;
+
 app.listen(PORT, () => {
   logger.info(`Start Open Terms Archive Federation API on http://localhost:${PORT}${BASE_PATH}`);
 });

src/routes/collections.test.js

@@ -1,31 +1,9 @@
 import { expect } from 'chai';
-import nock from 'nock';
 import request from 'supertest';
 
 import app, { BASE_PATH } from '../index.js';
 
-const COLLECTIONS_RESULT = [
-  {
-    name: 'Collection 1',
-    id: 'collection-1',
-    endpoint: 'http://collection-1.example/api/v1',
-  },
-  {
-    name: 'Collection 2',
-    id: 'collection-2',
-    endpoint: 'https://2.collection.example/api/v1',
-  },
-];
-
 describe('Routes: Collections', () => {
-  before(() => {
-    nock('https://opentermsarchive.org/collections.json').persist().get('').reply(200, COLLECTIONS_RESULT);
-  });
-
-  after(() => {
-    nock.cleanAll();
-  });
-
   describe('GET /collections', () => {
     let response;
 

src/routes/services.test.js

@@ -4,19 +4,6 @@ import request from 'supertest';
 
 import app, { BASE_PATH } from '../index.js';
 
-export const COLLECTIONS_RESULT = [
-  {
-    name: 'Collection 1',
-    id: 'collection-1',
-    endpoint: 'http://collection-1.example/api/v1',
-  },
-  {
-    name: 'Collection 2',
-    id: 'collection-2',
-    endpoint: 'https://2.collection.example/api/v1',
-  },
-];
-
 const COLLECTION_1_SERVICES_RESULT = [
   {
     id: 'service-1',
@@ -67,11 +54,11 @@ const COLLECTION_2_SERVICES_RESULT = [
   },
 ];
 
+// Use the global HTTP request mock for the URL 'https://opentermsarchive.org/collections.json' defined in 'test/helpers.js'
 describe('Routes: Services', () => {
   const serviceWithUrlEncodedChineseCharactersName = '%E6%8A%96%E9%9F%B3%E7%9F%AD%E8%A7%86%E9%A2%91';
 
-  before(() => {
-    nock('https://opentermsarchive.org/collections.json').persist().get('').reply(200, COLLECTIONS_RESULT);
+  before(async () => {
     nock('http://collection-1.example').persist().get('/api/v1/services').reply(200, COLLECTION_1_SERVICES_RESULT);
     nock('https://2.collection.example').persist().get('/api/v1/services').reply(200, COLLECTION_2_SERVICES_RESULT);
   });
@@ -268,7 +255,6 @@ describe('Routes: Services', () => {
     context('when an error occurs in one of the underlying collections', () => {
       before(async () => {
         nock.cleanAll();
-        nock('https://opentermsarchive.org/collections.json').persist().get('').reply(200, COLLECTIONS_RESULT);
         nock('http://collection-1.example').persist().get('/api/v1/services').reply(200, COLLECTION_1_SERVICES_RESULT);
         nock('https://2.collection.example').get('/api/v1/services').replyWithError({
           message: 'something went wrong',

src/services/collections.js

@@ -3,42 +3,65 @@ import { URL } from 'url';
 import fetch from '../utils/fetch.js';
 import logger from '../utils/logger.js';
 
-export const fetchCollections = async collectionsConfig => {
+export async function fetchCollections(collectionsConfig) {
+  const errors = [];
   const uniqueCollections = new Map();
 
   (await Promise.allSettled(collectionsConfig.map(async item => {
     let collections = [];
 
     if (typeof item === 'string') {
-      collections = await fetch(item);
+      collections = await fetch(item).catch(error => {
+        throw new Error(`${error.message} when fetching ${item}`);
+      });
+      if (!Array.isArray(collections)) {
+        throw new Error(`Invalid format; an array of collections is expected when fetching ${item}`);
+      }
     }
 
     if (typeof item === 'object') {
       collections = [item];
     }
 
     return filterInvalidCollections(collections);
-  }))).forEach(({ value }) => {
+  }))).forEach(({ value, reason }) => {
+    if (reason) {
+      return errors.push(reason);
+    }
     value.forEach(collection => {
       uniqueCollections.set(collection.id, collection);
     });
   });
 
+  if (errors.length) {
+    throw new Error(`\n${errors.join('\n')}`);
+  }
+
   return Array.from(uniqueCollections.values());
-};
+}
 
-export function filterInvalidCollections(collections) {
+function filterInvalidCollections(collections) {
   return collections.filter(collection => {
     const hasMandatoryFields = collection.id && collection.name && collection.endpoint;
 
+    const errors = [];
+
     if (!hasMandatoryFields) {
-      logger.warn(`Ignore the following collection lacking mandatory fields 'id', 'name', or 'endpoint': \n${JSON.stringify(collection, null, 4)}`);
+      errors.push('lack mandatory fields "id", "name", or "endpoint"');
     }
 
-    const isEndpointValidUrl = isURL(collection.endpoint);
+    let isEndpointValidUrl = true;
+
+    if (collection.endpoint) {
+      isEndpointValidUrl = isURL(collection.endpoint);
+
+      if (!isEndpointValidUrl) {
+        errors.push('the endpoint is not a valid URL');
+      }
+    }
 
-    if (!isEndpointValidUrl) {
-      logger.warn(`Ignore the following collection as 'endpoint' is not a valid URL: \n${JSON.stringify(collection, null, 4)}`);
+    if (errors.length) {
+      logger.warn(`Ignore collection "${JSON.stringify(collection)}" due to following errors: \n- ${errors.join('\n- ')}`);
     }
 
     return hasMandatoryFields && isEndpointValidUrl;

src/services/collections.test.js

@@ -86,14 +86,14 @@ describe('Services: Collections', () => {
         });
       });
 
-      context('when endpoint is not a valid URL ', () => {
+      context('when endpoint is not a valid URL', () => {
         it('removes invalid collection', async () => {
           const config = [
             COLLECTION_1,
             {
               id: 'invalid-endpoint',
               name: 'Invalid collection endpoint',
-              endpoint: 'no url endoint',
+              endpoint: 'no url endpoint',
             },
             COLLECTION_3,
           ];

test/helper.js

@@ -0,0 +1,18 @@
+import nock from 'nock';
+
+export const COLLECTIONS_RESULT = [
+  {
+    name: 'Collection 1',
+    id: 'collection-1',
+    endpoint: 'http://collection-1.example/api/v1',
+  },
+  {
+    name: 'Collection 2',
+    id: 'collection-2',
+    endpoint: 'https://2.collection.example/api/v1',
+  },
+];
+
+nock('https://opentermsarchive.org/collections.json').persist().get('').reply(200, COLLECTIONS_RESULT);
+
+console.log('test/helpers.js: Globally mock HTTP request for "https://opentermsarchive.org/collections.json"\n');