From b97bc997c97a225d3c294392e3a0d01cefe93c28 Mon Sep 17 00:00:00 2001 From: Samuel Gerard Date: Thu, 27 Jun 2024 16:24:11 -0700 Subject: [PATCH 1/2] [BUG][NRPTI-1242] Add support for creating & updating permits from multiple permit codes (#1248) * Add support for creating permits from multiple permit codes * Fix test method calls * Fix whitespace and replace if-then-else with single return statement --- api/src/integrations/core/datasource.js | 87 ++++++++++++------- api/src/integrations/core/datasource.test.js | 6 +- api/src/integrations/core/permit-utils.js | 2 +- .../integrations/core/permit-utils.test.js | 6 +- 4 files changed, 64 insertions(+), 37 deletions(-) diff --git a/api/src/integrations/core/datasource.js b/api/src/integrations/core/datasource.js index 38a8f1fd3..c79af8625 100644 --- a/api/src/integrations/core/datasource.js +++ b/api/src/integrations/core/datasource.js @@ -250,8 +250,7 @@ class CoreDataSource { } /** - * Gets valid permit for a Core mine. A valid permit must meet the following criteria: - * - Must not be exploratory + * Gets all valid permits for a Core mine. A valid permit must meet the following criteria: * - Must not be historical * . * @@ -259,33 +258,54 @@ class CoreDataSource { * @returns {object} Valid permit. * @memberof CoreDataSource */ - async getMinePermit(nrptiRecord) { + async getMinePermits(nrptiRecord) { if (!nrptiRecord) { - throw Error('getMinePermit - required nrptiRecord is null.'); + throw Error('getMinePermits - required nrptiRecord is null.'); } // Get permits with detailed information. const url = getIntegrationUrl(CORE_API_HOST, `/api/mines/${nrptiRecord._sourceRefId}/permits`); const { records: permits } = await integrationUtils.getRecords(url, getAuthHeader(this.client_token)); - // First, any mines with 'X' as their second character are considered exploratory. Remove them. - // const nonExploratoryPermits = permits.filter(permit => permit.permit_no[1].toLowerCase() !== 'x'); + return permits.filter(p => this.isValidPermit(p,nrptiRecord)); + } + + isValidPermit(permit,nrptiRecord) { + //Mine must not be historical which is indicated by an authorized year of '9999' on the latest amendment. + if ((permit.permit_amendments.length && !permit.permit_amendments[0].authorization_end_date) + || permit.permit_status_code === 'O') { + + // Do not use 'G-4-352' for Lumby + // https://bcmines.atlassian.net/browse/NRPT-684 + return !nrptiRecord.name === 'Lumby Mine' && permit.permit_no === 'G-4-352'; + } + return false; + } + + /** + * Gets valid permit for a Core mine. A valid permit must meet the following criteria: + * - Must not be exploratory + * - Must not be historical + * . + * + * @param {object} nrptiRecord NRPTI record + * @returns {object} Valid permit. + * @memberof CoreDataSource + */ + getValidPermit(permits) { + // First, any mines with 'X' as their second character are considered exploratory. Remove them unless they are the only valid permits + let nonExploratoryPermits = permits.filter(permit => permit.permit_no[1].toLowerCase() !== 'x'); + if (nonExploratoryPermits.length === 0) { + nonExploratoryPermits = permits; + } + // Second, mine must not be historical which is indicated by an authorized year of '9999' on the latest amendment. let validPermit; - for (const permit of permits) { + for (const permit of nonExploratoryPermits) { // Confirm that the most recent amendment is not historical, which is always the first index. // If 'null' then it is considered valid. // Otherwise, if the status is O, it's valid. (date 9999 check removed) - if ( - (permit.permit_amendments.length && !permit.permit_amendments[0].authorization_end_date) || - permit.permit_status_code === 'O' - ) { - // Do not use 'G-4-352' for Lumby - // https://bcmines.atlassian.net/browse/NRPT-684 - if (nrptiRecord.name === 'Lumby Mine' && permit.permit_no === 'G-4-352') { - continue; - } // There should only be a single record. If there is more then we need to identify the most // recent permit as the official valid permit @@ -309,7 +329,6 @@ class CoreDataSource { } else { validPermit = permit; } - } } return validPermit; @@ -332,14 +351,18 @@ class CoreDataSource { throw new Error('createMinePermit - nrptiRecord is required'); } - const permit = await this.getMinePermit(nrptiRecord); + const permits = await this.getMinePermits(nrptiRecord); + const validPermit = this.getValidPermit(permits); - if (!permit) { + if (!validPermit) { throw new Error('createMinePermit - Cannot find valid permit'); } // Transform the permit and amendments into single permits. Each document in an amendment will create a single permit. - const transformedPermits = await permitUtils.transformRecord(permit, nrptiRecord); + let transformedPermits = []; + for (const permit of permits) { + transformedPermits = transformedPermits.concat(await permitUtils.transformRecord(permit, nrptiRecord)); + } // To trigger flavour for this import. const preparedPermits = transformedPermits.map(amendment => ({ ...amendment, PermitBCMI: {} })); @@ -348,9 +371,9 @@ class CoreDataSource { await Promise.all(promises); return { - permitNumber: permit.permit_no, - permittee: permit.current_permittee, - permit + permitNumber: validPermit.permit_no, + permittee: validPermit.current_permittee, + validPermit }; } @@ -364,17 +387,21 @@ class CoreDataSource { */ async updateMinePermit(permitUtils, mineRecord) { // Get the updated Core permit. - const permit = await this.getMinePermit(mineRecord); + const permits = await this.getMinePermits(mineRecord); + const validPermit = this.getValidPermit(permits); - if (!permit || !permit.permit_amendments || !permit.permit_amendments.length) { + if (!validPermit || !validPermit.permit_amendments || !validPermit.permit_amendments.length) { throw new Error('updateMinePermit - Cannot find valid permit'); } // Get the current permits for the mine. - const currentPermits = await permitUtils.getMinePermits(mineRecord._sourceRefId); + const currentPermits = await permitUtils.getCurrentMinePermits(mineRecord._sourceRefId); // Transform into permits. - const transformedPermits = permitUtils.transformRecord(permit, mineRecord); + let transformedPermits = []; + for (const permit of permits) { + transformedPermits = transformedPermits.concat(await permitUtils.transformRecord(permit, mineRecord)); + } // Find the new permits that need to be created, otherwise update the permits if needed const newPermits = []; @@ -416,9 +443,9 @@ class CoreDataSource { await Promise.all(promises); return { - permitNumber: permit.permit_no, - permittee: permit.current_permittee, - permit + permitNumber: validPermit.permit_no, + permittee: validPermit.current_permittee, + validPermit }; } diff --git a/api/src/integrations/core/datasource.test.js b/api/src/integrations/core/datasource.test.js index c1f5a0705..f5e927dc7 100644 --- a/api/src/integrations/core/datasource.test.js +++ b/api/src/integrations/core/datasource.test.js @@ -147,14 +147,14 @@ describe('Core DataSource', () => { }); }); - describe('getMinePermit', () => { + describe('getMinePermits', () => { - it('throws an error when no permit is found in getMinePermit method', async () => { + it('throws an error when no permit is found in getMinePermits method', async () => { const nrptiRecord = null; const dataSource = new DataSource(null); - await expect(dataSource.getMinePermit(nrptiRecord)).rejects.toThrow('getMinePermit - required nrptiRecord is null.'); + await expect(dataSource.getMinePermits(nrptiRecord)).rejects.toThrow('getMinePermits - required nrptiRecord is null.'); }); }); diff --git a/api/src/integrations/core/permit-utils.js b/api/src/integrations/core/permit-utils.js index 789ff4658..1c5c5e093 100644 --- a/api/src/integrations/core/permit-utils.js +++ b/api/src/integrations/core/permit-utils.js @@ -103,7 +103,7 @@ class Permits extends BaseRecordUtils { return await super.updateRecord(newPermit, existingPermit); } - async getMinePermits(mineId) { + async getCurrentMinePermits(mineId) { const Permit = mongoose.model('Permit'); const permits = await Permit.find({ _schemaName: 'Permit', mineGuid: mineId }); diff --git a/api/src/integrations/core/permit-utils.test.js b/api/src/integrations/core/permit-utils.test.js index 61b5f8667..d3684ac94 100644 --- a/api/src/integrations/core/permit-utils.test.js +++ b/api/src/integrations/core/permit-utils.test.js @@ -231,13 +231,13 @@ describe('Permits class', () => { }); }); - describe('getMinePermits method', () => { + describe('getCurrentMinePermits method', () => { it('returns null if no existing record found', async () => { mongoose.model = jest.fn(() => ({ find: jest.fn(() => null), })); - const result = await permitsInstance.getMinePermits('123'); + const result = await permitsInstance.getCurrentMinePermits('123'); expect(result).toBeNull(); }); @@ -247,7 +247,7 @@ describe('Permits class', () => { find: jest.fn(() => existingRecord), })); - const result = await permitsInstance.getMinePermits('123'); + const result = await permitsInstance.getCurrentMinePermits('123'); expect(result).toEqual(existingRecord); }); }); From cf61de17fa4303639f160bd56533594ba5f0733a Mon Sep 17 00:00:00 2001 From: acatchpole <113044739+acatchpole@users.noreply.github.com> Date: Thu, 27 Jun 2024 16:35:19 -0700 Subject: [PATCH 2/2] [BUG][NRPTI-1165] token refresh for core importer (#1247) * unrelated typo fix * refactored getCoreAccessToken to return whole data obj instead of just the token * refactoring to account for a changed return type in supporting function * added a check for expired api token to the core-documents importer * fixed typo * removed migration to update bcmi collections as it is not needed. --- ...619192233-updateBCMICollectionTypeField.js | 74 ------------------- .../integrations/core-documents/datasource.js | 57 ++++++++++++-- api/src/integrations/core/datasource.js | 3 +- api/src/integrations/integration-utils.js | 16 ++-- tools/README.md | 2 +- 5 files changed, 60 insertions(+), 92 deletions(-) delete mode 100644 api/migrations/20240619192233-updateBCMICollectionTypeField.js diff --git a/api/migrations/20240619192233-updateBCMICollectionTypeField.js b/api/migrations/20240619192233-updateBCMICollectionTypeField.js deleted file mode 100644 index 6c5687661..000000000 --- a/api/migrations/20240619192233-updateBCMICollectionTypeField.js +++ /dev/null @@ -1,74 +0,0 @@ -'use strict'; - -var dbm; -var type; -var seed; - -/** - * Update the 'type' of CollectionBCMI entries - * Checks each permit in the 'record' array for its typeCode - * Assigns the type to the Collection based on the permit's typeCode, with ALG having priority - */ -exports.setup = function(options, seedLink) { - dbm = options.dbmigrate; - type = dbm.dataType; - seed = seedLink; -}; - -exports.up = async function(db) { - console.log('**** Updating bcmi collections type ****'); - const mClient = await db.connection.connect(db.connectionString, { native_parser: true }); - - try { - const nrpti = await mClient.collection('nrpti'); - - const collections = await nrpti.find({ _schemaName: 'CollectionBCMI' }).toArray(); - const permits = await nrpti.find({ _schemaName: { $in: ['Permit', 'PermitBCMI'] } }).toArray(); - - let count = 0; - - collections.forEach(async collection => { - let typeCode = ''; - // Go through each record attached to each collection - // For each record, check the typeCode - const oldType = collection.type; - for (const recordId of collection.records) { - const record = permits.find(permit => permit._id.toString() == recordId.toString()); - // ALG takes precedence, so don't overwrite - if (record && record.typeCode && typeCode != record.typeCode && typeCode != 'ALG') { - typeCode = record.typeCode; - } - } - if (typeCode) { - switch (typeCode) { - case 'OGP': - collection.type = 'Permit'; - break; - case 'ALG': - collection.type = 'Amalgamated Permit'; - break; - default: - collection.type = 'Permit Amendment'; - } - if (oldType != collection.type) { - count += 1; - await nrpti.update({ _id: collection._id }, { $set: collection }); - } - } - }); - - console.log('**** Finished updating ' + count + ' bcmi collection types ****'); - } catch (error) { - console.error(`Migration did not complete. Error processing collections: ${error.message}`); - } - - mClient.close(); -}; - -exports.down = function(db) { - return null; -}; - -exports._meta = { - version: 1 -}; diff --git a/api/src/integrations/core-documents/datasource.js b/api/src/integrations/core-documents/datasource.js index 9c78b799e..5241c3c12 100644 --- a/api/src/integrations/core-documents/datasource.js +++ b/api/src/integrations/core-documents/datasource.js @@ -40,7 +40,7 @@ class CoreDocumentsDataSource { try { // Get Core API access token. - this.client_token = await getCoreAccessToken(CORE_CLIENT_ID, CORE_CLIENT_SECRET, CORE_GRANT_TYPE); + await this.setClientToken(); // Run main process. await this.updateRecords(); @@ -68,17 +68,17 @@ class CoreDocumentsDataSource { const jobCount = process.env.PARALLEL_IMPORT_LIMIT ? parseInt(process.env.PARALLEL_IMPORT_LIMIT) : 1; const permits = await this.getPermits(); - - this.status.itemTotal = permits.length; + const numPermits = permits.length; + this.status.itemTotal = numPermits; await this.taskAuditRecord.updateTaskRecord({ itemTotal: this.status.itemTotal }); const permitUtils = new PermitUtils(this.auth_payload, RECORD_TYPE.Permit); // Push records to proccess into a Promise array to be processed in parallel. - for (let i = 0; i < permits.length; i += jobCount) { + for (let i = 0; i < numPermits; i += jobCount) { const promises = []; - for (let j = 0; j < jobCount && i + j < permits.length; j++) { - defaultLog.info(`Processing permit ${i + j + 1} out of ${permits.length}`); + for (let j = 0; j < jobCount && i + j < numPermits; j++) { + defaultLog.info(`Processing permit ${i + j + 1} out of ${numPermits}`); promises.push(this.processRecord(permits[i + j], permitUtils)); } @@ -161,7 +161,9 @@ class CoreDocumentsDataSource { if (!documentId) { throw new Error('getDownloadToken - param documentId must not be null'); } - + if (this.isAPITokenExpired(this.apiAccessExpiry)) { + this.setClientToken(); + } try { const url = getIntegrationUrl(CORE_API_HOST, `/api/download-token/${documentId}`); const { token_guid } = await integrationUtils.getRecords(url, getAuthHeader(this.client_token)); @@ -269,12 +271,51 @@ class CoreDocumentsDataSource { const transformedAmendment = permitUtils.transformRecord(permit); const result = await permitUtils.updateRecord(transformedAmendment, permit); - if(result.length && result[0].status && result[0].status === 'failure') + if (result.length && result[0].status && result[0].status === 'failure') throw Error(`permitUtils.updateRecord failed: ${result[0].errorMessage}`); } catch (error) { throw new Error(`updateAmendment - unexpected error: ${error.message}`); } } + + /** + * Sets the CORE API token and marks when the token will expire + * + * @memberof CoreDocumentsDataSource + */ + async setClientToken() { + console.log('Updating Client Token...'); + const apiAccess = await getCoreAccessToken(CORE_CLIENT_ID, CORE_CLIENT_SECRET, CORE_GRANT_TYPE); + this.apiAccessExpiry = this.getExpiryTime(apiAccess.expires_in); + this.client_token = apiAccess.access_token; + console.log('Client Token updated.'); + } + + /** + * Gives a time for when the given duration will pass with a buffer + * + * @param {int} tokenDuration the number of seconds that the token is valid for. + * @returns {int} the epoch time when the token is expected to expire ( - the buffer ) = current time + token duration - buffer + * + * @memberof CoreDocumentsDataSource + */ + getExpiryTime(tokenDuration) { + const TIME_BUFFER = 30000; + const SECONDS_TO_MILLISECONDS_MULTIPLIER = 1000; + return Date.now() + ( tokenDuration * SECONDS_TO_MILLISECONDS_MULTIPLIER ) - TIME_BUFFER; + } + + /** + * checks if the given time has passed + * + * @param {int} expiryTime the epoch time we are checking for + * @returns {boolean} true if the provided time is in the past + * + * @memberof CoreDocumentsDataSource + */ + isAPITokenExpired(expiryTime) { + return Date.now() >= expiryTime; + } } module.exports = CoreDocumentsDataSource; diff --git a/api/src/integrations/core/datasource.js b/api/src/integrations/core/datasource.js index c79af8625..211494b31 100644 --- a/api/src/integrations/core/datasource.js +++ b/api/src/integrations/core/datasource.js @@ -43,7 +43,8 @@ class CoreDataSource { try { // Get a new API access token. - this.client_token = await getCoreAccessToken(CORE_CLIENT_ID, CORE_CLIENT_SECRET, CORE_GRANT_TYPE); + const apiAccess = await getCoreAccessToken(CORE_CLIENT_ID, CORE_CLIENT_SECRET, CORE_GRANT_TYPE); + this.client_token = apiAccess.access_token; // Run main process. await this.updateRecords(); diff --git a/api/src/integrations/integration-utils.js b/api/src/integrations/integration-utils.js index afae6c715..8291ba82c 100644 --- a/api/src/integrations/integration-utils.js +++ b/api/src/integrations/integration-utils.js @@ -56,7 +56,7 @@ exports.getRecords = async function(url, options = undefined) { * @param {string} clientId Core client ID. * @param {string} clientSecret Core client secret. * @param {string} grantType Core SSO grant type. - * @returns {string} Core API access token. + * @returns {Object?} payload - the res.data obj which includes the access_token as well as its time to live. * @memberof CoreDataSource */ exports.getCoreAccessToken = async function(clientId, clientSecret, grantType) { @@ -73,9 +73,9 @@ exports.getRecords = async function(url, options = undefined) { } const requestBody = { - client_id: clientId, - client_secret: clientSecret, - grant_type: grantType + client_id: clientId, + client_secret: clientSecret, + grant_type: grantType }; const config = { @@ -92,12 +92,12 @@ exports.getRecords = async function(url, options = undefined) { throw new Error('coreLogin - unable to log in to Core API.'); } - return payload.access_token; - } + return payload; + }; /** * Creates the authentication header for protected requests. - * + * * @param {string} token Bearer token. * @param {object} additionalOptions Additional HTTP options. * @returns {object} Axios header with the bearer token set @@ -110,4 +110,4 @@ exports.getRecords = async function(url, options = undefined) { }, ...additionalOptions }; - } \ No newline at end of file + }; diff --git a/tools/README.md b/tools/README.md index c2d5b25a7..459daf6d7 100644 --- a/tools/README.md +++ b/tools/README.md @@ -66,7 +66,7 @@ Moving the **App** from `dev` (`:latest`) to `test` ```sh oc project f00029-tools oc tag nrpti:test nrpti:test-backup -oc tag nrpti:lastest nrpti:test +oc tag nrpti:latest nrpti:test ``` Moving **NRCED** from `dev` (`:latest`) to `test`