From 9a4624ec4ca5b4e7ce3afc42e7a51df901d1e37a Mon Sep 17 00:00:00 2001 From: David Date: Mon, 16 Dec 2024 08:36:27 -0800 Subject: [PATCH] copy over migrate.js with small sdk update, small tweaks Signed-off-by: David --- .../migrate-historical/2000-2016/migrate.js | 628 +++++++++++++++ .../legacy-data-constants.js | 714 ++++++++++++++++++ .../legacy-data-functions.js | 297 ++++++++ 3 files changed, 1639 insertions(+) create mode 100644 arSam/migrations/migrate-historical/2000-2016/migrate.js create mode 100644 arSam/migrations/migrate-historical/legacy-data-constants.js create mode 100644 arSam/migrations/migrate-historical/legacy-data-functions.js diff --git a/arSam/migrations/migrate-historical/2000-2016/migrate.js b/arSam/migrations/migrate-historical/2000-2016/migrate.js new file mode 100644 index 0000000..516108c --- /dev/null +++ b/arSam/migrations/migrate-historical/2000-2016/migrate.js @@ -0,0 +1,628 @@ +const AWS = require('aws-sdk'); +const readXlsxFile = require('read-excel-file/node'); +const { + TABLE_NAME, + dynamoClient, + marshall, + TransactWriteItemsCommand +} = require('../../../layers/baseLayer/baseLayer'); +const { + createCSV, + getDBSnapshot, + validateSchema, + updateConsoleProgress, + determineActivities, + createLegacySubAreaObject, + createLegacyParkObject, + getConsoleInput, + createLegacyRecordObject, + clientIDsAR +} = require('../legacy-data-functions'); +const { schema } = require('../legacy-data-constants'); + +const MAX_TRANSACTION_SIZE = 25; + +// Change file path based on year(s) being uploaded +let filePath = 'ar-records-2003-2005.xlsx'; + +let xdb; // Existing DB snapshot +let rows; // Raw historical data rows. +let changes = []; // Rows that will be changed in the migration +let newParks = {}; // Legacy parks to be created in the migration. +let newSubAreas = {}; // Legacy subareas to be created in the migration. +let newRecordsCount = 0; // Quantity of legacy records to be created. +let failures = []; // List of errors encountered. + +async function run() { + console.log('A&R HISTORICAL DATA MIGRATION\n'); + + try { + if (process.argv.length <= 2) { + console.log('Invalid parameters.'); + console.log(''); + console.log('Usage: node migrate.js '); + console.log(''); + console.log('Options'); + console.log(' : dev/test/prod'); + console.log(''); + console.log('example: node migrate.js dev xxxx'); + console.log(''); + return; + } else { + env = process.argv[2]; + const environment = env === 'prod' ? '' : env + '.'; + const clientID = clientIDsAR[env]; + } + // 1. validate schema (safety check). + validateSchema(schema); + // 2. Get simplified map of existing db so we don't have to continually hit the db when checking for existing entries. + xdb = await getDBSnapshot(); + // 3. Get new data + rows = await getData(); + // 4. Collect proposed changes in change log for user review. + await createChangelog(); + // 5. Execute change log changes. + await executeChangelog(); + } catch (error) { + console.log('ERROR:', error); + return; + } +} + +async function getData() { + console.log('Loading data...'); + try { + let { rows, errors } = await readXlsxFile(filePath, { schema }); + console.log('Items found:', rows.length); + return rows; + } catch (error) { + throw `Error loading historical data: ` + error; + } +} + +async function createChangelog() { + console.log('********************'); + console.log('CREATING CHANGE LOG'); + + // Get largest subareaId in the system so we can dynamically generate new subareaIds. + let ids = []; + for (const park in xdb) { + for (const subarea in xdb[park]) { + ids.push(parseInt(subarea, 10) ? parseInt(subarea, 10) : 0); + } + } + let nextSubAreaId = Math.max(...ids) + 1; + + try { + let intervalStartTime = new Date().getTime(); + for (const row of rows) { + let subAreaId; + updateConsoleProgress(intervalStartTime, 'Collecting', rows.indexOf(row) + 1, rows.length, 1); + + // Check data for fatal errors and omissions (missing orcs, names, etc). + // We can't load these in, so we should flag them. + if (!validateRow(row)) { + failures.push({ item: { data: row }, reason: `Invalid row - critical data error or omission` }); + continue; + } + + // Determine legacy record activities by field presence. + let activities = []; + try { + activities = determineActivities(row, schema); + if (activities.length === 0) { + throw 'No activities found.'; + } + newRecordsCount += activities.length ? activities.length : 0; + } catch (error) { + // Failed to determine activities. + failures.push({ + item: { data: row }, + reason: `Error collecting activities from historical data. ${error}`, + error: error + }); + continue; + } + + // Check if park exists for legacy record. + try { + if (!xdb.hasOwnProperty(row.legacy_orcs) && !newParks.hasOwnProperty(row.legacy_orcs)) { + // Park doesn't exist, create new + let newPark = createLegacyParkObject(row); + newParks[row.legacy_orcs] = newPark; + } + } catch (error) { + // Failed to create new park. + failures.push({ + item: { data: row }, + reason: `An error occurred while creating a legacy park: ${error}`, + error: error + }); + continue; + } + + try { + // Check if subarea exists. Get subarea name. + // We must also store the existing or newly created subarea id for the row + let subAreaNameArray = row.legacy_parkSubArea.split(' - '); + subAreaNameArray.splice(0, 1); + let checkSubAreaName = subAreaNameArray.join(' - '); + let existingSubArea = []; + // Determine if subarea already exists in xdb. + if (xdb[row.legacy_orcs]) { + existingSubArea = Object.keys(xdb[row.legacy_orcs]).filter( + (id) => xdb[row.legacy_orcs]?.[id].subAreaName === checkSubAreaName + ); + subAreaId = existingSubArea[0] ? existingSubArea[0] : null; + } + // Determine if subarea is already in legacy subarea list. + if (!existingSubArea.length) { + existingSubArea = Object.keys(newSubAreas).filter((id) => newSubAreas?.[id].subAreaName === checkSubAreaName); + subAreaId = existingSubArea[0] ? existingSubArea[0] : null; + } + // If nothing found, subarea doesn't exist. Create legacy. + if (!existingSubArea.length) { + let newSubArea; + if (row.legacy_orcs === 'HIST') { + newSubArea = createLegacySubAreaObject(row, 'HIST', checkSubAreaName, activities); + subAreaId = 'HIST'; + } else { + newSubArea = createLegacySubAreaObject(row, nextSubAreaId, checkSubAreaName, activities); + subAreaId = nextSubAreaId; + nextSubAreaId++; + } + newSubAreas[newSubArea.sk] = newSubArea; + } + } catch (error) { + // Failed to create new subarea. + failures.push({ + item: { data: row }, + reason: `An error occurred while creating a legacy subarea: ${error}`, + error: error + }); + continue; + } + + if (!subAreaId) { + // Something occurred with collecting the subarea id. + failures.push({ + item: { data: row }, + reason: `An error occurred determining a subarea id for the historical data` + }); + continue; + } + + // If we get here, the row is vetted and can be migrated. + changes.push({ data: row, activities: activities, subAreaId: subAreaId }); + } + + // Get user to review and approve change log. + process.stdout.write('\n'); + console.log('********************'); + console.log('CHANGE LOG CREATED:'); + console.log( + 'Please review and approve the following change log. The migration will commence once the change log is approved.' + ); + console.log('Note: The changes listed below will be captured in an output file for future review.\n'); + + // Review new parks. + if (Object.keys(newParks).length) { + let viewParks = await getConsoleInput( + `There are ${Object.keys(newParks).length || 0} legacy parks to be created.\nDo you want to review these parks? [Y/N] >>> ` + ); + process.stdout.write('\n'); + if (viewParks === 'Y' || viewParks === 'y') { + console.log('New Parks:', newParks); + await awaitContinue(); + process.stdout.write('\n'); + } + } + + // Review new subareas. + if (Object.keys(newSubAreas).length) { + let viewSubAreas = await getConsoleInput( + `There are ${Object.keys(newSubAreas).length || 0} legacy subareas to be created.\nDo you want to review these subareas? [Y/N] >>> ` + ); + process.stdout.write('\n'); + if (viewSubAreas === 'Y' || viewSubAreas === 'y') { + console.log('New Subareas:', newSubAreas); + await awaitContinue(); + process.stdout.write('\n'); + } + } + + // Review failures. + if (failures.length) { + process.stdout.write('\n'); + let viewErrors = await getConsoleInput( + `There were ${failures.length} rows containing errors that will prevent their contained data from being migrated.\nThey will not be migrated, but they will be captured for future processing.\nDo you want to review these failures? [Y/N] >>> ` + ); + process.stdout.write('\n'); + if (viewErrors === 'Y' || viewErrors === 'y') { + let failureList = []; + for (const failure of failures) { + let failureString = `FAILURE: ${failure?.item?.legacy_parkSubArea} (${failure?.item?.legacy_month / failure?.item?.legacy_year}): ${failure?.reason}`; + let index = failureList.findIndex((item) => item.string == failureString); + if (index !== -1) { + failureList[index].count++; + } else { + failureList.push({ string: failureString, count: 1 }); + } + } + for (const item of failureList) { + console.log(`${item.string} (${item.count} INSTANCE(S))`); + } + process.stdout.write('\n'); + await awaitContinue(); + } + } + + // Approve change log. + process.stdout.write('\n'); + console.log('********************'); + console.log('CHANGE LOG COMPLETE.'); + let viewRecords = await getConsoleInput( + `Legacy records to be created: ${newRecordsCount}\nDo you approve the proposed change log? Responding "Y" will begin the migration process. [Y/N] >>> ` + ); + if (viewRecords !== 'Y' && viewRecords !== 'y') { + throw 'User must approve of the change log (enter "Y"). Migration process cancelled.'; + } + process.stdout.write('\n'); + // Change log is approved. + return; + } catch (error) { + process.stdout.write('\n'); + throw `Failed to create change log: ${error}`; + } +} + +async function executeChangelog() { + // Execute the change log via batch transactions to reduce migrations time. + // We should not need to do any conditional checking for parks/subareas at this point - + // All checking should have been done in the createChangeLog step. + // We still should do conditional check for records, if we want true idempotency + // where new records are NOT overwritten on re-run =. + console.log('********************'); + console.log('EXECUTING MIGRATION:'); + + let successes = []; + + // 1. Put legacy parks into DB. + // Create transaction object - We can only transact 25 items at a time. + if (Object.keys(newParks).length) { + try { + console.log('Legacy parks:', Object.keys(newParks).length); + let parkTransactObjList = []; + let intervalStartTime = new Date().getTime(); + let legacyParksList = Object.keys(newParks).map((orcs) => newParks[orcs]); + let parkTransactChunk = { TransactItems: [] }; + try { + for (const park of legacyParksList) { + updateConsoleProgress( + intervalStartTime, + 'Creating legacy parks transaction', + legacyParksList.indexOf(park) + 1, + legacyParksList.length, + 1 + ); + // If MAX_TRANSACTION_SIZE limit reached, start new transaction chunk + if (parkTransactChunk.TransactItems.length + 1 > MAX_TRANSACTION_SIZE) { + parkTransactObjList.push(parkTransactChunk); + parkTransactChunk = { TransactItems: [] }; + } + let parkObj = { + TableName: TABLE_NAME, + Item: marshall(park), + ConditionExpression: 'attribute_not_exists(pk) AND attribute_not_exists(sk)' + }; + parkTransactChunk.TransactItems.push({ + Put: parkObj + }); + } + // collect the remainder of chunks + if (parkTransactChunk.TransactItems.length) { + parkTransactObjList.push(parkTransactChunk); + } + process.stdout.write('\n'); + } catch (error) { + process.stdout.write('\n'); + throw `Failed to create legacy parks transaction. Park transaction must succeed to continue: ${error}`; + } + + // Execute park transaction + try { + intervalStartTime = new Date().getTime(); + for (const transaction of parkTransactObjList) { + updateConsoleProgress( + intervalStartTime, + 'Executing legacy parks transaction', + parkTransactObjList.indexOf(transaction) + 1, + parkTransactObjList.length, + 1 + ); + await dynamoClient(new TransactWriteItemsCommand(transaction)); + } + } catch (error) { + throw `Failed to execute legacy parks transaction. Park transaction must succeed to continue: ${error}`; + } + process.stdout.write('\n'); + console.log('Legacy parks completed.\n'); + } catch (error) { + process.stdout.write('\n'); + throw `Failed to execute change log: ${error}`; + } + } + + // 2. Put legacy subareas into DB. + // Create transaction object - We can only transact 25 items at a time. + if (Object.keys(newSubAreas).length) { + console.log('Legacy subareas:', Object.keys(newSubAreas).length); + let subAreaTransactObjList = []; + let intervalStartTime = new Date().getTime(); + let legacySubAreasList = Object.keys(newSubAreas).map((id) => newSubAreas[id]); + let subAreaTransactChunk = { TransactItems: [] }; + let touchedParksList = []; + try { + for (const subArea of legacySubAreasList) { + updateConsoleProgress( + intervalStartTime, + 'Creating legacy subarea transaction', + legacySubAreasList.indexOf(subArea) + 1, + legacySubAreasList.length, + 1 + ); + if ( + subAreaTransactChunk.TransactItems.length + 2 > MAX_TRANSACTION_SIZE || + touchedParksList.indexOf(subArea.orcs) !== -1 + ) { + // We need to execute 2 transactions per subarea, and ensure both are successful. + // If MAX_TRANSACTION_SIZE limit reached, start new transaction chunk. + // Additionally, We have to update the parent park with a list of its subareas. + // We cannot update the same park twice in the same transaction chunk. + // Whenever we hit duplicate parks, we have to break the transaction chunk up. + subAreaTransactObjList.push(subAreaTransactChunk); + subAreaTransactChunk = { TransactItems: [] }; + touchedParksList = []; + } + let parkUpdateObj = { + TableName: TABLE_NAME, + Key: { + pk: { S: `park` }, + sk: { S: subArea.orcs } + }, + UpdateExpression: 'SET subAreas = list_append(subAreas, :subAreas)', + ExpressionAttributeValues: { + ':subAreas': { + L: [ + { + M: { + id: { S: subArea.sk }, + name: { S: subArea.subAreaName }, + isLegacy: { BOOL: true } + } + } + ] + } + } + }; + subAreaTransactChunk.TransactItems.push({ + Update: parkUpdateObj + }); + touchedParksList.push(subArea.orcs); + + // Create legacy subarea + let subAreaObj = { + TableName: TABLE_NAME, + Item: Object.assign(marshall(subArea), { activities: { SS: subArea.activities } }), + ConditionExpression: 'attribute_not_exists(pk) AND attribute_not_exists(sk)' + }; + subAreaTransactChunk.TransactItems.push({ + Put: subAreaObj + }); + } + // collect the remainder of chunks + if (subAreaTransactChunk.TransactItems.length) { + subAreaTransactObjList.push(subAreaTransactChunk); + } + process.stdout.write('\n'); + } catch (error) { + process.stdout.write('\n'); + throw `Failed to create legacy subarea transaction. Subarea transaction must succeed to continue: ${error}`; + } + } + + // 3. Put legacy records into the DB. + // Create transaction object - We can only transact 25 items at a time. + console.log('Legacy records:', newRecordsCount); + let transactionMap = []; + let transactionMapChunk = { rows: [], transactions: { TransactItems: [] } }; + let intervalStartTime = new Date().getTime(); + try { + for (const item of changes) { + try { + transactionMapChunk.rows.push(item); + updateConsoleProgress( + intervalStartTime, + 'Creating legacy record transaction', + changes.indexOf(item) + 1, + changes.length, + 100 + ); + if (transactionMapChunk.transactions.TransactItems.length + item.activities.length > MAX_TRANSACTION_SIZE) { + // Check if there are enough slots left in the transaction chunk to carry out every legacy record in the item. + // If MAX_TRANSACTION_SIZE limit reached, start new transaction chunk. + transactionMap.push(transactionMapChunk); + transactionMapChunk = { rows: [], transactions: { TransactItems: [] } }; + } + // Create new record for every activity in the time + for (const activity of item.activities) { + let activityObj = createLegacyRecordObject(item.data, activity, item.subAreaId); + let recordObj = { + TableName: TABLE_NAME, + Item: marshall(activityObj), + ConditionExpression: 'attribute_not_exists(pk)' + }; + transactionMapChunk.transactions.TransactItems.push({ + Put: recordObj + }); + } + } catch (error) { + failures.push({ item: item, reason: `Error in legacy records transaction creation: ${error}`, error: error }); + } + } + // collect the remainder of chunks + if (transactionMapChunk.transactions.TransactItems.length) { + transactionMap.push(transactionMapChunk); + } + process.stdout.write('\n'); + } catch (error) { + process.stdout.write('\n'); + throw `Failed to create legacy records transaction. The migration cannot continue: ${error}`; + } + + // Execute record transactions + successfulRecordCount = 0; + intervalStartTime = new Date().getTime(); + for (const transaction of transactionMap) { + try { + updateConsoleProgress( + intervalStartTime, + 'Executing legacy record transaction', + transactionMap.indexOf(transaction) + 1, + transactionMap.length, + 10 + ); + await dynamoClient.send(new TransactWriteItemsCommand(transaction.transactions)); + successes = successes.concat(transaction.rows); + successfulRecordCount += transaction.transactions.TransactItems.length; + } catch (error) { + for (const row of transaction.rows) { + failures.push({ + item: row, + reason: `This row failed as part of an aborted bulk transaction and might not include any errors: ${error}`, + error: error + }); + } + } + } + process.stdout.write('\n'); + console.log('Legacy records complete.\n'); + + console.log('********************'); + console.log('MIGRATION SUMMARY:'); + + console.log('Legacy records created:', successfulRecordCount); + console.log('(Rows migrated:', successes.length, ')'); + console.log('Failures:', failures.length); + + // Review newly created areas. + let saveAreas = await getConsoleInput( + `Do you want to save lists of the successfully created parks and subarea [Y/N] >>> ` + ); + if (saveAreas === 'Y' || saveAreas === 'y') { + // Save parks to file + if (Object.keys(newParks).length) { + let parksCSVList = []; + let parks = Object.keys(newParks); + let parkSchema = Object.keys(newParks[parks[0]]); + for (const park in newParks) { + parksCSVList.push(newParks[park]); + } + createCSV(parkSchema, parksCSVList, `legacy_parks${new Date().toISOString()}`); + process.stdout.write('\n'); + } + // Save subareas to file + if (Object.keys(newSubAreas).length) { + let subAreaCSVList = []; + let subAreas = Object.keys(newSubAreas); + let subAreaSchema = Object.keys(newSubAreas[subAreas[0]]); + for (const subArea in newSubAreas) { + subAreaCSVList.push(newSubAreas[subArea]); + } + createCSV(subAreaSchema, subAreaCSVList, `legacy_subareas${new Date().toISOString()}`); + process.stdout.write('\n'); + } + } + + let saveSchema = []; + for (const entry in schema) { + saveSchema.push(schema[entry].prop); + } + + // Review successes. + let saveSuccesses = await getConsoleInput( + `Do you want to save a list of the successfully created legacy records? [Y/N] >>> ` + ); + if (saveSuccesses === 'Y' || saveSuccesses === 'y') { + // Save successes to file + let successData = []; + for (const success of successes) { + successData.push(success.data); + } + createCSV(saveSchema, successData, `migration_successes${new Date().toISOString()}`); + process.stdout.write('\n'); + } + // Review failures. + let saveFailures = await getConsoleInput( + `Do you want to save a list of the items that failed to migrate? [Y/N] >>> ` + ); + if (saveFailures === 'Y' || saveFailures === 'y') { + // Save failures to file + let failureData = []; + for (const failure of failures) { + failure.item.data['failureReason'] = failure.reason; + failureData.push(failure.item.data); + } + let failureSchema = [...saveSchema].concat('failureReason'); + createCSV(failureSchema, failureData, `migration_failures${new Date().toISOString()}`); + process.stdout.write('\n'); + } + + // Sanity check + const diff = rows.length - successes.length - failures.length; + if (diff === 0) { + console.log(`All ${rows.length} items accounted for.`); + } else if (diff > 0) { + console.log(`${diff} items unaccounted for.`); + } else { + console.log(`${Math.abs(diff)} items accounted for more than once.`); + } + + console.log('********************'); + console.log('MIGRATION COMPLETE:'); + console.log('********************'); +} + +function validateRow(row) { + // Row is invalid under any of the following conditions: + if ( + // Row is full of test data: + row.legacy_orcs && + row.legacy_orcs === 'HIST' + ) { + return true; + } + if ( + // Missing ORCS + !row.legacy_orcs || + // ORCS is not a number + !parseInt(row.legacy_orcs, 10) || + // Missing Park Name + !row.legacy_park || + // Missing Subarea Name + !row.legacy_parkSubArea || + // Missing Year + !row.legacy_year || + // Missing Month + !row.legacy_month + ) { + return false; + } + return true; +} + +async function awaitContinue() { + await getConsoleInput('Press ENTER to continue. >>>'); +} + +run(); diff --git a/arSam/migrations/migrate-historical/legacy-data-constants.js b/arSam/migrations/migrate-historical/legacy-data-constants.js new file mode 100644 index 0000000..3ec7bd9 --- /dev/null +++ b/arSam/migrations/migrate-historical/legacy-data-constants.js @@ -0,0 +1,714 @@ +// Every column in the legacy data should fall under one of these categories. +// If we can't ascertain one of the first 0-7 categories, assign to cat 8 'Legacy' + +const activitiesEnum = { + 0: 'Meta', + 1: 'Frontcountry Camping', + 2: 'Frontcountry Cabins', + 3: 'Backcountry Camping', + 4: 'Backcountry Cabins', + 5: 'Group Camping', + 6: 'Day Use', + 7: 'Boating', + 8: 'Legacy Data' +}; + +// We must provide a schema to guarantee the matchup of legacy fields to the new system. +// This schema is the master map of legacy to new. + +/** + * '' : { + * prop: '', + * type: + * activity: (which new activity does the legacy field belong to? For sorting legacy data records.) + * hasMatch: (does the legacy field have an exact map to the new system?, If no, new legacy field created.) + * fieldMap: '' + * } + */ +const schema = { + ORCS: { + prop: 'legacy_orcs', + type: String, + activity: activitiesEnum[0], + hasMatch: true, + fieldMap: 'orcs', + fieldName: 'ORCS' + }, + Region: { + prop: 'legacy_region', + type: String, + activity: activitiesEnum[0], + hasMatch: true, + fieldMap: 'region', + fieldName: 'Region' + }, + Section: { + prop: 'legacy_section', + type: String, + hasMatch: true, + activity: activitiesEnum[0], + fieldMap: 'section', + fieldName: 'Section' + }, + Bundle: { + prop: 'legacy_bundle', + type: String, + activity: activitiesEnum[0], + hasMatch: true, + fieldMap: 'bundle', + fieldName: 'Bundle' + }, + Park: { + prop: 'legacy_park', + type: String, + activity: activitiesEnum[0], + hasMatch: true, + fieldMap: 'parkName', + fieldName: 'Park Name' + }, + 'Park Sub Area': { + prop: 'legacy_parkSubArea', + type: String, + activity: activitiesEnum[0], + hasMatch: true, + fieldMap: 'subAreaName', + fieldName: 'Sub Area Name' + }, + Year: { + prop: 'legacy_year', + type: String, + activity: activitiesEnum[0], + hasMatch: false, + fieldMap: 'legacy_year', + fieldName: null + }, + Month: { + prop: 'legacy_month', + type: String, + activity: activitiesEnum[0], + hasMatch: false, + fieldMap: 'legacy_month', + fieldName: null + }, + 'Regular Frontcountry Camping Parties': { + prop: 'legacy_frontcountryCampingRegularCampingParties', + type: Number, + activity: activitiesEnum[1], + hasMatch: true, + fieldMap: 'campingPartyNightsAttendanceStandard', + fieldName: 'Frontcountry Camping - Camping Party Nights - Standard' + }, + 'Senior Frontcountry Camping Parties': { + prop: 'legacy_frontcountryCampingSeniorCampingParties', + type: Number, + activity: activitiesEnum[1], + hasMatch: true, + fieldMap: 'campingPartyNightsAttendanceSenior', + fieldName: 'Frontcountry Camping - Camping Party Nights - Senior' + }, + 'SSCFE Frontcountry Camping Parties': { + prop: 'legacy_frontcountryCampingSSCFECampingParties', + type: Number, + activity: activitiesEnum[1], + hasMatch: true, + fieldMap: 'campingPartyNightsAttendanceSocial', + fieldName: 'Frontcountry Camping - Camping Party Nights - Social' + }, + 'Long-stay Frontcountry Camping Parties': { + prop: 'legacy_frontcountryCampingLongStayCampingParties', + type: Number, + activity: activitiesEnum[1], + hasMatch: true, + fieldMap: 'campingPartyNightsAttendanceLongStay', + fieldName: 'Frontcountry Camping - Camping Party Nights - Long stay' + }, + 'Frontcountry Cabin Parties': { + prop: 'legacy_frontcountryCabinsCabinParties', + type: Number, + activity: activitiesEnum[2], + hasMatch: true, + fieldMap: 'totalAttendanceParties', + fieldName: 'Frontcountry Cabins - Parties' + }, + // Note necessary additional whitespace in title + 'Total Frontcountry Camping Parties': { + prop: 'legacy_frontcountryCampingTotalCampingParties', + type: Number, + activity: activitiesEnum[1], + hasMatch: false, + fieldMap: 'legacy_frontcountryCampingTotalCampingParties', + fieldName: 'Frontcountry Camping - Camping Party Nights - Total attendance' + }, + // Can't ascertain whether both Frontcountry Cabins, Group Camping, and Frontcountry Camping are included in this or not. + // Note necessary additional whitespace in title + 'Total Frontcountry Camping People': { + prop: 'legacy_frontcountryCampingTotalCampingAttendancePeople', + type: Number, + activity: activitiesEnum[8], + hasMatch: false, + fieldMap: 'legacy_frontcountryCampingTotalCampingAttendancePeople', + fieldName: null + }, + 'Total Gross Frontcountry Cabin Revenue': { + prop: 'legacy_frontcountryCabinsTotalCabinGrossRevenue', + type: Number, + activity: activitiesEnum[2], + hasMatch: true, + fieldMap: 'revenueGrossCamping', + fieldName: 'Frontcountry Cabins - Camping - Gross camping revenue' + }, + 'Total Gross Frontcountry Camping Revenue': { + prop: 'legacy_frontcountryCampingTotalCampingGrossRevenue', + type: Number, + activity: activitiesEnum[1], + hasMatch: false, + fieldMap: 'legacy_frontcountryCampingTotalCampingGrossRevenue', + fieldName: 'Frontcountry Camping - Camping Party Nights - Gross camping revenue' + }, + 'Total Net Frontcountry Camping Revenue': { + prop: 'legacy_frontcountryCampingTotalCampingNetRevenue', + type: Number, + activity: activitiesEnum[1], + hasMatch: false, + fieldMap: 'legacy_frontcountryCampingTotalCampingNetRevenue', + fieldName: 'Frontcountry Camping - Camping Party Nights - Net revenue' + }, + '# Paid Regular 2nd Cars': { + prop: 'legacy_frontcountryCampingRegularSecondCarsAttendance', + type: Number, + activity: activitiesEnum[1], + hasMatch: true, + fieldMap: 'secondCarsAttendanceStandard', + fieldName: 'Frontcountry Camping - Second Cars - Standard' + }, + // note necessary additional whitespace here + '# Senior 2nd Cars': { + prop: 'legacy_frontcountryCampingSeniorSecondCarsAttendance', + type: Number, + activity: activitiesEnum[1], + hasMatch: true, + fieldMap: 'secondCarsAttendanceSenior', + fieldName: 'Frontcountry Camping - Second Cars - Senior' + }, + '# SSCFE 2nd Cars': { + prop: 'legacy_frontcountryCampingSSCFESecondCarsAttendance', + type: Number, + activity: activitiesEnum[1], + hasMatch: true, + fieldMap: 'secondCarsAttendanceSocial', + fieldName: 'Frontcountry Camping - Second Cars - Social' + }, + 'Total # 2nd Cars': { + prop: 'legacy_frontcountryCampingTotalSecondCarsAttendance', + type: Number, + activity: activitiesEnum[1], + hasMatch: false, + fieldMap: 'legacy_frontcountryCampingTotalSecondCarsAttendance', + fieldName: 'Frontcountry Camping - Second Cars - Total attendance' + }, + 'Gross Second Car Revenue': { + prop: 'legacy_frontcountryCampingSecondCarsGrossRevenue', + type: Number, + activity: activitiesEnum[1], + hasMatch: true, + fieldMap: 'secondCarsRevenueGross', + fieldName: 'Frontcountry Camping - Second Cars - Gross 2nd car revenue' + }, + 'Net 2nd Car Revenue': { + prop: 'legacy_frontcountryCampingSecondCarsNetRevenue', + type: Number, + activity: activitiesEnum[1], + hasMatch: false, + fieldMap: 'legacy_frontcountryCampingSecondCarsNetRevenue', + fieldName: null + }, + 'Frontcountry Camping Variance Note': { + prop: 'legacy_frontcountryCampingVarianceNote', + type: String, + activity: activitiesEnum[1], + hasMatch: true, + fieldMap: 'notes', + fieldName: 'Frontcountry Camping - Variance Notes' + }, + '# Backcountry Persons': { + prop: 'legacy_backcountryCampingAttendancePeople', + type: Number, + activity: activitiesEnum[3], + hasMatch: true, + fieldMap: 'people', + fieldName: 'Backcountry Camping - People - People' + }, + // Can't ascertain whether both Backcountry Cabins and Backcountry Camping are included in this or not. + 'Gross Backcountry Revenue': { + prop: 'legacy_backcountryTotalGrossRevenue', + type: Number, + activity: activitiesEnum[8], + hasMatch: false, + fieldMap: 'legacy_backcountryTotalGrossRevenue', + fieldName: null + }, + // Can't ascertain whether both Backcountry Cabins and Backcountry Camping are included in this or not. + 'Net Backcountry Revenue': { + prop: 'legacy_backcountryTotalNetRevenue', + type: Number, + activity: activitiesEnum[8], + hasMatch: false, + fieldMap: 'legacy_backcountryTotalNetRevenue', + fieldName: null + }, + 'Camping Backcountry Variance Note': { + prop: 'legacy_backcountryCampingVarianceNote', + activity: activitiesEnum[3], + type: String, + hasMatch: true, + fieldMap: 'notes', + fieldName: 'Backcountry Camping - Variance Notes' + }, + '# Backcountry Cabin Adults': { + prop: 'legacy_backcountryCabinsAttendanceAdults', + type: Number, + activity: activitiesEnum[4], + hasMatch: true, + fieldMap: 'peopleAdult', + fieldName: 'Backcountry Cabins - People - Adult' + }, + '# Backcountry Cabin Child': { + prop: 'legacy_backcountryCabinsAttendanceKids', + type: Number, + activity: activitiesEnum[4], + hasMatch: true, + fieldMap: 'peopleChild', + fieldName: 'Backcountry Cabins - People - Child' + }, + '# Backcountry Cabin Family': { + prop: 'legacy_backcountryCabinsAttendanceFamilies', + type: Number, + activity: activitiesEnum[4], + hasMatch: true, + fieldMap: 'peopleFamily', + fieldName: 'Backcountry Cabins - People - Family' + }, + 'Total # Backcountry Cabin Persons': { + prop: 'legacy_backcountryCabinsTotalAttendancePeople', + type: Number, + activity: activitiesEnum[4], + hasMatch: false, + fieldMap: 'legacy_backcountryCabinsTotalAttendancePeople', + fieldName: 'Backcountry Cabins - People - Total people' + }, + 'Gross Backcountry Cabin Revenue': { + prop: 'legacy_backcountryCabinsGrossRevenue', + type: Number, + activity: activitiesEnum[4], + hasMatch: true, + fieldMap: 'revenueFamily', + fieldName: 'Backcountry Cabins - Family - Gross family revenue' + }, + 'Net Total Backcountry Cabin Revenue': { + prop: 'legacy_backcountryCabinsNetRevenue', + type: Number, + activity: activitiesEnum[4], + hasMatch: false, + fieldMap: 'legacy_backcountryCabinsNetRevenue', + fieldName: 'Backcountry Cabins - Family - Net revenue' + }, + 'Backcountry Cabin Variance Note': { + prop: 'legacy_backcountryCabinsVarianceNote', + type: String, + activity: activitiesEnum[4], + hasMatch: true, + fieldMap: 'notes', + fieldName: 'Backcountry Cabins - Variance Notes' + }, + // Can't ascertain whether both Backcountry Cabins and Backcountry Camping are included in this or not. + 'Total Backcountry People': { + prop: 'legacy_backcountryTotalAttendancePeople', + type: Number, + activity: activitiesEnum[8], + hasMatch: false, + fieldMap: 'legacy_backcountryTotalAttendancePeople', + fieldName: null + }, + '# Youth Group Party Members': { + prop: 'legacy_groupCampingYouthRateAttendancePeople', + type: Number, + activity: activitiesEnum[5], + hasMatch: true, + fieldMap: 'youthRateGroupsAttendancePeople', + fieldName: 'Group Camping - Youth Rate - People' + }, + '# Youth Group Nights': { + prop: 'legacy_groupCampingYouthRateNights', + type: Number, + activity: activitiesEnum[5], + hasMatch: true, + fieldMap: 'youthRateGroupsAttendanceGroupNights', + fieldName: 'Group Camping - Youth Rate - Group nights' + }, + 'Gross Youth Group Revenue': { + prop: 'legacy_groupCampingYouthRateGrossRevenue', + type: Number, + activity: activitiesEnum[5], + hasMatch: true, + fieldMap: 'youthRateGroupsRevenueGross', + fieldName: 'Group Camping - Youth Rate - Gross youth group revenue' + }, + '# Group Adults': { + prop: 'legacy_groupCampingStandardAttendanceAdults', + type: Number, + activity: activitiesEnum[5], + hasMatch: true, + fieldMap: 'standardRateGroupsTotalPeopleAdults', + fieldName: 'Group Camping - Standard Rate - Adults' + }, + '# Group Youth': { + prop: 'legacy_groupCampingStandardAttendanceYouth', + type: Number, + activity: activitiesEnum[5], + hasMatch: true, + fieldMap: 'standardRateGroupsTotalPeopleYouth', + fieldName: 'Group Camping - Standard Rate - Youth' + }, + '# Group Children': { + prop: 'legacy_groupCampingStandardAttendanceKids', + type: Number, + activity: activitiesEnum[5], + hasMatch: true, + fieldMap: 'standardRateGroupsTotalPeopleKids', + fieldName: 'Group Camping - Standard Rate - Kids' + }, + 'Total # Group Persons': { + prop: 'legacy_groupCampingStandardTotalAttendancePeople', + type: Number, + activity: activitiesEnum[5], + hasMatch: false, + fieldMap: 'legacy_groupCampingStandardTotalAttendancePeople', + fieldName: 'Group Camping Total Attendance (People)' + }, + '# Regular Group Nights': { + prop: 'legacy_groupCampingStandardNights', + type: Number, + activity: activitiesEnum[5], + hasMatch: true, + fieldMap: 'standardRateGroupsTotalPeopleStandard', + fieldName: 'Group Camping - Standard Rate - Standard' + }, + 'Gross Regular Group Revenue': { + prop: 'legacy_groupCampingStandardGrossRevenue', + type: Number, + activity: activitiesEnum[5], + hasMatch: true, + fieldMap: 'standardRateGroupsRevenueGross', + fieldName: 'Group Camping - Standard Rate - Gross standard group revenue' + }, + 'Net Total Group Camping Revenue': { + prop: 'legacy_groupCampingTotalNetRevenue', + type: Number, + activity: activitiesEnum[5], + hasMatch: false, + fieldMap: 'legacy_groupCampingTotalNetRevenue', + fieldName: 'Net Total Group Camping Revenue' + }, + 'Gross Total Group Camping Revenue': { + prop: 'legacy_groupCampingTotalGrossRevenue', + type: Number, + activity: activitiesEnum[5], + hasMatch: false, + fieldMap: 'legacy_groupCampingTotalGrossRevenue', + fieldName: 'Gross Total Group Camping Revenue' + }, + 'Camping Group Variance Note': { + prop: 'legacy_groupCampingVarianceNote', + type: String, + activity: activitiesEnum[5], + hasMatch: true, + fieldMap: 'notes', + fieldName: 'Group Camping - Variance Notes' + }, + 'Gross Sani Station Revenue': { + prop: 'legacy_dayUseMiscSaniStationGrossRevenue', + type: Number, + activity: activitiesEnum[1], + hasMatch: true, + fieldMap: 'otherRevenueGrossSani', + fieldName: 'Frontcountry Camping - Other - Gross sani revenue' + }, + 'Net Sani Station Revenue': { + prop: 'legacy_dayUseMiscSaniStationNetRevenue', + type: Number, + activity: activitiesEnum[1], + hasMatch: false, + fieldMap: 'legacy_dayUseMiscSaniStationNetRevenue', + fieldName: 'Frontcountry Camping - Other - Net sani revenue' + }, + 'Gross Shower Revenue': { + prop: 'legacy_dayUseMiscShowerGrossRevenue', + type: Number, + activity: activitiesEnum[1], + hasMatch: true, + fieldMap: 'otherRevenueShower', + fieldName: 'Frontcountry Camping - Other - Gross shower revenue' + }, + 'Net Shower Revenue': { + prop: 'legacy_dayUseMiscShowerNetRevenue', + type: Number, + activity: activitiesEnum[1], + hasMatch: false, + fieldMap: 'legacy_dayUseMiscShowerNetRevenue', + fieldName: 'Frontcountry Camping - Other - Net shower revenue' + }, + 'Gross Electrification Revenue': { + prop: 'legacy_dayUseMiscElectricalGrossRevenue', + type: Number, + activity: activitiesEnum[1], + hasMatch: true, + fieldMap: 'otherRevenueElectrical', + fieldName: 'Frontcountry Camping - Other - Gross electrical fee revenue' + }, + // Total for multiple activities + 'Total Camping Attendance (People)': { + prop: 'legacy_totalCampingAttendancePeople', + type: Number, + activity: activitiesEnum[8], + hasMatch: false, + fieldMap: 'legacy_totalCampingAttendancePeople', + fieldName: 'Total Camping Attendance (People)' + }, + // Total for multiple activities + 'Total Gross Camping Revenue': { + prop: 'legacy_totalCampingGrossRevenue', + type: Number, + activity: activitiesEnum[8], + hasMatch: false, + fieldMap: 'legacy_totalCampingGrossRevenue', + fieldName: 'Total Camping Gross Revenue' + }, + // Total for multiple activities + 'Total Net Camping Revenue': { + prop: 'legacy_totalCampingNetRevenue', + type: Number, + activity: activitiesEnum[8], + hasMatch: false, + fieldMap: 'legacy_totalCampingNetRevenue', + fieldName: 'Total Camping Net Revenue' + }, + 'Day Use Vehicles': { + prop: 'legacy_dayUseVehicles', + type: Number, + activity: activitiesEnum[6], + hasMatch: true, + fieldMap: 'peopleAndVehiclesVehicle', + fieldName: 'Day Use - People and Vehicles - Vehicle count' + }, + 'Day Use Busses': { + prop: 'legacy_dayUseBuses', + type: Number, + activity: activitiesEnum[6], + hasMatch: true, + fieldMap: 'peopleAndVehiclesBus', + fieldName: 'Day Use - People and Vehicles - Bus count' + }, + 'Day Use People': { + prop: 'legacy_dayUseAttendancePeople', + type: Number, + activity: activitiesEnum[6], + hasMatch: true, + fieldMap: 'peopleAndVehiclesTrail', + fieldName: 'Day Use - People and Vehicles - Trail count' + }, + 'Total Day Use People': { + prop: 'legacy_dayUseTotalPeopleAndVehiclesAttendancePeople', + type: Number, + activity: activitiesEnum[6], + hasMatch: false, + fieldMap: 'legacy_dayUseTotalPeopleAndVehiclesAttendancePeople', + fieldName: 'Day Use - People and Vehicles - Total attendance (People)' + }, + 'Gross Misc Day Use Revenue': { + prop: 'legacy_dayUseMiscGrossRevenue', + type: Number, + activity: activitiesEnum[6], + hasMatch: false, + fieldMap: 'legacy_dayUseMiscGrossRevenue', + fieldName: null + }, + 'Net Misc Day Use Revenue': { + prop: 'legacy_dayUseMiscNetRevenue', + type: Number, + activity: activitiesEnum[6], + hasMatch: false, + fieldMap: 'legacy_dayUseMiscNetRevenue', + fieldName: null + }, + 'Day Use FC Variance Note': { + prop: 'legacy_dayUseVarianceNote', + type: String, + activity: activitiesEnum[6], + hasMatch: true, + fieldMap: 'notes', + fieldName: 'Day Use - Variance Notes' + }, + 'Total Picnic Shelter People': { + prop: 'legacy_dayUsePicnicShelterAttendancePeople', + type: Number, + activity: activitiesEnum[6], + hasMatch: true, + fieldMap: 'picnicShelterPeople', + fieldName: 'Day Use - Picnic Shelters - Picnic shelter people' + }, + // (fieldMap) This variable is a quantity, not a revenue + 'Picnic Shelter Rentals': { + prop: 'legacy_dayUsePicnicShelterRentals', + type: Number, + activity: activitiesEnum[6], + hasMatch: true, + fieldMap: 'picnicRevenueShelter', + fieldName: 'Day Use - Picnic Shelters - Picnic shelter rentals' + }, + 'Net Picnic Shelter Revenue': { + prop: 'legacy_dayUsePicnicShelterNetRevenue', + type: Number, + activity: activitiesEnum[6], + hasMatch: true, + fieldMap: 'picnicRevenueGross', + fieldName: 'Day Use - Picnic Shelters - Net revenue' + }, + 'Gross Picnic Shelter Revenue': { + prop: 'legacy_dayUsePicnicShelterGrossRevenue', + type: Number, + activity: activitiesEnum[6], + hasMatch: false, + fieldMap: 'picnicRevenueGross', + fieldName: 'Day Use - Picnic Shelters - Gross picnic revenue' + }, + 'Picnic Shelter Variance Note': { + prop: 'legacy_dayUsePicnicShelterVarianceNote', + type: String, + activity: activitiesEnum[6], + hasMatch: false, + fieldMap: 'legacy_dayUsePicnicShelterVarianceNote', + fieldName: null + }, + 'Total Net Day Use Revenue': { + prop: 'legacy_dayUseTotalNetRevenue', + type: Number, + activity: activitiesEnum[6], + hasMatch: false, + fieldMap: 'legacy_dayUseTotalNetRevenue', + fieldName: 'Day Use Net Revenue' + }, + 'Total Gross Day Use Revenue': { + prop: 'legacy_dayUseTotalGrossRevenue', + type: Number, + activity: activitiesEnum[6], + hasMatch: false, + fieldMap: 'legacy_dayUseTotalGrossRevenue', + fieldName: 'Day Use Gross Revenue' + }, + 'Total Day Use attendance (people)': { + prop: 'legacy_dayUseTotalAttendancePeople', + type: Number, + activity: activitiesEnum[6], + hasMatch: false, + fieldMap: 'legacy_dayUseTotalAttendancePeople', + fieldName: 'Day Use Total Attendance (People)' + }, + '# Misc Boats': { + prop: 'legacy_boatingMiscBoatAttendance', + type: Number, + activity: activitiesEnum[7], + hasMatch: true, + fieldMap: 'boatAttendanceMiscellaneous', + fieldName: 'Boating - Boats - Miscellaneous boats' + }, + // (fieldMap) Note typo in 'buoys' + '# Boats on Buoys': { + prop: 'legacy_boatingBoatsOnBuoy', + type: Number, + activity: activitiesEnum[7], + hasMatch: true, + fieldMap: 'boatAttendanceNightsOnBouys', + fieldName: 'Boating - Boats - Nights on buoys' + }, + '# Boats on Docks': { + prop: 'legacy_boatingBoatsOnDock', + type: Number, + activity: activitiesEnum[7], + hasMatch: true, + fieldMap: 'boatAttendanceNightsOnDock', + fieldName: 'Boating - Boats - Nights on dock' + }, + 'Total Boating People': { + prop: 'legacy_boatingTotalAttendancePeople', + type: Number, + activity: activitiesEnum[7], + hasMatch: false, + fieldMap: 'legacy_boatingTotalAttendancePeople', + fieldName: 'Boating - Boats - Boat attendance' + }, + 'Total Net Boating Revenue': { + prop: 'legacy_boatingTotalNetRevenue', + type: Number, + activity: activitiesEnum[7], + hasMatch: false, + fieldMap: 'legacy_boatingTotalNetRevenue', + fieldName: 'Boating - Boats - Net revenue' + }, + 'Total Gross Boating Revenue': { + prop: 'legacy_boatingTotalGrossRevenue', + type: Number, + activity: activitiesEnum[7], + hasMatch: true, + fieldMap: 'boatRevenueGross', + fieldName: 'Boating - Boats - Gross boating revenue' + }, + 'Boating Variance Note': { + prop: 'legacy_boatingVarianceNote', + type: String, + activity: activitiesEnum[7], + hasMatch: true, + fieldMap: 'notes', + fieldName: 'Boating - Variance Notes' + }, + // No parent activity + 'Additional Note': { + prop: 'legacy_additionalNote', + type: String, + activity: activitiesEnum[8], + hasMatch: false, + fieldMap: 'legacy_additionalNote', + fieldName: null + }, + // No parent activity + 'TOTAL NET REVENUE': { + prop: 'legacy_totalsTotalNetRevenue', + type: Number, + activity: activitiesEnum[8], + hasMatch: false, + fieldMap: 'legacy_totalsTotalNetRevenue', + fieldName: 'Total Net Revenue' + }, + // No parent activity + 'TOTAL ATTENDANCE (PERSONS)': { + prop: 'legacy_totalsTotalAttendancePeople', + type: Number, + activity: activitiesEnum[8], + hasMatch: false, + fieldMap: 'legacy_totalsTotalAttendancePeople', + fieldName: 'Total Attendance' + }, + // No parent activity + 'Data Source': { + prop: 'legacy_dataSource', + type: String, + activity: activitiesEnum[8], + hasMatch: false, + fieldMap: 'legacy_dataSource', + fieldName: null + } +}; + +module.exports = { + activitiesEnum, + schema +}; diff --git a/arSam/migrations/migrate-historical/legacy-data-functions.js b/arSam/migrations/migrate-historical/legacy-data-functions.js new file mode 100644 index 0000000..301f833 --- /dev/null +++ b/arSam/migrations/migrate-historical/legacy-data-functions.js @@ -0,0 +1,297 @@ +const fs = require('fs'); +const readline = require('readline'); +const { getParks, getSubAreas } = require('../../layers/baseLayer/baseLayer'); +const { activitiesEnum } = require('./legacy-data-constants'); +const axios = require('axios'); +const jwt = require('jsonwebtoken'); + +let recordFieldsByActivity = {}; + +const clientIDsAR = { + dev: 'e530debc-b4e0-417a-947d-2907d70404da', + test: '2246a87f-96b7-4907-ba54-6202339560a1', + prod: '4dc679f8-c726-4e65-afb9-0edf664b93e0' +}; + +function formatTime(time) { + let sec = parseInt(time / 1000, 10); + let hours = Math.floor(sec / 3600); + let minutes = Math.floor((sec - hours * 3600) / 60); + let seconds = sec - hours * 3600 - minutes * 60; + if (hours < 10) { + hours = '0' + hours; + } + if (minutes < 10) { + minutes = '0' + minutes; + } + if (seconds < 10) { + seconds = '0' + seconds; + } + return hours + ':' + minutes + ':' + seconds; +} + +function updateConsoleProgress(intervalStartTime, text, complete = 0, total = 1, modulo = 1) { + if (complete % modulo === 0 || complete === total) { + const currentTime = new Date().getTime(); + let currentElapsed = currentTime - intervalStartTime; + let remainingTime = NaN; + if (complete !== 0) { + let totalTime = (total / complete) * currentElapsed; + remainingTime = totalTime - currentElapsed; + } + const percent = (complete / total) * 100; + process.stdout.write( + ` ${text}: ${complete}/${total} (${percent.toFixed(1)}%) completed in ${formatTime(currentElapsed)} (~${formatTime(remainingTime)} remaining)\r` + ); + } +} + +async function getConsoleInput(query) { + const rl = readline.createInterface({ + input: process.stdin, + output: process.stdout + }); + + return new Promise((resolve) => + rl.question(query, (ans) => { + rl.close(); + resolve(ans); + }) + ); +} + +// The legacy schema is very large - this function is to prevent accidental edits to the schema while building the code. +function validateSchema(schema) { + console.log('Checking schema...'); + try { + // Master sheet is 83 columns wide. + if (Object.keys(schema).length !== 83) { + throw 'Incorrect schema length.'; + } + // Ensure all schema entries have the same structure. + const entryProps = ['prop', 'type', 'activity', 'hasMatch', 'fieldMap', 'fieldName']; + for (const col in schema) { + let entry = schema[col]; + for (const prop of entryProps) { + if (Object.keys(entry).indexOf(prop) === -1) { + throw `${col}: Malformed schema entry - malformed property`; + } + } + } + // Create fieldmap by activities for future use in legacy record generation. + createFieldListByActivity(schema); + } catch (error) { + throw error; + } +} + +async function getDBSnapshot() { + console.log('Loading existing DB snapshot...'); + let parks = await getParks(); + let map = {}; + try { + for (const park of parks) { + let entry = {}; + let subareas = await getSubAreas(park.orcs); + for (const subarea of subareas) { + entry[subarea.sk] = { + subAreaName: subarea.subAreaName, + activities: subarea.activities.values + }; + } + map[park.orcs] = entry; + } + return map; + } catch (error) { + throw `Error getting current DB snapshot ` + error; + } +} + +function determineActivities(row, schema) { + let activityList = []; + for (const entry in schema) { + let prop = row[schema[entry].prop]; + let type = schema[entry].type; + let activity = schema[entry].activity; + // ignore 'Meta' activity + if (activityList.indexOf(activity) === -1 && activity !== 'Meta') { + if (type === String && prop && prop !== '' && prop !== '0') { + // valid string field + activityList.push(activity); + } else if (type === Number && prop && prop !== 0) { + // valid numerical field + activityList.push(activity); + } + } + } + return activityList; +} + +function createCSV(header, data, filename) { + try { + let csv = header.join(',') + '\n'; + for (const item of data) { + let row = []; + for (const field of header) { + let str = String(item[field]).replaceAll(',', '_'); + row.push(str || ''); + } + csv += row.join(',') + '\n'; + } + fs.writeFileSync(`${filename}.csv`, csv); + console.log(`${filename} created.`); + } catch (error) { + console.log(`Failed to create csv: ${filename}: ${error}`); + } +} + +// Quickly generate CSV of current schema to forward for client validation +function createSchemaCSV(schema) { + try { + validateSchema(schema); + } catch (error) { + console.log('error:', error); + return; + } + const fields = ['Legacy Name', 'fieldName', 'activity', 'type', 'calculated', 'hasMatch', 'fieldMap']; + const replacer = function(key, value) { + return value === null ? '' : value; + }; + let csv = Object.keys(schema).map(function(entry) { + let name = JSON.stringify(entry); + let str = fields + .map(function(field) { + if (field === 'calculated') { + if (!schema[entry].hasMatch && schema[entry].fieldName !== null) { + // field is calculated + return JSON.stringify(true, replacer); + } + } + if (field === 'type') { + return JSON.stringify(typeof schema[entry][field](), replacer); + } + return JSON.stringify(schema[entry][field], replacer); + }) + .join(','); + return name.concat(str); + }); + csv.unshift(fields.join(',')); // add header column + csv = csv.join('\r\n'); + // console.log("csv:", csv); + fs.writeFileSync(`ar-fieldmap_${new Date().toISOString()}.csv`, csv); +} + +function createFieldListByActivity(schema) { + for (const activity of Object.keys(activitiesEnum)) { + let list = []; + for (const col of Object.keys(schema)) { + if (schema[col].activity === activitiesEnum[activity]) { + list.push({ key: schema[col].fieldMap, value: schema[col].prop, hasMatch: schema[col].hasMatch }); + } + } + recordFieldsByActivity[activitiesEnum[activity]] = list; + } +} + +function createLegacyParkObject(data) { + try { + let obj = { + pk: 'park', + sk: data.legacy_orcs, + parkName: data.legacy_park, + subAreas: [], + orcs: data.legacy_orcs, + roles: ['sysadmin', data.legacy_orcs], + isLegacy: true + }; + return obj; + } catch (error) { + throw `Failed to create legacy park object (${data.legacy_orcs}): ` + error; + } +} + +function createLegacySubAreaObject(data, id, subAreaName, activities) { + try { + let obj = { + pk: `park::${data.legacy_orcs}`, + sk: String(id).padStart(4, '0'), + parkName: data.legacy_park, + subAreaName: subAreaName, + activities: activities || [], + orcs: data.legacy_orcs, + roles: ['sysadmin', `${data.legacy_orcs}::${id}`], + managementArea: '', + section: data.legacy_section, + region: data.legacy_region, + bundle: data.legacy_bundle, + isLegacy: true + }; + return obj; + } catch (error) { + throw ( + `Failed to create legacy subarea object ${String(id).padStart(4, '0')} (${data?.legacy_parkSubArea}): ` + error + ); + } +} + +function createLegacyRecordObject(data, activity, subAreaId) { + try { + let obj = createLegacyBaseRecord(data, activity, subAreaId); + obj = { ...obj, ...createLegacyActivityRecord(data, activity) }; + return obj; + } catch (error) { + throw `Failed to create legacy ${activity} record for ${data?.legacy_parkSubArea}, ${data?.legacy_month} ${data.legacy_year}: ${error}`; + } +} + +function createLegacyBaseRecord(data, activity, subAreaId) { + // create date + const month = new Date(Date.parse(data.legacy_month + ' 1, ' + data.legacy_year)).getMonth() + 1; + const date = `${data.legacy_year}${String(month).padStart(2, '0')}`; + return { + pk: `${subAreaId}::${activity}`, + sk: date, + date: date, + activity: activity, + orcs: data.legacy_orcs, + parkName: data.legacy_park, + subAreaId: subAreaId, + lastUpdated: new Date().toISOString(), + isLegacy: true, + isLocked: true + }; +} + +function createLegacyActivityRecord(data, activity) { + let properties = [...recordFieldsByActivity[activity]]; + let recordObj = {}; + let legacyObj = {}; + for (const prop of properties) { + if (data[prop.value] || data[prop.value] === 0) { + if (prop.hasMatch) { + recordObj[prop.key] = data[prop.value]; + } else { + legacyObj[prop.key] = data[prop.value]; + } + } + } + if (Object.keys(legacyObj).length) { + recordObj.legacyData = legacyObj; + } + return recordObj; +} + +module.exports = { + updateConsoleProgress, + createFieldListByActivity, + validateSchema, + determineActivities, + createSchemaCSV, + createLegacySubAreaObject, + createLegacyParkObject, + getConsoleInput, + createLegacyRecordObject, + getDBSnapshot, + createCSV, + clientIDsAR +};