From 3586b579118cc7ce107ad4b84a84567e60697e7c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mark=20Lis=C3=A9?= Date: Wed, 23 Oct 2024 15:11:12 -0400 Subject: [PATCH 1/2] Create CODEOWNERS --- .github/CODEOWNERS | 4 ++++ 1 file changed, 4 insertions(+) create mode 100644 .github/CODEOWNERS diff --git a/.github/CODEOWNERS b/.github/CODEOWNERS new file mode 100644 index 0000000..c7a5721 --- /dev/null +++ b/.github/CODEOWNERS @@ -0,0 +1,4 @@ +# These owners will be the default owners for everything in +# the repo. Unless a later match takes precedence, +# they will be requested for review when someone opens a pull request. +* @marklise @danieltruong @cameronpettit @meredithom @Christopher-walsh22 @davidclaveau From a0df045946afe97bc9251766abe40d9ed8dad6d5 Mon Sep 17 00:00:00 2001 From: Dave <62899351+davidclaveau@users.noreply.github.com> Date: Wed, 13 Nov 2024 09:23:41 -0800 Subject: [PATCH 2/2] BRS-377: Fix Missing Export for Null Values (#383) * fix missing export for null values, 374 bug fix * ar user request to remove blank rows Signed-off-by: David --------- Signed-off-by: David --- .../export-missing/invokable/index.js | 412 ++++++++++-------- 1 file changed, 238 insertions(+), 174 deletions(-) diff --git a/arSam/handlers/export-missing/invokable/index.js b/arSam/handlers/export-missing/invokable/index.js index b528c38..d5105ab 100644 --- a/arSam/handlers/export-missing/invokable/index.js +++ b/arSam/handlers/export-missing/invokable/index.js @@ -5,23 +5,21 @@ const { VARIANCE_STATE_DICTIONARY, EXPORT_VARIANCE_CONFIG, MISSING_CSV_HEADERS, - EXPORT_MONTHS, + EXPORT_MONTHS } = require('/opt/constantsLayer'); const { getParks, TABLE_NAME, dynamoClient, PutItemCommand, - getOne, marshall, s3Client, PutObjectCommand, flattenConfig, runQuery, logger, + getSubAreas } = require('/opt/baseLayer'); -const { pbkdf2 } = require('crypto'); -const { info } = require('console'); const FILE_PATH = process.env.FILE_PATH || '/tmp/'; const FILE_NAME = process.env.FILE_NAME || 'A&R_Missing_Report'; @@ -63,21 +61,13 @@ exports.handler = async (event, context) => { logger.info(`=== Exporting filtered data ===`); // collect missing records, use VARIANCE_STATE as it's the same - const records = await getMissingRecords(fiscalYearEnd, roles, orcs); - - if (!records.length) { - await updateJobWithState(VARIANCE_STATE_DICTIONARY.NODATA); - return - } - + const missingRecords = await getMissingRecords(fiscalYearEnd, roles, orcs); await updateJobWithState(VARIANCE_STATE_DICTIONARY.FORMATTING); // format records for csv - formatRecords(records); - await updateJobWithState(VARIANCE_STATE_DICTIONARY.GENERATING); // create csv - const csv = await createCSV(records, fiscalYearEnd); + const csv = await createCSV(missingRecords, fiscalYearEnd); await updateJobWithState(VARIANCE_STATE_DICTIONARY.UPLOADING); // upload csv to S3 @@ -87,7 +77,7 @@ exports.handler = async (event, context) => { // success! LAST_SUCCESSFUL_JOB = { key: S3_KEY, - dateGenerated: new Date().toISOString(), + dateGenerated: new Date().toISOString() }; await updateJobWithState(VARIANCE_STATE_DICTIONARY.UPLOADING, 95); await updateJobWithState(VARIANCE_STATE_DICTIONARY.COMPLETE); @@ -107,7 +97,7 @@ async function updateJobWithState(state, percentageOverride = null) { state = 'error'; message = 'Job failed. Exporter encountered an error.'; break; - + // no data, no report case 0: state = 'no_data'; @@ -153,7 +143,7 @@ async function updateJobWithState(state, percentageOverride = null) { lastSuccessfulJob: LAST_SUCCESSFUL_JOB, key: S3_KEY, params: PARAMS, - dateGenerated: new Date().toISOString(), + dateGenerated: new Date().toISOString() }; try { await updateJobEntry(jobObj); @@ -166,7 +156,7 @@ async function updateJobWithState(state, percentageOverride = null) { async function updateJobEntry(jobObj) { const putObj = { TableName: TABLE_NAME, - Item: marshall(jobObj), + Item: marshall(jobObj) }; await dynamoClient.send(new PutItemCommand(putObj)); } @@ -176,6 +166,7 @@ async function getMissingRecords(fiscalYearEnd, roles, singleOrcs = undefined) { let orcsList = []; let saidList = []; + if (isAdmin) { // must check all parks. const parks = await getParks(); @@ -200,20 +191,10 @@ async function getMissingRecords(fiscalYearEnd, roles, singleOrcs = undefined) { } // determine months in fiscal year - const dates = []; - for (let i = 1; i <= 12; i++) { - let year = fiscalYearEnd; - if (i > 3) { - year -= 1; - } - dates.push(year + String(i).padStart(2, '0')); - } + const dates = getMonthsInFiscal(fiscalYearEnd); - // get all missing records - const missingQueryObj = { - TableName: TABLE_NAME, - ExpressionAttributeValues: {}, - }; + // All activities we're looping through + const activityList = Object.keys(EXPORT_VARIANCE_CONFIG); let missingRecords = []; @@ -222,35 +203,61 @@ async function getMissingRecords(fiscalYearEnd, roles, singleOrcs = undefined) { for (const orcs of orcsList) { updateHighAccuracyJobState(1, orcsList.indexOf(orcs), orcsList.length, 70); - // cycle through months - for (const date of dates) { - // add to query - const missingQueryObj = { + // Need to get the subareas from the orc + if (isAdmin) { + saidList = await getSubAreas(orcs); + } + + // cycle through subareas + for (let said of saidList) { + // saidList gets the sk from the kc role, but if sysadmin we need to dig + // into the object a bit to get the subarea id + if (isAdmin) { + said = said['sk']; + } + + // We have to get the bundle and this is probably the best time to do so. + // Also get the subAreaName for ease later + let bundle; + let subAreaName; + const subArea = await runQuery({ TableName: TABLE_NAME, ExpressionAttributeValues: { - ':pk': { S: `variance::${orcs}::${date}` }, + ':pk': { S: `park::${orcs}` }, + ':sk': { S: `${said}` } }, - KeyConditionExpression: 'pk = :pk', - }; - - // get records - let records = await runQuery(missingQueryObj); - - // unless user is admin, filter out subareas they don't have access to - if (!isAdmin) { - records = records.filter((record) => { - const said = record?.sk?.split('::')[0]; - return saidList.includes(said); - }); - } - - // Loop through the variance fields - if (records.length > 0) { - for (const record of records) { - // Looking for fields that have at least one field -1 - if (record.fields && record.fields.length > 0 && hasMissingField(record.fields)) { - missingRecords.push(record); + KeyConditionExpression: 'pk = :pk AND sk = :sk' + }); + + // cycle through the activities + for (const activity of activityList) { + // Get all the subarea records + const subAreaQueryObj = { + TableName: TABLE_NAME, + ExpressionAttributeValues: { + ':pk': { S: `${said}::${activity}` } + }, + KeyConditionExpression: 'pk = :pk' + }; + + // Get all the records for the subarea and activity + const records = await runQuery(subAreaQueryObj); + + // This is where we add the bundle to the records + if (records && records.length > 0) { + // Append the bundle here + for (let record of records) { + record.bundle = subArea[0].bundle; } + + // We format the records so they're easier to parse later, making them + // an object that's nested as { bundle: { park: { date: {...}}}} + const formattedRecords = formatRecords(records); + + // We then parse through the object to find where/when items are + // missing data. We pass in activity and subAreaName for records that + // are missing any information, as we need to build "no data" records + missingRecords.push(findMissingRecords(formattedRecords, dates, activity, subArea[0].subAreaName)); } } } @@ -262,84 +269,130 @@ async function getMissingRecords(fiscalYearEnd, roles, singleOrcs = undefined) { } } -function updateHighAccuracyJobState(state, index, total, size) { - if (index % JOB_UPDATE_MODULO === 0) { - const increment = (JOB_UPDATE_MODULO * size) / total; - const percentage = Math.floor(CURRENT_PROGRESS_PERCENT + increment); - updateJobWithState(state, percentage); +function findMissingRecords(records, fiscalYearDates, activity, subAreaName) { + const missingRecords = {}; + + // Get all keys + const bundles = Object.keys(records); + let parks; + + for (const bundle of bundles) { + parks = Object.keys(records?.[bundle] ?? {}); + + for (const park of parks) { + for (const date of fiscalYearDates) { + let recordCheck = {}; + + if (!records[bundle][park]?.[date]) { + // We also need to add any "no data" items for dates that are missing up to + // today's date. These will be omitted if the previous year's months are missing + // data anyway (i.e. December never has data, won't trigger as missing later). + records[bundle][park][date] = { + bundle, + parkName: park, + date, + subAreaName + }; + } + + recordCheck = records[bundle][park][date]; + + // If we have a match for this bundle, park, and date, get the activities for it + const requiredFields = EXPORT_VARIANCE_CONFIG[activity]; + // Check if the activity's fields have any values in the current record + const missingFields = Object.keys(requiredFields).filter( + (field) => !Object.prototype.hasOwnProperty.call(recordCheck, field) + ); + // Now that we know what fields are missing, check the previous years + for (const missingField of missingFields) { + const prevYearsData = checkPreviousYears(records, bundle, park, date, missingField); + if (prevYearsData.length > 0) { + missingRecords[bundle] = missingRecords[bundle] || {}; + missingRecords[bundle][park] = missingRecords[bundle][park] || {}; + // Add the current year + missingRecords[bundle][park][date] = records[bundle][park][date]; + // Add the records of yesteryear + for (const record of prevYearsData) { + missingRecords[record.bundle][record.parkName][record.date] = record; + } + } + } + } + } } -} -// Return as soon as we find percentageChange of -1, i.e. 'Missing' -function hasMissingField(fields) { - return fields.some((field) => field.percentageChange === -1); + return missingRecords; } -// Make sure that we're matching the VARIANCE config -function formatRecords(records) { - for (const record of records) { - if (record.fields.length > 0) { - for (const field of record.fields) { - // We want to filter any records that have -100% variance - const flattenedConfig = flattenConfig(EXPORT_VARIANCE_CONFIG); - if (flattenedConfig.includes(field.key) && field.percentageChange == -1) { - record[field.key] = 'Data Missing'; - } +function checkPreviousYears(record, bundle, park, date, missingField) { + let missingRecords = []; + let missingFound = false; + let year = parseInt(date.slice(0, 4), 10); // 2024 + let month = date.slice(4); + + let prevYear = year - 1; + // Get the previous three years beyond the current year + while (prevYear >= year - 3) { + const prevDate = `${prevYear}${month}`; + // Check if the date exists + if (record[bundle][park][prevDate]) { + // Check if the activity that's missing this year exists in the prev year + if (record[bundle][park][prevDate][missingField]) { + missingFound = true; } - } - // We'll use month for grouping when creating CSV - const date = record.pk.split('::')[2]; - record['month'] = convertMonth(parseInt(date.slice(4))); - if (/\r\n|\n|\r/.test(record.notes)) { - record.notes = record.notes.replace(/(\r\n|\n|\r)/g, ' '); + // We push every record regardless, so if one is missing we have all prev + // years to share in the report + missingRecords.push(record[bundle][park][prevDate]); + } else { + // No data exists for that year/month at all, so we add a "no data" item + const noData = { + bundle: [bundle], + parkName: [park], + date: [prevDate] + }; + missingRecords.push(noData); } + + prevYear--; + } + + // Only send back the previous records if at least one is missing + if (!missingFound) { + missingRecords = []; } + + return missingRecords; } -async function getPreviousYearData(years, subAreaId, activity, date) { - logger.info('Getting previous year data', years, subAreaId, activity, date); - // Get records for up to the past N years, limited to no farther than January 2022 - let currentDate = DateTime.fromFormat(date, 'yyyyMM'); - const targetYear = 202201; - let records = []; - - // Go back 3 years until no more than 2022 - for (let i = 1; i <= years; i++) { - let selectedYear = currentDate.minus({ years: i }).toFormat('yyyyMM'); - if (selectedYear >= targetYear) { - logger.info(`Selected year: ${selectedYear}`); - try { - const data = await getOne(`${subAreaId}::${activity}`, selectedYear); - logger.info('Read Activity Record Returning.'); - logger.debug('DATA:', data); - if (Object.keys(data).length !== 0) { - records.push(data); - } - } catch (err) { - // Skip on errors - logger.error(err); - } - } +function updateHighAccuracyJobState(state, index, total, size) { + if (index % JOB_UPDATE_MODULO === 0) { + const increment = (JOB_UPDATE_MODULO * size) / total; + const percentage = Math.floor(CURRENT_PROGRESS_PERCENT + increment); + updateJobWithState(state, percentage); + } +} - // Want to have three items (even if there's no data) when we send this back - if (records.length < 3) { - const itemsToAdd = 3 - records.length; - for (let i = 0; i < itemsToAdd; i++) { - records.push({}); - } +function getMonthsInFiscal(fiscalYearEnd) { + const dates = []; + for (let i = 1; i <= 12; i++) { + let year = fiscalYearEnd; + if (i > 3) { + year -= 1; } + dates.push(year + String(i).padStart(2, '0')); } - - return records; + return dates; } -async function createCSV(records, year) { - const startYear = Number(year); +function createCSV(missingRecords, fiscalYearEnd) { + const dates = getMonthsInFiscal(fiscalYearEnd); + const todayDate = DateTime.now().toFormat('yyyyLL'); + const startYear = Number(fiscalYearEnd); const yearRanges = generateYearRanges(startYear); const { missingHeadersRow, subHeadersRow } = constructHeaderRows(MISSING_CSV_HEADERS, yearRanges, [ - `${startYear - 1}-${startYear} Missing?`, - 'Notes', + 'Missing', + 'Notes' ]); // Add space before the date ranges for bundle, park, subarea, and months @@ -347,12 +400,70 @@ async function createCSV(records, year) { let content = [missingHeadersRow, subHeadersRow]; - // Get an object where the records are sorted by bundle, then park, and then by month - // { bundle: { park: { month: [records] }, { next month: [records] } }, ... } - const recordsGroupedBundleParkMonth = records.reduce((groupedRecords, record) => { + for (const missingRecord of missingRecords) { + for (const bundle of Object.keys(missingRecord)) { + for (const park of Object.keys(missingRecord[bundle])) { + let subAreaName; + for (const date of dates) { + if (date < todayDate && missingRecord[bundle][park][date]) { + // We're going to add the bundle, park, subarea, and months as they appear + subAreaName = missingRecord[bundle][park][date]?.['subAreaName'] || subAreaName; + let subAreaRow = []; + const year = parseInt(date.slice(0, 4), 10); // 2024 + const month = date.slice(4); + + for (const item of flattenConfig(EXPORT_VARIANCE_CONFIG)) { + subAreaRow.push(missingRecord[bundle][park][`${year - 3}${month}`][item] || ''); // Three years ago + subAreaRow.push(missingRecord[bundle][park][`${year - 2}${month}`][item] || ''); // Two years ago + subAreaRow.push(missingRecord[bundle][park][`${year - 1}${month}`][item] || ''); // One year ago + subAreaRow.push(missingRecord[bundle][park][date][item] || ''); // Current year + + if ( + !missingRecord[bundle][park][date][item] && + ((missingRecord[bundle][park][`${year - 1}${month}`][item] && + missingRecord[bundle][park][`${year - 1}${month}`][item] !== 0) || + (missingRecord[bundle][park][`${year - 2}${month}`][item] && + missingRecord[bundle][park][`${year - 2}${month}`][item] !== 0) || + (missingRecord[bundle][park][`${year - 3}${month}`][item] && + missingRecord[bundle][park][`${year - 3}${month}`][item] !== 0)) + ) { + subAreaRow.push('Missing Data'); + } else { + // Not missing data, add empty space to that column + subAreaRow.push(''); + } + subAreaRow.push(missingRecord[bundle][park].notes || ''); // Notes + } + + // Add these to the start of the array because we might not have subAreaName early on + // Throw these into this strange format: `"${variable}"` as the double quotes help + // with any rogue special names/characters we don't want to affect the csv output + subAreaRow.unshift(`"${bundle}"`, `"${park}"`, `"${subAreaName}"`, convertMonth(date.slice(4))); + + content.push(subAreaRow); + } + } + } + } + } + + let csvData = ''; + for (const row of content) { + csvData += row.join(',') + '\r\n'; + } + return csvData; +} + +// We format the records so they're easier to parse later. +// The records are organized from an array of records into +// an object of { bundle: { parkName: { year: { month: [records] }}} +function formatRecords(records) { + const groupedRecords = {}; + + for (const record of records) { const bundle = record.bundle; const park = record.parkName; - const month = record.month; + const date = record.date; if (!groupedRecords[bundle]) { groupedRecords[bundle] = {}; @@ -362,61 +473,16 @@ async function createCSV(records, year) { groupedRecords[bundle][park] = {}; } - if (!groupedRecords[bundle][park][month]) { - groupedRecords[bundle][park][month] = []; - } - - groupedRecords[bundle][park][month].push(record); - return groupedRecords; - }, {}); - - // We go through each bundle, then each park, and then each month for that park - // and then each row is the subareas and the activity data for that subarea - for (const bundle of Object.keys(recordsGroupedBundleParkMonth)) { - for (const park of Object.keys(recordsGroupedBundleParkMonth[bundle])) { - for (const month of Object.keys(recordsGroupedBundleParkMonth[bundle][park])) { - // Start looping for each month in the park - for (const record of recordsGroupedBundleParkMonth[bundle][park][month]) { - const startDate = record.pk.split('::')[2]; - const subAreaId = record.subAreaId; - const activity = record.sk.split('::')[1]; - - const previousRecords = await getPreviousYearData(3, subAreaId, activity, startDate); - const currentRecord = await getOne(`${subAreaId}::${activity}`, startDate); - - // Row starts with bundle, park, subarea, and month - let subAreaRow = [bundle, park, record.subAreaName, month]; - - // For each matching item/activity, push the data that's found - for (const item of flattenConfig(EXPORT_VARIANCE_CONFIG)) { - subAreaRow.push(previousRecords[2][item] || ''); // e.g. 2020-2021 - subAreaRow.push(previousRecords[1][item] || ''); // e.g. 2021-2022 - subAreaRow.push(previousRecords[0][item] || ''); // e.g. 2022-2023 - subAreaRow.push(currentRecord[item]); // e.g. 2023-2024 - subAreaRow.push(record[item] || ''); // Variance heading (Missing) - subAreaRow.push(currentRecord['notes']?.replace(/,/g, '') || ''); - } - - content.push(subAreaRow); - } - } - - content.push(['']); // new row for readability between parks + if (!groupedRecords[bundle][park][date]) { + groupedRecords[bundle][park][date] = {}; } - content.push(['']); // new row for readability between bundles + groupedRecords[bundle][park][date] = { ...record }; } - let csvData = ''; - for (const row of content) { - csvData += row.join(',') + '\r\n'; - } - return csvData; + return groupedRecords; } -// Used for the main headers. -// We have four years, Variance and Notes columns for each activity, so we add empty -// strings/cells to account for the additional columns beneath the activity header function constructHeaderRows(MISSING_CSV_HEADERS, yearRanges, staticSubHeaders) { let missingHeadersRow = ['Bundle, Park, Subarea, Month']; let subHeadersRow = []; @@ -431,7 +497,6 @@ function constructHeaderRows(MISSING_CSV_HEADERS, yearRanges, staticSubHeaders) return { missingHeadersRow, subHeadersRow }; } -// Used for the subheaders, creates the "2021-2022, 2022-2023, ..." headings function generateYearRanges(startYear) { let ranges = []; for (let i = 4; i > 0; i--) { @@ -451,7 +516,7 @@ async function uploadToS3(csvData) { const params = { Bucket: process.env.S3_BUCKET_DATA, Key: S3_KEY, - Body: buffer, + Body: buffer }; const command = new PutObjectCommand(params); @@ -472,7 +537,7 @@ function convertMonth(monthNumber) { 'September', 'October', 'November', - 'December', + 'December' ]; if (monthNumber >= 1 && monthNumber <= 12) { @@ -481,4 +546,3 @@ function convertMonth(monthNumber) { return 'Invalid month number'; } } -