From 27901f73ed9c0af4b4eb69e3938a634d05e85f46 Mon Sep 17 00:00:00 2001 From: Cameron Pettit <71421099+cameronpettit@users.noreply.github.com> Date: Mon, 17 Jan 2022 10:31:59 -0800 Subject: [PATCH] BRS-196 - case insensitive search on first and last names (#67) Changing IS_OFFLINE to boolean updating README --- lambda/readPass/index.js | 16 +-- lambda/writePass/index.js | 2 + migrations/caseInsensitiveNames.js | 217 +++++++++++++++++++++++++++++ 3 files changed, 227 insertions(+), 8 deletions(-) create mode 100644 migrations/caseInsensitiveNames.js diff --git a/lambda/readPass/index.js b/lambda/readPass/index.js index ad2cfa1d..185b4e49 100644 --- a/lambda/readPass/index.js +++ b/lambda/readPass/index.js @@ -68,19 +68,19 @@ exports.handler = async (event, context) => { // Filter first/last if (event.queryStringParameters.firstName) { queryObj = checkAddExpressionAttributeNames(queryObj); - queryObj.ExpressionAttributeNames['#firstName'] = 'firstName'; - queryObj.ExpressionAttributeValues[':firstName'] = AWS.DynamoDB.Converter.input( - event.queryStringParameters.firstName + queryObj.ExpressionAttributeNames['#searchFirstName'] = 'searchFirstName'; + queryObj.ExpressionAttributeValues[':searchFirstName'] = AWS.DynamoDB.Converter.input( + event.queryStringParameters.firstName.toLowerCase() ); - queryObj.FilterExpression += ' AND #firstName =:firstName'; + queryObj.FilterExpression += ' AND #searchFirstName =:searchFirstName'; } if (event.queryStringParameters.lastName) { queryObj = checkAddExpressionAttributeNames(queryObj); - queryObj.ExpressionAttributeNames['#lastName'] = 'lastName'; - queryObj.ExpressionAttributeValues[':lastName'] = AWS.DynamoDB.Converter.input( - event.queryStringParameters.lastName + queryObj.ExpressionAttributeNames['#searchLastName'] = 'searchLastName'; + queryObj.ExpressionAttributeValues[':searchLastName'] = AWS.DynamoDB.Converter.input( + event.queryStringParameters.lastName.toLowerCase() ); - queryObj.FilterExpression += ' AND #lastName =:lastName'; + queryObj.FilterExpression += ' AND #searchLastName =:searchLastName'; } // Filter email if (event.queryStringParameters.email) { diff --git a/lambda/writePass/index.js b/lambda/writePass/index.js index 2a846e2c..13c55d34 100644 --- a/lambda/writePass/index.js +++ b/lambda/writePass/index.js @@ -136,7 +136,9 @@ exports.handler = async (event, context) => { passObject.Item['pk'] = { S: 'pass::' + parkName }; passObject.Item['sk'] = { S: registrationNumber }; passObject.Item['firstName'] = { S: firstName }; + passObject.Item['searchFirstName'] = { S: firstName.toLowerCase() }; passObject.Item['lastName'] = { S: lastName }; + passObject.Item['searchLastName'] = { S: lastName.toLowerCase() }; passObject.Item['facilityName'] = { S: facilityName }; passObject.Item['email'] = { S: email }; passObject.Item['date'] = { S: date }; diff --git a/migrations/caseInsensitiveNames.js b/migrations/caseInsensitiveNames.js new file mode 100644 index 00000000..0688cfbb --- /dev/null +++ b/migrations/caseInsensitiveNames.js @@ -0,0 +1,217 @@ +/* +Case-insensitive names - this migration is to update existing passes with new search fields: searchFirstName & searchLastName. +These fields are copies of the firstName and lastName fields cast to lowercase characters since DynamoDB does not support case-insensitive string searching. +Passes will not update if they do not contain both firstName and lastName fields. + +To run locally: Ensure your dyanmodb-local db is running in a docker instance or similar. + +`export IS_OFFLINE=1` +`node caseInsensitiveNames.js` + +To run in AWS environment: Configure AWS credentials to target AWS env. + +`export IS_OFFLINE=0` +`node caseInsensitiveNames.js` + +To revert changes (removes searchFirstName & searchLastName fields from all passes), add revert argument: + +`node caseInsensitiveNames.js revert` + +To list passes that failed to update or revert, add show-failures argument : + +`node caseInsensitiveNames.js show-failures` +*/ + +const AWS = require('aws-sdk'); + +TABLE_NAME = process.env.TABLE_NAME || 'parksreso'; + +const args = process.argv; +let revert = false; +let showFailures = false; + +if (args.includes('show-failures')){ + showFailures = true; +} + +if (args.includes('revert')){ + revert = true; +} + +const options = { + region: 'ca-central-1' +}; + +if (process.env.IS_OFFLINE) { + options.endpoint = 'http://localhost:8000'; +} + +const dynamodb = new AWS.DynamoDB(options); + +exports.dynamodb = new AWS.DynamoDB(); + +// Scan for all passes +async function getAllPasses() { + const scanObj = { + TableName: TABLE_NAME, + ConsistentRead: true, + ExpressionAttributeNames: { + '#pk': 'pk' + }, + ExpressionAttributeValues: { + ':beginsWith': { + S: 'pass::' + } + }, + FilterExpression: 'begins_with(#pk, :beginsWith)' + }; + + try { + const res = await runScan(scanObj); + return res; + } catch (err) { + console.log('Error getting passes:', err); + return null; + } +} + +async function runScan(scan, paginated = false) { + const data = await dynamodb.scan(scan).promise(); + var unMarshalled = data.Items.map(item => { + return AWS.DynamoDB.Converter.unmarshall(item); + }); + if (paginated) { + return { + LastEvaluatedKey: data.LastEvaluatedKey, + data: unMarshalled + }; + } else { + return unMarshalled; + } +} + +async function runUpdate(update, paginated = false) { + const data = await dynamodb.updateItem(update).promise(); + if (paginated) { + return { + LastEvaluatedKey: data.LastEvaluatedKey, + data: data + }; + } else { + return data; + } +} + +// add searchFirstName and searchLastName fields to all passes +async function updatePasses(passList) { + console.log('collected ' + passList.length + ' passes...'); + let updatedItems = []; + let failedItems = []; + let failureCount = 0; + let skippedCount = 0; + for (let item of passList) { + if (item.firstName && item.lastName && item.pk && item.sk) { + const updateObj = { + TableName: TABLE_NAME, + Key: { + pk: { S: item.pk }, + sk: { S: item.sk } + }, + UpdateExpression: 'set #searchFirstName = :searchFirstName, #searchLastName = :searchLastName', + ExpressionAttributeNames: { + '#searchFirstName': 'searchFirstName', + '#searchLastName': 'searchLastName' + }, + ExpressionAttributeValues: { + ':searchFirstName': {S: item.firstName.toLowerCase()}, + ':searchLastName': {S: item.lastName.toLowerCase()} + }, + ReturnValues: 'ALL_NEW' + }; + try { + const data = await runUpdate(updateObj); + updatedItems.push(data); + } catch (err) { + failureCount++; + console.log('Error updating pass sk: ' + item.sk + '. ' + err); + break; + } + } else { + skippedCount++; + } + } + if (failureCount > 0){ + if(showFailures){ + console.log("Failures:\n", failedItems); + } else { + console.log('run "node caseInsensitiveNames.js showFailures" to list failed items.') + } + } + console.log('--------') + console.log('updated ' + updatedItems.length + ' items.'); + console.log(failureCount + ' failures.' ); + console.log(skippedCount + ' passes were skipped for missing necessary fields.'); +} + +// remove searchFirstName and searchLastName fields from all passes +async function revertPasses(passList) { + console.log('collected ' + passList.length + ' passes...'); + let updatedItems = []; + let failedItems = []; + let failureCount = 0; + let skippedCount = 0; + for (let item of passList) { + if (item.searchFirstName && item.searchLastName && item.pk && item.sk) { + const revertObj = { + TableName: TABLE_NAME, + Key: { + pk: { S: item.pk }, + sk: { S: item.sk } + }, + UpdateExpression: 'remove #searchFirstName, #searchLastName', + ExpressionAttributeNames: { + '#searchFirstName': 'searchFirstName', + '#searchLastName': 'searchLastName' + }, + ReturnValues: 'ALL_NEW' + }; + try { + const data = await runUpdate(revertObj); + updatedItems.push(data); + } catch (err) { + console.log('Error removing pass sk: ' + item.sk + '. ' + err); + failedItems.push(item); + failureCount++; + break; + } + } else { + skippedCount++; + } + } + if (failureCount > 0){ + if(showFailures){ + console.log(failedItems); + } else { + console.log('run "node caseInsensitiveNames.js showFailures" to list failed items.') + } + } + console.log('--------') + console.log('reverted ' + updatedItems.length + ' items.'); + console.log(failureCount + ' failures.'); + console.log(skippedCount + ' passes were skipped for missing necessary fields.'); +} + +async function run() { + const allPasses = await getAllPasses(); + if (revert) { + console.log('REVERTING case-insensitive search fields for firstName & lastName...'); + await revertPasses(allPasses); + } else { + console.log('ADDING case-insensitive search fields for firstName & lastName...'); + await updatePasses(allPasses); + } + console.log('--------') + console.log('Done.'); +} + +run(); \ No newline at end of file