BRS-196 - case insensitive search on first and last names (#67)

Changing IS_OFFLINE to boolean updating README
bcgov · Jan 17, 2022 · 27901f7 · 27901f7
1 parent 68431e4
commit 27901f7
Show file tree

Hide file tree

Showing 3 changed files with 227 additions and 8 deletions.
diff --git a/lambda/readPass/index.js b/lambda/readPass/index.js
@@ -68,19 +68,19 @@ exports.handler = async (event, context) => {
       // Filter first/last
       if (event.queryStringParameters.firstName) {
         queryObj = checkAddExpressionAttributeNames(queryObj);
-        queryObj.ExpressionAttributeNames['#firstName'] = 'firstName';
-        queryObj.ExpressionAttributeValues[':firstName'] = AWS.DynamoDB.Converter.input(
-          event.queryStringParameters.firstName
+        queryObj.ExpressionAttributeNames['#searchFirstName'] = 'searchFirstName';
+        queryObj.ExpressionAttributeValues[':searchFirstName'] = AWS.DynamoDB.Converter.input(
+          event.queryStringParameters.firstName.toLowerCase()
         );
-        queryObj.FilterExpression += ' AND #firstName =:firstName';
+        queryObj.FilterExpression += ' AND #searchFirstName =:searchFirstName';
       }
       if (event.queryStringParameters.lastName) {
         queryObj = checkAddExpressionAttributeNames(queryObj);
-        queryObj.ExpressionAttributeNames['#lastName'] = 'lastName';
-        queryObj.ExpressionAttributeValues[':lastName'] = AWS.DynamoDB.Converter.input(
-          event.queryStringParameters.lastName
+        queryObj.ExpressionAttributeNames['#searchLastName'] = 'searchLastName';
+        queryObj.ExpressionAttributeValues[':searchLastName'] = AWS.DynamoDB.Converter.input(
+          event.queryStringParameters.lastName.toLowerCase()
         );
-        queryObj.FilterExpression += ' AND #lastName =:lastName';
+        queryObj.FilterExpression += ' AND #searchLastName =:searchLastName';
       }
       // Filter email
       if (event.queryStringParameters.email) {

diff --git a/lambda/writePass/index.js b/lambda/writePass/index.js
@@ -136,7 +136,9 @@ exports.handler = async (event, context) => {
     passObject.Item['pk'] = { S: 'pass::' + parkName };
     passObject.Item['sk'] = { S: registrationNumber };
     passObject.Item['firstName'] = { S: firstName };
+    passObject.Item['searchFirstName'] = { S: firstName.toLowerCase() };
     passObject.Item['lastName'] = { S: lastName };
+    passObject.Item['searchLastName'] = { S: lastName.toLowerCase() };
     passObject.Item['facilityName'] = { S: facilityName };
     passObject.Item['email'] = { S: email };
     passObject.Item['date'] = { S: date };

diff --git a/migrations/caseInsensitiveNames.js b/migrations/caseInsensitiveNames.js
@@ -0,0 +1,217 @@
+/*
+Case-insensitive names - this migration is to update existing passes with new search fields: searchFirstName & searchLastName. 
+These fields are copies of the firstName and lastName fields cast to lowercase characters since DynamoDB does not support case-insensitive string searching.
+Passes will not update if they do not contain both firstName and lastName fields.
+
+To run locally: Ensure your dyanmodb-local db is running in a docker instance or similar.
+
+`export IS_OFFLINE=1`
+`node caseInsensitiveNames.js`
+
+To run in AWS environment: Configure AWS credentials to target AWS env.
+
+`export IS_OFFLINE=0`
+`node caseInsensitiveNames.js`
+
+To revert changes (removes searchFirstName & searchLastName fields from all passes), add revert argument:
+
+`node caseInsensitiveNames.js revert`
+
+To list passes that failed to update or revert, add show-failures argument :
+
+`node caseInsensitiveNames.js show-failures`
+*/
+
+const AWS = require('aws-sdk');
+
+TABLE_NAME = process.env.TABLE_NAME || 'parksreso';
+
+const args = process.argv;
+let revert = false;
+let showFailures = false;
+
+if (args.includes('show-failures')){
+  showFailures = true;
+}
+
+if (args.includes('revert')){
+  revert = true;
+}
+
+const options = {
+  region: 'ca-central-1'
+};
+
+if (process.env.IS_OFFLINE) {
+  options.endpoint = 'http://localhost:8000';
+}
+
+const dynamodb = new AWS.DynamoDB(options);
+
+exports.dynamodb = new AWS.DynamoDB();
+
+// Scan for all passes
+async function getAllPasses() {
+  const scanObj = {
+    TableName: TABLE_NAME,
+    ConsistentRead: true,
+    ExpressionAttributeNames: {
+      '#pk': 'pk'
+    },
+    ExpressionAttributeValues: {
+      ':beginsWith': {
+        S: 'pass::'
+      }
+    },
+    FilterExpression: 'begins_with(#pk, :beginsWith)'
+  };
+
+  try {
+    const res = await runScan(scanObj);
+    return res;
+  } catch (err) {
+    console.log('Error getting passes:', err);
+    return null;
+  }
+}
+
+async function runScan(scan, paginated = false) {
+  const data = await dynamodb.scan(scan).promise();
+  var unMarshalled = data.Items.map(item => {
+    return AWS.DynamoDB.Converter.unmarshall(item);
+  });
+  if (paginated) {
+    return {
+      LastEvaluatedKey: data.LastEvaluatedKey,
+      data: unMarshalled
+    };
+  } else {
+    return unMarshalled;
+  }
+}
+
+async function runUpdate(update, paginated = false) {
+  const data = await dynamodb.updateItem(update).promise();
+  if (paginated) {
+    return {
+      LastEvaluatedKey: data.LastEvaluatedKey,
+      data: data
+    };
+  } else {
+    return data;
+  }
+}
+
+// add searchFirstName and searchLastName fields to all passes
+async function updatePasses(passList) {
+  console.log('collected ' + passList.length + ' passes...');
+  let updatedItems = [];
+  let failedItems = [];
+  let failureCount = 0;
+  let skippedCount = 0;
+  for (let item of passList) {
+    if (item.firstName && item.lastName && item.pk && item.sk) {
+      const updateObj = {
+        TableName: TABLE_NAME,
+        Key: {
+          pk: { S: item.pk },
+          sk: { S: item.sk }
+        },
+        UpdateExpression: 'set #searchFirstName = :searchFirstName, #searchLastName = :searchLastName',
+        ExpressionAttributeNames: {
+          '#searchFirstName': 'searchFirstName',
+          '#searchLastName': 'searchLastName'
+        },
+        ExpressionAttributeValues: {
+          ':searchFirstName': {S: item.firstName.toLowerCase()},
+          ':searchLastName': {S: item.lastName.toLowerCase()}
+        },
+        ReturnValues: 'ALL_NEW'
+      };
+      try {
+        const data = await runUpdate(updateObj);
+        updatedItems.push(data);
+      } catch (err) {
+        failureCount++;
+        console.log('Error updating pass sk: ' + item.sk + '. ' + err);
+        break;
+      }
+    } else {
+      skippedCount++;
+    }
+  }
+  if (failureCount > 0){
+    if(showFailures){
+      console.log("Failures:\n", failedItems);
+    } else {
+      console.log('run "node caseInsensitiveNames.js showFailures" to list failed items.')
+    }
+  }
+  console.log('--------')
+  console.log('updated ' + updatedItems.length + ' items.');
+  console.log(failureCount + ' failures.' );
+  console.log(skippedCount + ' passes were skipped for missing necessary fields.');
+}
+
+// remove searchFirstName and searchLastName fields from all passes
+async function revertPasses(passList) {
+  console.log('collected ' + passList.length + ' passes...');
+  let updatedItems = [];
+  let failedItems = [];
+  let failureCount = 0;
+  let skippedCount = 0;
+  for (let item of passList) {
+    if (item.searchFirstName && item.searchLastName && item.pk && item.sk) {
+      const revertObj = {
+        TableName: TABLE_NAME,
+        Key: {
+          pk: { S: item.pk },
+          sk: { S: item.sk }
+        },
+        UpdateExpression: 'remove #searchFirstName, #searchLastName',
+        ExpressionAttributeNames: {
+          '#searchFirstName': 'searchFirstName',
+          '#searchLastName': 'searchLastName'
+        },
+        ReturnValues: 'ALL_NEW'
+      };
+      try {
+        const data = await runUpdate(revertObj);
+        updatedItems.push(data);
+      } catch (err) {
+        console.log('Error removing pass sk: ' + item.sk + '. ' + err);
+        failedItems.push(item);
+        failureCount++;
+        break;
+      }
+    } else {
+      skippedCount++;
+    }
+  }
+  if (failureCount > 0){
+    if(showFailures){
+      console.log(failedItems);
+    } else {
+      console.log('run "node caseInsensitiveNames.js showFailures" to list failed items.')
+    }
+  }
+  console.log('--------')
+  console.log('reverted ' + updatedItems.length + ' items.');
+  console.log(failureCount + ' failures.');
+  console.log(skippedCount + ' passes were skipped for missing necessary fields.');
+}
+
+async function run() {
+  const allPasses = await getAllPasses();
+  if (revert) {
+    console.log('REVERTING case-insensitive search fields for firstName & lastName...');
+    await revertPasses(allPasses);
+  } else {
+    console.log('ADDING case-insensitive search fields for firstName & lastName...');
+    await updatePasses(allPasses);
+  }
+  console.log('--------')
+  console.log('Done.');
+}
+
+run();