Lambdas now return HTTP status 400 for errors instead of 200 with ret…

…urn = false, updates to README

README.md

@@ -1,4 +1,15 @@
 # PHSA MRI CODE 
+- UBC CIC Medical Imaging Scheduler System 
+- To start this project, you will need to create the following before running the cloudformation script: 
+> S3 bucket 
+> EC2 instance
+> Security Groups
+> VPC with Private and Public Subnets 
+- To run the cloudformation code, use the following commands: 
+```
+sam package --s3-bucket <s3 bucket name> --output-template-file out.yaml
+sam deploy --template-file out.yaml --capabilities CAPABILITY_IAM CAPABILITY_AUTO_EXPAND --stack-name <stack name>
+```
 
 ## Setup PostgreSQL in EC2
 - Follow [these instructions](https://installvirtual.com/install-postgresql-10-on-amazon-ec2/) to download postgres onto EC2
@@ -23,7 +34,7 @@ host    all             all             0.0.0.0/0               md5
 systemctl restart postgresql
 ```
 - On the AWS console, you'll need to add access to Postgres to the EC2 instance to the security group 
-- To initialize the database, run the init_db.sql script in the terminal using `psql -U <user> -h <host> -f init_db.sql`. _Note this does assume you have postgresql database named rules._
+- To initialize the database, run the init_db.sql script in the terminal using `psql -U <user> -h <host> -f init_db.sql` after adding the file locations. _Note this does assume you have postgresql database named rules and psql installed on your own computer. Otherwise, you copy and run script directly in Postgres._
 - The functions expect the database keys to be stored on SSM Parameter Store. You can either use the AWS Console or the following AWS CLI commands: 
 ```bash
 aws ssm put-parameter --name /mri-phsa/dbserver_ec2 --value <host> --type SecureString --overwrite
@@ -39,21 +50,27 @@ aws ssm put-parameter --name /mri-phsa/ec2 --value <instance id> --type SecureSt
 - [sample_output.json](sample_output.json): Sample JSON output for first 100 rows of data of sample.csv
 - To create a sample_output.json, in the terminal run `python .\preprocess\preprocess_data.py`
 
+![Preprocessing Decision Tree](/media/decisionTree_preprocess.png)
+
 ## Rule Processing 
 - [rules.py](/rule_processing/rules.py): Main python script to obtain the priority value 
 - [update_weights.py](/rule_processing/update_weights.py): Creates weighted tokens in the database for the descriptions found in mri_rules
 - [postgresql.py](/rule_processing/postgresql.py): Connecting to the postgres database with a SSM parameter store =
 - To update the weighted rule tokens, run `python .\rules_processing\update_weights.py` in the terminal
 - To apply the rules and obtain a Rule ID + P-Value, run `python .\rule_processing\rules.py` in the terminal
 
+![Rule Decision Tree](/media/decisionTree_rules.png)
+
+
 ## Lambdas
-- Python Code used for lambda functions
-- To run the cloudformation code, use the following commands: 
-```
-sam package --s3-bucket <s3 bucket name> --output-template-file out.yaml
-sam deploy --template-file out.yaml --capabilities CAPABILITY_IAM CAPABILITY_AUTO_EXPAND --stack-name <stack name>
-```
-- For the Thesaurus Lambda, you will need to create a [Run Command called copyFile](copyFile.json) in AWS Systems Manager through the Console
+- Python Code used for Lambda Functions 
+
+## Layers 
+- Custom libraries used for Lambda Functions not found on AWS 
+- To update or add a new layer, they must be zipped with the top level folder named __python__
 
 ## Rule Database Analysis
-- [Sample Result](/csv/mri_dataset_results_0820.xlsx): Form to P-Value Data (most recent)
+- [Sample Result](/csv/mri_dataset_results_0820.xlsx): Form to P-Value Data (most recent)
+
+### Further Recommendations 
+This is a proof of concept and it

init_db.sql

@@ -69,7 +69,7 @@ CREATE TABLE IF NOT EXISTS specialty_tags (
 )
 
 -- SELECT * FROM mri_rules; 
-\copy mri_rules(body_part, info, contrast, priority) FROM 'C:\Users\jackhou\Documents\mri_project\mri_app\csv\rules.csv' DELIMITER ',' CSV HEADER;
+\copy mri_rules(body_part, info, contrast, priority) FROM '<PROJECT LOCATION>\csv\rules.csv' DELIMITER ',' CSV HEADER;
 
 UPDATE mri_rules 
 SET bp_tk = to_tsvector(body_part);
@@ -82,16 +82,16 @@ CREATE INDEX tags_idx
 ON data_results
 USING GIN(tags);
 
-\copy word_weights FROM 'C:\Users\jackhou\Documents\mri_project\mri_app\csv\wordweights.csv' DELIMITER ',' CSV;
+\copy word_weights FROM '<PROJECT LOCATION>\csv\wordweights.csv' DELIMITER ',' CSV;
 
-\copy spellchecker FROM 'C:\Users\jackhou\Documents\mri_project\mri_app\csv\spellchecker.csv' DELIMITER ',' CSV; 
+\copy spellchecker FROM '<PROJECT LOCATION>\csv\spellchecker.csv' DELIMITER ',' CSV;
 
-\copy specialty_tags FROM 'C:\Users\jackhou\Documents\mri_project\mri_app\csv\specialty_exams.csv' DELIMITER ',' CSV;
+\copy specialty_tags FROM '<PROJECT LOCATION>\csv\specialty_exams.csv' DELIMITER ',' CSV;
 
 UPDATE word_weights
 SET word = TRIM(word); 
 
 UPDATE spellchecker 
 SET word = TRIM(word);
 
-\copy conjunctions FROM 'C:\Users\jackhou\Documents\mri_project\mri_app\csv\conjunctions.csv' DELIMITER ',' CSV;
+\copy conjunctions FROM '<PROJECT LOCATION>\csv\conjunctions.csv' DELIMITER ',' CSV;

lambdas/preprocess/index.py

@@ -202,7 +202,7 @@ def handler(event, context):
     logger.info(event)
     if 'body' not in event:
         logger.error( 'Missing parameters')
-        return {'result': False, 'msg': 'Missing parameters' }
+        return {"isBase64Encoded": False, "statusCode": 400, "body": "Missing Body Parameter", "headers": {"Content-Type": "application/json"}}
 
     data_df = json.loads(event['body']) # use for postman tests
     # data_df = event['body'] # use for console tests
@@ -274,14 +274,17 @@ def handler(event, context):
     #formatted_df.to_json('sample_output.json', orient='index')
     #print("output is: ", formatted_df)
 
-    response = lambda_client.invoke(
+    rules_response = lambda_client.invoke(
             FunctionName=RuleProcessingLambdaName,
             InvocationType='RequestResponse',
             Payload=json.dumps(formatted_df)
         )
-
-    data = json.loads(response['Payload'].read())
-
+
+    if rule_response['ResponseMetadata']['HTTPStatusCode'] != 200: 
+        return rule_response
+
+    data = json.loads(rule_response['Payload'].read())
+
     response = { 
         'result': data, 
         'context': formatted_df

lambdas/query_conjunctions/index.py

@@ -52,7 +52,7 @@ def handler(event, context):
     logger.info(event)
     if 'body' not in event:
         logger.error( 'Missing parameters')
-        return {'result': False, 'msg': 'Missing parameters' }
+        return {"isBase64Encoded": False, "statusCode": 400, "body": "Missing Body Parameter", "headers": {"Content-Type": "application/json"}}
 
     data = json.loads(event['body']) # use for postman tests
     # data = event['body'] # use for console tests
@@ -82,7 +82,7 @@ def handler(event, context):
         except Exception as error:
             logger.error(error)
             logger.error("Exception Type: %s" % type(error))
-            return {'result': False, 'msg': f'{error}'}
+            return {"isBase64Encoded": False, "statusCode": 400, "body": f'{type(error)}', "headers": {"Content-Type": "application/json"}}
 
     return {'result': True}
 

lambdas/query_rules_table/index.py

@@ -103,7 +103,8 @@ def handler(event, context):
     logger.info(event)
     if 'body' not in event:
         logger.error( 'Missing parameters')
-        return {'result': False, 'msg': 'Missing parameters body' }
+        return {"isBase64Encoded": False, "statusCode": 400, "body": "Missing Body Parameter", "headers": {"Content-Type": "application/json"}}
+
 
     data = json.loads(event['body']) # use for postman tests
     # data = event['body'] # use for console tests
@@ -142,7 +143,7 @@ def handler(event, context):
         except Exception as error:
             logger.error(error)
             logger.error("Exception Type: %s" % type(error))
-            return {'result': False, 'msg': f'{error}'}
+            return {"isBase64Encoded": False, "statusCode": 400, "body": f'{type(error)}', "headers": {"Content-Type": "application/json"}}
 
     return {'result': True}
 

lambdas/query_specialty_tags/index.py

@@ -36,7 +36,7 @@ def handler(event, context):
     logger.info(event)
     if 'body' not in event:
         logger.error( 'Missing parameters')
-        return {'result': False, 'msg': 'Missing parameters' }
+        return {"isBase64Encoded": False, "statusCode": 400, "body": "Missing Body Parameter", "headers": {"Content-Type": "application/json"}}
 
     data = json.loads(event['body']) # use for postman tests
     # data = event['body'] # use for console tests
@@ -63,7 +63,7 @@ def handler(event, context):
         except Exception as error:
             logger.error(error)
             logger.error("Exception Type: %s" % type(error))
-            return {'result': False, 'msg': f'{error}'}
+            return {"isBase64Encoded": False, "statusCode": 400, "body": f'{type(error)}', "headers": {"Content-Type": "application/json"}}
 
     return {'result': True}
 

lambdas/query_spellchecker/index.py

@@ -36,7 +36,7 @@ def handler(event, context):
     logger.info(event)
     if 'body' not in event:
         logger.error( 'Missing parameters')
-        return {'result': False, 'msg': 'Missing parameters' }
+        return {"isBase64Encoded": False, "statusCode": 400, "body": "Missing Body Parameter", "headers": {"Content-Type": "application/json"}}
 
     data = json.loads(event['body']) # use for postman tests
     # data = event['body'] # use for console tests
@@ -64,7 +64,7 @@ def handler(event, context):
         except Exception as error:
             logger.error(error)
             logger.error("Exception Type: %s" % type(error))
-            return {'result': False, 'msg': f'{error}'}
+            return {"isBase64Encoded": False, "statusCode": 400, "body": f'{type(error)}', "headers": {"Content-Type": "application/json"}}
 
     return {'result': True}
 

lambdas/query_weights/index.py

@@ -64,7 +64,7 @@ def handler(event, context):
     logger.info(event)
     if 'body' not in event:
         logger.error( 'Missing parameters')
-        return {'result': False, 'msg': 'Missing parameters' }
+        return {"isBase64Encoded": False, "statusCode": 400, "body": "Missing Body Parameter", "headers": {"Content-Type": "application/json"}}
 
     data = json.loads(event['body']) # use for postman tests
     # data = event['body'] # use for console tests
@@ -96,7 +96,7 @@ def handler(event, context):
         except Exception as error:
             logger.error(error)
             logger.error("Exception Type: %s" % type(error))
-            return {'result': False, 'msg': f'{error}'}
+            return {"isBase64Encoded": False, "statusCode": 400, "body": f'{type(error)}', "headers": {"Content-Type": "application/json"}}
 
     return {'result': True}
 

lambdas/results/index.py

@@ -96,7 +96,7 @@ def handler(event, context):
     logger.info(event)
     if 'body' not in event:
         logger.error( 'Missing parameters')
-        return {'result': False, 'msg': 'Missing parameters body' }
+        return {"isBase64Encoded": False, "statusCode": 400, "body": "Missing Body Parameter", "headers": {"Content-Type": "application/json"}}
 
     data = json.loads(event['body']) # use for postman tests
     # data = event['body'] # use for console tests
@@ -127,6 +127,6 @@ def handler(event, context):
         except Exception as error:
             logger.error(error)
             logger.error("Exception Type: %s" % type(error))
-            return {'result': False, 'msg': f'{error}'}
+            return {"isBase64Encoded": False, "statusCode": 400, "body": f'{type(error)}', "headers": {"Content-Type": "application/json"}}
 
     return {'result': True}

lambdas/rule_processing/index.py

@@ -99,7 +99,7 @@ def handler(event, context):
         except Exception as error:
             logger.error(error)
             logger.error("Exception Type: %s" % type(error))         
-            return {'result': False, 'msg': f'{error}'}
+            return {"isBase64Encoded": False, "statusCode": 500, "body": f'{type(error)}', "headers": {"Content-Type": "application/json"}}
 
 
 

lambdas/thesaurus/index.py

@@ -9,6 +9,7 @@
 logger.setLevel(logging.INFO)
 destPath = os.getenv('DEST_PATH')
 sendCommandName = os.getenv('SEND_COMMAND_NAME')
+ssmPath = os.getenv("SSM_PATH")
 s3 = boto3.client('s3')
 ec2 = boto3.client('ec2')
 ssm = boto3.client('ssm')
@@ -32,19 +33,18 @@
 
 def handler(event, context):
     #get parameters from ssm
-    p_ec2 = '/mri-phsa/ec2'
     params = ssm.get_parameters(
         Names=[
-            p_ec2
+            ssmPath
         ],
         WithDecryption = True
     )
     logger.info("Finished Acquiring Params")
     if params['ResponseMetadata']['HTTPStatusCode'] != 200:
         logger.info('ParameterStore Error: ', str(params['ResponseMetadata']['HTTPStatusCode']))
-        sys.exit(1)
+        return {"isBase64Encoded": False, "statusCode": params['ResponseMetadata']['HTTPStatusCode'], "body": "SSM Error", "headers": {"Content-Type": "application/json"}}
     for p in params['Parameters']:
-        if p['Name'] == p_ec2:
+        if p['Name'] == ssmPath:
             ec2 = p['Value']
     #get bucket name
     bucket = event['Records'][0]['s3']['bucket']['name']
@@ -83,4 +83,4 @@ def handler(event, context):
             logger.info("Updated PostgreSQL Text Configuration")
         except Exception as error:
             logger.error(error)
-            logger.error("Exception Type: %s" % type(error))         
+            return {"isBase64Encoded": False, "statusCode": 400, "body": f'{type(error)}', "headers": {"Content-Type": "application/json"}}

preprocess/preprocess_data.py

@@ -25,7 +25,7 @@
 data_df['Hip & Knee'] = preProcessText(data_df['Appropriateness Checklist - Hip & Knee'])
 data_df['Spine'] = preProcessText(data_df['Appropriateness Checklist - Spine'])
 
-# New Dataframe with
+# New Dataframe with preprocessed columns and columns for comprehend data 
 formatted_df = data_df[['CIO_ID', 'height', 'weight', 'Sex','age', 'Preferred MRI Site', 'priority']]
 formatted_df.loc[:,'p5'] = 'f'
 formatted_df.loc[:,'medical_condition'] = ''
@@ -68,16 +68,12 @@
         # elif(contains_word('spine',anatomy)):
         #     # apply comprehend to spine column
         #     formatted_df['Spine'][row] = find_entities(f'{data_df["Appropriateness Checklist - Spine"][row]}') 
-
+
+    # Use comprehend medical infer icd10cm on Reason for Exam column
     infer_icd10_cm(preprocessed_text, medical_conditions, diagnosis, symptoms)
+    # Use comprehend key phrases on Reason for Exam column
     find_key_phrases(preprocessed_text, key_phrases, medical_conditions+diagnosis+symptoms, anatomy_list)
-
-    # formatted_df['anatomy'][row] = anatomy_list    
-    # formatted_df['medical_condition'][row] = medical_conditions
-    # formatted_df['diagnosis'][row] = diagnosis
-    # formatted_df['symptoms'][row] = symptoms
-    # formatted_df['phrases'][row] = key_phrases
-    # formatted_df['other_info'][row] = other_info
+    # Set values in the formatted_df 
     formatted_df.at[row, 'anatomy'] = anatomy_list    
     formatted_df.at[row, 'medical_condition'] = medical_conditions
     formatted_df.at[row, 'diagnosis'] = diagnosis

template.yaml

@@ -382,6 +382,7 @@ Resources:
         Variables:
           DEST_PATH: "/usr/share/pgsql/tsearch_data/" # Path to shared postgresql directory
           SEND_COMMAND_NAME: !Ref copyFile
+          SSM_PATH: "/mri-phsa/ec2"
 
 Outputs:
   Preprocess: