-
Notifications
You must be signed in to change notification settings - Fork 13
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
BRS-449: creating warmup functionality for Lambda (#86)
- Loading branch information
1 parent
048c502
commit 976c9d9
Showing
14 changed files
with
415 additions
and
7 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,66 @@ | ||
| const { v4: uuidv4 } = require('uuid'); | ||
|
|
||
| const mainThreadConfigDefaults = { | ||
| configArray: [], | ||
| log: true, | ||
| delay: 1000 | ||
| }; | ||
|
|
||
| const configDefaults = { | ||
| funcName: null, | ||
| funcVersion: null, | ||
| concurrency: 'concurrency', // default concurrency field | ||
| log: true, // default logging to true | ||
| correlationId: uuidv4(), // default the correlationId | ||
| }; | ||
|
|
||
| const { Worker } = require('worker_threads'); | ||
| exports.handler = async (event) => { | ||
| const mainThreadConfig = Object.assign({}, mainThreadConfigDefaults, event); | ||
| if (mainThreadConfig.configArray.length > 0) { | ||
| const threads = new Set(); | ||
| for (let i = 0; i < mainThreadConfig.configArray.length; i++) { | ||
| const config = Object.assign({}, configDefaults, mainThreadConfig.configArray[i]); | ||
| if (config.funcName && config.funcVersion) { | ||
| threads.add(new Worker(__dirname + '/worker.js', { workerData: { config: config } })); | ||
| logMessage(mainThreadConfigDefaults.log, `Worker added to thread list with config: ${config}`); | ||
| } | ||
| } | ||
|
|
||
| for (let worker of threads) { | ||
| worker.on('error', (err) => { | ||
| logMessage(mainThreadConfigDefaults.log, `We have an error: ${err}`); | ||
| threads.delete(worker); | ||
| }); | ||
| worker.on('exit', () => { | ||
| threads.delete(worker); | ||
| logMessage(mainThreadConfigDefaults.log, `Thread exiting, ${threads.size} running...`); | ||
| if (threads.size === 0) { | ||
| logMessage(mainThreadConfigDefaults.log, 'Warm up complete.'); | ||
| } | ||
| }); | ||
| worker.on('message', (msg) => { | ||
| logMessage(mainThreadConfigDefaults.log, msg); | ||
| }); | ||
| } | ||
|
|
||
| return new Promise(async (resolve) => { | ||
| while (true) { | ||
| await new Promise(r => setTimeout(r, mainThreadConfig.delay)); | ||
| if (threads.size === 0) { | ||
| logMessage(mainThreadConfigDefaults.log, 'All threads complete.'); | ||
| break; | ||
| } else { | ||
| logMessage(mainThreadConfigDefaults.log, '.'); | ||
| } | ||
| } | ||
| resolve(); | ||
| }); | ||
| } | ||
| }; // end module | ||
|
|
||
| function logMessage(logBoolean, message) { | ||
| if (logBoolean === true) { | ||
| console.log(message); | ||
| } | ||
| } |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,64 @@ | ||
| # Lambda Warm Up Functions | ||
| This function utilizes threading generate concurrent calls to a given function. | ||
|
|
||
| ## Issue | ||
| In the case of DUP, we often get high ammounts of traffic at 7AM. This is because 7 AM is when people are allowed to book passes. Since we received this wave of users at the same time, some people reported communication errors with our servers. This is because Lambda takes time to automatically spin up instances to handle load. | ||
|
|
||
| ## Solution | ||
| One part of the solution was to create a way to warm up relevant Lambda functions prior to spikes in traffic. This function allows us to do that. The following functions have been equiped with a warm up break out: | ||
|
|
||
| ``` | ||
| readConfig | ||
| readPark | ||
| readFacility | ||
| generateCaptcha | ||
| verifyCaptcha | ||
| writePass | ||
| ``` | ||
|
|
||
| When creating a call to these functions, use the following payload: | ||
|
|
||
| ``` | ||
| { | ||
| "warmup": true | ||
| } | ||
| ``` | ||
|
|
||
| This will cause the function to skip all functionality and return a 200 OK. This is all that is required for Lambda to spin up instances of itself, given there are enough concurrent calls to the function. | ||
|
|
||
| ## Current AWS Solution | ||
| The pipeline for this code is as follows: | ||
|
|
||
| ``` | ||
| yarn build > Terraform > Lambda | ||
| Terraform > Eventbridge > Invoke warmup | ||
| ``` | ||
|
|
||
| At the moment, we have EventBridge invoke the warm up function at 6:57 AM, 6:58 AM and 6:59 AM every day. | ||
|
|
||
| ## Example payload | ||
| ``` | ||
| { | ||
| "configArray": [ | ||
| { | ||
| "funcName": "readConfig", | ||
| "funcVersion": "latest", | ||
| "concurrency": "5", | ||
| "log": true | ||
| }, | ||
| { | ||
| "funcName": "readPark", | ||
| "funcVersion": "latest", | ||
| "concurrency": "10" | ||
| }, | ||
| { | ||
| "funcName": "readFacility", | ||
| "funcVersion": "2", | ||
| "concurrency": "100" | ||
| } | ||
| ], | ||
| "delay": 1000, | ||
| "log": true | ||
| } | ||
| ``` |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,64 @@ | ||
| const AWS = require('aws-sdk'); | ||
| const lambda = new AWS.Lambda(); | ||
| const { parentPort, workerData } = require('worker_threads'); | ||
|
|
||
| const config = workerData.config; | ||
|
|
||
| let concurrency = config.concurrency | ||
| && !isNaN(config.concurrency) | ||
| && config.concurrency > 1 | ||
| ? config.concurrency : 1; | ||
|
|
||
| // Log it | ||
| if (config.log) { | ||
| const lastAccess = Date.now(); | ||
| // Create log record | ||
|
|
||
| const log = { | ||
| action: 'warmer', | ||
| function: config.funcName + ':' + config.funcVersion, | ||
| correlationId: config.correlationId, | ||
| count: 1, | ||
| concurrency: concurrency, | ||
| lastAccessed: lastAccess, | ||
| lastAccessedSeconds: lastAccess === null ? null : ((Date.now() - lastAccess) / 1000).toFixed(1) | ||
| }; | ||
| parentPort.postMessage(`{"message":"${config.funcName}: ${log}"}`); | ||
| } | ||
|
|
||
| // Fan out if concurrency is set higher than 1 | ||
| if (concurrency > 1) { | ||
| // init promise array | ||
| let invocations = []; | ||
|
|
||
| // loop through concurrency count | ||
| for (let i = 2; i <= concurrency; i++) { | ||
|
|
||
| // Set the params and wait for the final function to finish | ||
| let params = { | ||
| FunctionName: config.funcName + ':' + config.funcVersion, | ||
| InvocationType: i === concurrency ? 'RequestResponse' : 'Event', | ||
| LogType: 'None', | ||
| Payload: Buffer.from(JSON.stringify({ | ||
| '__WARMER_INVOCATION__': i, // send invocation number | ||
| '__WARMER_CONCURRENCY__': concurrency, // send total concurrency | ||
| '__WARMER_CORRELATIONID__': config.correlationId, // send correlation id | ||
| 'warmup': true | ||
| })) | ||
| }; | ||
|
|
||
| // Add promise to invocations array | ||
| invocations.push(lambda.invoke(params).promise()); | ||
| } // end for | ||
|
|
||
| // Invoke concurrent functions | ||
| try { | ||
| Promise.all(invocations).then(() => { | ||
| if (config.log) { | ||
| parentPort.postMessage(`{"message":"${config.funcName} has been warmed up successfully."}`); | ||
| } | ||
| }) | ||
| } catch (error) { | ||
| throw error; | ||
| } | ||
| } |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Oops, something went wrong.