diff --git a/app.js b/app.js index 739a9a20..c0dfa06f 100644 --- a/app.js +++ b/app.js @@ -104,6 +104,10 @@ function createApp(config) { crawlerSecret ) + // enable heap stats logging at an interval if configured + const trySetHeapLoggingAtInterval = require('./lib/heapLogger') + trySetHeapLoggingAtInterval(config, logger) + const app = express() app.use(cors()) app.options('*', cors()) diff --git a/bin/config.js b/bin/config.js index e3011eab..3ec0993e 100644 --- a/bin/config.js +++ b/bin/config.js @@ -104,5 +104,9 @@ module.exports = { crawlerKey: config.get('APPINSIGHTS_CRAWLER_APIKEY') }, appVersion: config.get('APP_VERSION'), - buildsha: config.get('BUILD_SHA') + buildsha: config.get('BUILD_SHA'), + heapstats: { + logHeapstats: config.get('LOG_NODE_HEAPSTATS'), + logInverval: config.get('LOG_NODE_HEAPSTATS_INTERVAL_MS') + } } diff --git a/full.env.json b/full.env.json index 8deb15c9..3c668a21 100644 --- a/full.env.json +++ b/full.env.json @@ -47,5 +47,9 @@ "CRAWLER_QUEUE_PROVIDER": "memory", "CRAWLER_SERVICE_PORT": "5000", "CRAWLER_GITHUB_TOKEN": "< GitHub PAT here >", - "SCANCODE_HOME": "< ScanCode install location e.g., c:\\installs\\scancode-toolkit-2.2.1 >" + "SCANCODE_HOME": "< ScanCode install location e.g., c:\\installs\\scancode-toolkit-2.2.1 >", + + "========== Heapstats Logging settings (OPTIONAL) ==========": "", + "LOG_NODE_HEAPSTATS": "", + "LOG_NODE_HEAPSTATS_INTERVAL_MS": "" } diff --git a/lib/heapLogger.js b/lib/heapLogger.js new file mode 100644 index 00000000..34522110 --- /dev/null +++ b/lib/heapLogger.js @@ -0,0 +1,86 @@ +// (c) Copyright 2024, Microsoft and ClearlyDefined contributors. Licensed under the MIT license. +// SPDX-License-Identifier: MIT + +// =================================================== +// Log the heap statistics at regular intervals +// =================================================== +// NOTE: set 'LOG_NODE_HEAPSTATS' env var to 'true' to log heap stats +// NOTE: set 'LOG_NODE_HEAPSTATS_INTERVAL_MS' env var to '' for logging interval +// NOTE: To better understand heap stats being logged, check: +// - https://nodejs.org/docs/v22.12.0/api/v8.html#v8getheapspacestatistics +// - https://nodejs.org/docs/v22.12.0/api/v8.html#v8getheapstatistics +function trySetHeapLoggingAtInterval(config, logger) { + logger.debug('heapLogger.js :: Entered "trySetHeapLoggingAtInterval"...') + + const shouldLogHeapstats = config.heapstats.logHeapstats + ? config.heapstats.logHeapstats.toLowerCase() === 'true' + : false + + logger.debug(`heapLogger.js :: "shouldLogHeapstats" set to "${shouldLogHeapstats}"`) + + if (shouldLogHeapstats) { + const v8 = require('v8') + + const addCommas = num => Number(num).toLocaleString() + const isNumeric = num => !isNaN(Number(num)) + + // Set the heapstats logging interval + const maybeInterval = config.heapstats.logInverval + const heapStatsInverval = maybeInterval && isNumeric(maybeInterval) ? maybeInterval : 30000 + + logger.debug(`heapLogger.js :: heap stats logging interval will be "${heapStatsInverval}" ms`) + + // Function to log the heap space statistics + const logHeapSpaceStats = () => { + // Get the current timestamp + const currentTimestamp = new Date().toISOString() + + // Get the heap space statistics + const heapSpaceStats = v8.getHeapSpaceStatistics() + + heapSpaceStats.forEach(space => { + const heapStatsMessage = + `[${currentTimestamp}] Heap Space Statistics: ` + + `Space Name: '${space.space_name}', ` + + `Space Size: '${addCommas(space.space_size)}' bytes, ` + + `Space Used Size: '${addCommas(space.space_used_size)}' bytes, ` + + `Space Available Size: '${addCommas(space.space_available_size)}' bytes, ` + + `Physical Space Size: '${addCommas(space.physical_space_size)}' bytes` + + '\n--------------------------' + + logger.info(heapStatsMessage) + }) + + // Get the heap statistics + const heapStats = v8.getHeapStatistics() + + const heapStatsMessage = + `[${currentTimestamp}] Heap Statistics: ` + + `Total Heap Size: '${addCommas(heapStats.total_heap_size)}' bytes, ` + + `Total Heap Size Executable: '${addCommas(heapStats.total_heap_size_executable)}' bytes, ` + + `Total Physical Size: '${addCommas(heapStats.total_physical_size)}' bytes, ` + + `Total Available Size: '${addCommas(heapStats.total_available_size)}' bytes, ` + + `Used Heap Size: '${addCommas(heapStats.used_heap_size)}' bytes, ` + + `Heap Size Limit: '${addCommas(heapStats.heap_size_limit)}' bytes` + + '\n--------------------------' + + logger.info(heapStatsMessage) + } + + // Only run if not in a test environment + if (process.argv.every(arg => !arg.includes('mocha'))) { + logger.debug(`heapLogger.js :: setting heap stats logging at "${heapStatsInverval}" ms interval...`) + + // Set the interval to log the heap space statistics + setInterval(logHeapSpaceStats, heapStatsInverval) + + logger.debug(`heapLogger.js :: set heap stats logging at "${heapStatsInverval}" ms interval.`) + } + } else { + logger.debug('heapLogger.js :: heap stats logging not enabled.') + } + + logger.debug('heapLogger.js :: Exiting "trySetHeapLoggingAtInterval".') +} + +module.exports = trySetHeapLoggingAtInterval diff --git a/minimal.env.json b/minimal.env.json index 9f6916c9..54e23abd 100644 --- a/minimal.env.json +++ b/minimal.env.json @@ -8,5 +8,9 @@ "========== Service Curation settings ==========": "", "CURATION_GITHUB_REPO": "sample-curated-data", - "CURATION_GITHUB_TOKEN": "" + "CURATION_GITHUB_TOKEN": "", + + "========== Heapstats Logging settings (OPTIONAL) ==========": "", + "LOG_NODE_HEAPSTATS": "", + "LOG_NODE_HEAPSTATS_INTERVAL_MS": "" } diff --git a/package.json b/package.json index 3db16f7b..2f556f42 100644 --- a/package.json +++ b/package.json @@ -12,7 +12,7 @@ "prettier:check": "prettier . --check", "prettier:write": "prettier . --write", "dev": "nodemon ./bin/www", - "start": "node ./bin/www", + "start": "node --max-old-space-size=8192 ./bin/www", "postinstall": "patch-package" }, "license": "MIT",