Merge branch 'master' into qt/definition_upgrade

app.js

@@ -104,6 +104,10 @@ function createApp(config) {
     crawlerSecret
   )
 
+  // enable heap stats logging at an interval if configured
+  const trySetHeapLoggingAtInterval = require('./lib/heapLogger')
+  trySetHeapLoggingAtInterval(config, logger)
+
   const app = express()
   app.use(cors())
   app.options('*', cors())

bin/config.js

@@ -104,5 +104,9 @@ module.exports = {
     crawlerKey: config.get('APPINSIGHTS_CRAWLER_APIKEY')
   },
   appVersion: config.get('APP_VERSION'),
-  buildsha: config.get('BUILD_SHA')
+  buildsha: config.get('BUILD_SHA'),
+  heapstats: {
+    logHeapstats: config.get('LOG_NODE_HEAPSTATS'),
+    logInverval: config.get('LOG_NODE_HEAPSTATS_INTERVAL_MS')
+  }
 }

full.env.json

@@ -47,5 +47,9 @@
   "CRAWLER_QUEUE_PROVIDER": "memory",
   "CRAWLER_SERVICE_PORT": "5000",
   "CRAWLER_GITHUB_TOKEN": "< GitHub PAT here >",
-  "SCANCODE_HOME": "< ScanCode install location e.g., c:\\installs\\scancode-toolkit-2.2.1 >"
+  "SCANCODE_HOME": "< ScanCode install location e.g., c:\\installs\\scancode-toolkit-2.2.1 >",
+
+  "========== Heapstats Logging settings (OPTIONAL) ==========": "",
+  "LOG_NODE_HEAPSTATS": "<true|false>",
+  "LOG_NODE_HEAPSTATS_INTERVAL_MS": "<time_in_milliseconds (e.g. '30000')>"
 }

lib/heapLogger.js

@@ -0,0 +1,86 @@
+// (c) Copyright 2024, Microsoft and ClearlyDefined contributors. Licensed under the MIT license.
+// SPDX-License-Identifier: MIT
+
+// ===================================================
+// Log the heap statistics at regular intervals
+// ===================================================
+// NOTE: set 'LOG_NODE_HEAPSTATS' env var to 'true' to log heap stats
+// NOTE: set 'LOG_NODE_HEAPSTATS_INTERVAL_MS' env var to '<time_in_milliseconds>' for logging interval
+// NOTE: To better understand heap stats being logged, check:
+//   - https://nodejs.org/docs/v22.12.0/api/v8.html#v8getheapspacestatistics
+//   - https://nodejs.org/docs/v22.12.0/api/v8.html#v8getheapstatistics
+function trySetHeapLoggingAtInterval(config, logger) {
+  logger.debug('heapLogger.js :: Entered "trySetHeapLoggingAtInterval"...')
+
+  const shouldLogHeapstats = config.heapstats.logHeapstats
+    ? config.heapstats.logHeapstats.toLowerCase() === 'true'
+    : false
+
+  logger.debug(`heapLogger.js :: "shouldLogHeapstats" set to "${shouldLogHeapstats}"`)
+
+  if (shouldLogHeapstats) {
+    const v8 = require('v8')
+
+    const addCommas = num => Number(num).toLocaleString()
+    const isNumeric = num => !isNaN(Number(num))
+
+    // Set the heapstats logging interval
+    const maybeInterval = config.heapstats.logInverval
+    const heapStatsInverval = maybeInterval && isNumeric(maybeInterval) ? maybeInterval : 30000
+
+    logger.debug(`heapLogger.js :: heap stats logging interval will be "${heapStatsInverval}" ms`)
+
+    // Function to log the heap space statistics
+    const logHeapSpaceStats = () => {
+      // Get the current timestamp
+      const currentTimestamp = new Date().toISOString()
+
+      // Get the heap space statistics
+      const heapSpaceStats = v8.getHeapSpaceStatistics()
+
+      heapSpaceStats.forEach(space => {
+        const heapStatsMessage =
+          `[${currentTimestamp}] Heap Space Statistics: ` +
+          `Space Name: '${space.space_name}', ` +
+          `Space Size: '${addCommas(space.space_size)}' bytes, ` +
+          `Space Used Size: '${addCommas(space.space_used_size)}' bytes, ` +
+          `Space Available Size: '${addCommas(space.space_available_size)}' bytes, ` +
+          `Physical Space Size: '${addCommas(space.physical_space_size)}' bytes` +
+          '\n--------------------------'
+
+        logger.info(heapStatsMessage)
+      })
+
+      // Get the heap statistics
+      const heapStats = v8.getHeapStatistics()
+
+      const heapStatsMessage =
+        `[${currentTimestamp}] Heap Statistics: ` +
+        `Total Heap Size: '${addCommas(heapStats.total_heap_size)}' bytes, ` +
+        `Total Heap Size Executable: '${addCommas(heapStats.total_heap_size_executable)}' bytes, ` +
+        `Total Physical Size: '${addCommas(heapStats.total_physical_size)}' bytes, ` +
+        `Total Available Size: '${addCommas(heapStats.total_available_size)}' bytes, ` +
+        `Used Heap Size: '${addCommas(heapStats.used_heap_size)}' bytes, ` +
+        `Heap Size Limit: '${addCommas(heapStats.heap_size_limit)}' bytes` +
+        '\n--------------------------'
+
+      logger.info(heapStatsMessage)
+    }
+
+    // Only run if not in a test environment
+    if (process.argv.every(arg => !arg.includes('mocha'))) {
+      logger.debug(`heapLogger.js :: setting heap stats logging at "${heapStatsInverval}" ms interval...`)
+
+      // Set the interval to log the heap space statistics
+      setInterval(logHeapSpaceStats, heapStatsInverval)
+
+      logger.debug(`heapLogger.js :: set heap stats logging at "${heapStatsInverval}" ms interval.`)
+    }
+  } else {
+    logger.debug('heapLogger.js :: heap stats logging not enabled.')
+  }
+
+  logger.debug('heapLogger.js :: Exiting "trySetHeapLoggingAtInterval".')
+}
+
+module.exports = trySetHeapLoggingAtInterval

minimal.env.json

@@ -8,5 +8,9 @@
 
   "========== Service Curation settings ==========": "",
   "CURATION_GITHUB_REPO": "sample-curated-data",
-  "CURATION_GITHUB_TOKEN": "<GitHub token here>"
+  "CURATION_GITHUB_TOKEN": "<GitHub token here>",
+
+  "========== Heapstats Logging settings (OPTIONAL) ==========": "",
+  "LOG_NODE_HEAPSTATS": "<true|false>",
+  "LOG_NODE_HEAPSTATS_INTERVAL_MS": "<time_in_milliseconds (e.g. '30000')>"
 }

package.json

@@ -12,7 +12,7 @@
     "prettier:check": "prettier . --check",
     "prettier:write": "prettier . --write",
     "dev": "nodemon ./bin/www",
-    "start": "node ./bin/www",
+    "start": "node --max-old-space-size=8192 ./bin/www",
     "postinstall": "patch-package"
   },
   "license": "MIT",