Improve job health messages

ChainSafe · Jan 30, 2024 · b60c1a8 · b60c1a8
1 parent 736e5c5
commit b60c1a8
Show file tree

Hide file tree

Showing 4 changed files with 20 additions and 8 deletions.
diff --git a/packages/cli/test/utils/simulation/constants.ts b/packages/cli/test/utils/simulation/constants.ts
@@ -23,3 +23,4 @@ export const LODESTAR_BINARY_PATH = `${__dirname}/../../../bin/lodestar.js`;
 export const MOCK_ETH1_GENESIS_HASH = "0xfbfbfbfbfbfbfbfbfbfbfbfbfbfbfbfbfbfbfbfbfbfbfbfbfbfbfbfbfbfbfbfb";
 export const SHARED_JWT_SECRET = "0xdc6457099f127cf0bac78de8b297df04951281909db4f58b43def7c7151e765d";
 export const SHARED_VALIDATOR_PASSWORD = "passwrod";
+export const JOB_HEALTH_TIMEOUT = 10000;
diff --git a/packages/cli/test/utils/simulation/runner/ChildProcessRunner.ts b/packages/cli/test/utils/simulation/runner/ChildProcessRunner.ts
@@ -7,6 +7,7 @@ import {
   ChildProcessResolve,
 } from "@lodestar/test-utils";
 import {Job, JobOptions, RunnerEnv, RunnerType} from "../interfaces.js";
+import {JOB_HEALTH_TIMEOUT} from "../constants.js";
 
 export class ChildProcessRunner implements RunnerEnv<RunnerType.ChildProcess> {
   type = RunnerType.ChildProcess as const;
@@ -23,7 +24,7 @@ export class ChildProcessRunner implements RunnerEnv<RunnerType.ChildProcess> {
     const health = jobOption.health;
 
     if (health) {
-      spawnOpts.healthTimeoutMs = 30000;
+      spawnOpts.healthTimeoutMs = JOB_HEALTH_TIMEOUT;
       spawnOpts.health = async (): Promise<ChildProcessHealthStatus> =>
         health()
           .then((status) => {

diff --git a/packages/cli/test/utils/simulation/runner/DockerRunner.ts b/packages/cli/test/utils/simulation/runner/DockerRunner.ts
@@ -10,6 +10,7 @@ import {
   ChildProcessResolve,
 } from "@lodestar/test-utils";
 import {Job, JobOptions, RunnerEnv, RunnerType} from "../interfaces.js";
+import {JOB_HEALTH_TIMEOUT} from "../constants.js";
 
 const dockerNetworkIpRange = "192.168.0";
 const dockerNetworkName = "sim-env-net";
@@ -98,7 +99,7 @@ export class DockerRunner implements RunnerEnv<RunnerType.Docker> {
     const health = jobOption.health;
 
     if (health) {
-      spawnOpts.healthTimeoutMs = 30000;
+      spawnOpts.healthTimeoutMs = JOB_HEALTH_TIMEOUT;
       spawnOpts.health = async (): Promise<ChildProcessHealthStatus> =>
         health()
           .then((status) => {

diff --git a/packages/test-utils/src/childProcess.ts b/packages/test-utils/src/childProcess.ts
@@ -198,10 +198,15 @@ const defaultStartOpts = {
   pipeOnlyError: false,
   logPrefix: "",
   healthCheckIntervalMs: 1000,
-  logHealthChecksAfterMs: 2000,
+  logHealthChecksAfterMs: 3000,
   resolveOn: ChildProcessResolve.Immediate,
 };
 
+function formatTime(timeMs: number): string {
+  const d = new Date(timeMs);
+  return `${d.getHours()}:${d.getMinutes()}:${d.getSeconds()}.${d.getMilliseconds()}`;
+}
+
 /**
  * Spawn child process and return it
  *
@@ -306,7 +311,9 @@ export async function spawnChildProcess(
                 const timeSinceHealthCheckStart = Date.now() - startHealthCheckMs;
                 if (timeSinceHealthCheckStart > logHealthChecksAfterMs) {
                   console.log(
-                    `Health check unsuccessful. logPrefix=${logPrefix} pid=${proc.pid}  timeSinceHealthCheckStart=${timeSinceHealthCheckStart}`
+                    `Health check unsuccessful. logPrefix=${logPrefix} pid=${
+                      proc.pid
+                    } timeSinceHealthCheckStart=${formatTime(timeSinceHealthCheckStart)}`
                   );
                 }
               }
@@ -322,7 +329,9 @@ export async function spawnChildProcess(
           if (intervalId !== undefined) {
             reject(
               new Error(
-                `Health check timeout. logPrefix=${logPrefix} pid=${proc.pid}  healthTimeoutMs=${healthTimeoutMs}`
+                `Health check timeout. logPrefix=${logPrefix} pid=${proc.pid} healthTimeout=${formatTime(
+                  healthTimeoutMs ?? 0
+                )}`
               )
             );
           }
@@ -336,9 +345,9 @@ export async function spawnChildProcess(
 
           reject(
             new Error(
-              `process exited before healthy. logPrefix=${logPrefix} pid=${
-                proc.pid
-              } healthTimeoutMs=${healthTimeoutMs} code=${code} command="${command} ${args.join(" ")}"`
+              `process exited before healthy. logPrefix=${logPrefix} pid=${proc.pid} healthTimeout=${formatTime(
+                healthTimeoutMs ?? 0
+              )} code=${code} command="${command} ${args.join(" ")}"`
             )
           );
         });