Runner timer (#1009)

* Runner timer * consider gh idle correction * avoid gh infinite * remove unused * remove redundant * safer check * restore unlimited time * edge case
iterative · May 25, 2022 · d8f27e1 · d8f27e1
1 parent b9ac0df
commit d8f27e1
Showing 1 changed file with 42 additions and 32 deletions.
diff --git a/bin/cml/runner.js b/bin/cml/runner.js
@@ -14,6 +14,7 @@ let RUNNER;
 let RUNNER_ID;
 let RUNNER_JOBS_RUNNING = [];
 let RUNNER_SHUTTING_DOWN = false;
+let RUNNER_TIMER = 0;
 const GH_5_MIN_TIMEOUT = (72 * 60 - 5) * 60 * 1000;
 
 const shutdown = async (opts) => {
@@ -229,6 +230,7 @@ const runLocal = async (opts) => {
       RUNNER_JOBS_RUNNING = RUNNER_JOBS_RUNNING.filter(
         (job) => job.id !== jobId
       );
+      RUNNER_TIMER = 0;
 
       if (single && cml.driver === 'bitbucket') {
         await shutdown({ ...opts, reason: 'single job' });
@@ -257,47 +259,55 @@ const runLocal = async (opts) => {
   RUNNER = proc;
   ({ id: RUNNER_ID } = await cml.runnerByName({ name }));
 
-  if (parseInt(idleTimeout) > 0) {
+  if (idleTimeout > 0) {
     const watcher = setInterval(async () => {
       let idle = RUNNER_JOBS_RUNNING.length === 0;
 
-      try {
-        if (cml.driver === 'github') {
-          const job = await cml.runnerJob({ runnerId: RUNNER_ID });
-
-          if (!job && !idle) {
-            winston.error(
-              `Runner is idle as per the GitHub API but busy as per CML internal state. Resetting jobs. Retrying in ${idleTimeout} seconds...`
-            );
-            winston.warn(`CML GitHub driver response: ${JSON.stringify(job)}`);
-            winston.warn(
-              `CML internal state: ${JSON.stringify(RUNNER_JOBS_RUNNING)}`
-            );
-
-            RUNNER_JOBS_RUNNING = [];
+      if (RUNNER_TIMER >= idleTimeout) {
+        try {
+          if (cml.driver === 'github') {
+            const job = await cml.runnerJob({ runnerId: RUNNER_ID });
+
+            if (!job && !idle) {
+              winston.error(
+                `Runner is idle as per the GitHub API but busy as per CML internal state. Resetting jobs. Retrying in ${idleTimeout} seconds...`
+              );
+              winston.warn(
+                `CML GitHub driver response: ${JSON.stringify(job)}`
+              );
+              winston.warn(
+                `CML internal state: ${JSON.stringify(RUNNER_JOBS_RUNNING)}`
+              );
+
+              RUNNER_JOBS_RUNNING = [];
+            }
+
+            if (job && idle) {
+              winston.error(
+                `Runner is busy as per the GitHub API but idle as per CML internal state. Retrying in ${idleTimeout} seconds...`
+              );
+
+              idle = false;
+            }
           }
+        } catch (err) {
+          winston.error(
+            `Error connecting the SCM: ${err.message}. Will try again in ${idleTimeout} secs`
+          );
 
-          if (job && idle) {
-            winston.error(
-              `Runner is busy as per the GitHub API but idle as per CML internal state. Retrying in ${idleTimeout} seconds...`
-            );
-
-            idle = false;
-          }
+          idle = false;
         }
-      } catch (err) {
-        winston.error(
-          `Error connecting the SCM: ${err.message}. Will try again in ${idleTimeout} secs`
-        );
 
-        idle = false;
+        if (idle) {
+          shutdown({ ...opts, reason: `timeout:${idleTimeout}` });
+          clearInterval(watcher);
+        } else {
+          RUNNER_TIMER = 0;
+        }
       }
 
-      if (idle) {
-        shutdown({ ...opts, reason: `timeout:${idleTimeout}` });
-        clearInterval(watcher);
-      }
-    }, idleTimeout * 1000);
+      RUNNER_TIMER++;
+    }, 1000);
   }
 
   if (!noRetry) {