Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

feat: get interactive shell into jobs running in a container #1460

Open
wants to merge 11 commits into
base: master
Choose a base branch
from
Prev Previous commit
Next Next commit
feat: add debug option to get a shell into failed jobs
  • Loading branch information
KoiFresh committed Dec 23, 2024
commit 9e4867c9401968e0fc50016ba62e215804327e08
4 changes: 4 additions & 0 deletions src/argv.ts
Original file line number Diff line number Diff line change
@@ -145,6 +145,10 @@ export class Argv {
return this.map.get("interactiveJobs") ?? [];
}

get debug (): boolean {
return this.map.get("debug") ?? false;
}

get variable (): {[key: string]: string} {
const val = this.map.get("variable");
const variables: {[key: string]: string} = {};
6 changes: 6 additions & 0 deletions src/index.ts
Original file line number Diff line number Diff line change
@@ -106,6 +106,12 @@ process.on("SIGUSR2", async () => await cleanupJobResources(jobs));
description: "Select jobs to run interactively",
requiresArg: false,
})
.option("debug", {
type: "boolean",
alias: "d",
description: "Open an interactive shell for failed jobs",
requiresArg: false,
})
.option("cwd", {
type: "string",
description: "Path to a current working directory",
55 changes: 55 additions & 0 deletions src/job.ts
Original file line number Diff line number Diff line change
@@ -594,6 +594,12 @@ export class Job {
await this.execPreScripts(expanded);
if (this._prescriptsExitCode == null) throw Error("this._prescriptsExitCode must be defined!");

if (this.argv.debug && this.jobStatus === "failed") {
// To successfully finish the job, someone has to call debug();
clearInterval(this._longRunningSilentTimeout);
return;
}

if (!this.interactive) {
await this.execAfterScripts(expanded);
}
@@ -612,6 +618,51 @@ export class Job {
this.cleanupResources();
}

async debug (): Promise<void> {
// stop still running message in debug mode
clearInterval(this._longRunningSilentTimeout);
this.writeStreams.stdout(chalk`${this.formattedJobName} {magentaBright starting debug shell}\n`);

const cwd = this.argv.cwd;
const expanded = Utils.unscape$$Variables(Utils.expandVariables({...this._variables, ...this._dotenvVariables}));
const imageName = this.imageName(expanded);

if (this._containerId) {
await execa(`${this.argv.containerExecutable} start ${this._containerId}`, {
shell: "bash",
env: imageName ? process.env : expanded,
});
}

try {
await execa(this._containerId ? `DOCKER_CLI_HINTS=false ${this.argv.containerExecutable} exec -it ${this._containerId} bash` : "bash", {
cwd,
shell: "bash",
stdio: "inherit",
env: imageName ? process.env : expanded,
});
} catch (e) {
// nothing to do, failing is allowed
}

if (!this.interactive) {
await this.execAfterScripts(expanded);
}

this._running = false;
this._endTime = this._endTime ?? process.hrtime(this._startTime);
this.printFinishedString();

await this.copyCacheOut(this.writeStreams, expanded);
await this.copyArtifactsOut(this.writeStreams, expanded);

if (this.jobData["coverage"]) {
this._coveragePercent = await Utils.getCoveragePercent(this.argv.cwd, this.argv.stateDir, this.jobData["coverage"], this.safeJobName);
}

this.cleanupResources();
}

async cleanupResources () {
clearTimeout(this._longRunningSilentTimeout);

@@ -894,6 +945,10 @@ export class Job {
await Utils.spawn([this.argv.containerExecutable, "cp", `${stateDir}/scripts/${safeJobName}_${this.jobId}`, `${this._containerId}:/gcl-cmd`], cwd);
}

if (this.interactive) {
this.writeStreams.stdout(chalk`${this.formattedJobName} {magentaBright starting interactive shell}\n`);
}

const cp = execa(this._containerId ? `${this.argv.containerExecutable} start --attach -i ${this._containerId}` : "bash", {
cwd,
shell: "bash",
32 changes: 28 additions & 4 deletions src/multi-job-runner.ts
Original file line number Diff line number Diff line change
@@ -13,12 +13,14 @@ export type MultiJobRunnerOptions = {
*/
export async function runMultipleJobs (jobs: Job[], options: MultiJobRunnerOptions): Promise<void> {
const activeJobsById: Map<number, Promise<void>> = new Map();
const jobsToDebug: Job[] = [];
const exceptions: unknown[] = [];

for (const job of jobs) {
await debugJobs(jobsToDebug);

if (job.interactive) {
await Promise.all(activeJobsById.values());
await job.start();
continue;
}

if (activeJobsById.size >= options.concurrency) {
@@ -29,10 +31,32 @@ export async function runMultipleJobs (jobs: Job[], options: MultiJobRunnerOptio
activeJobsById.set(job.jobId, execution);
execution.then(() => {
activeJobsById.delete(job.jobId);
});
continue;
if (job.argv.debug && job.jobStatus === "failed") {
jobsToDebug.push(job);
}
}).catch((e) => exceptions.push(e));

if (job.interactive) {
await execution;
}
}

await Promise.all(activeJobsById.values());
await throwExceptions(exceptions);
await debugJobs(jobsToDebug);
}

async function throwExceptions (exceptions: unknown[]): Promise<void> {
if (exceptions.length > 0) {
throw exceptions[0];
}
}

async function debugJobs (jobs: Job[]): Promise<void> {
while (jobs.length > 0) {
const job = jobs.shift();
if (job) {
await job.debug();
}
}
}