From d417038b90f1681740b76937c018fa085be33946 Mon Sep 17 00:00:00 2001 From: Tsiry Sandratraina Date: Thu, 30 May 2024 13:42:54 +0000 Subject: [PATCH 01/18] update README.md --- README.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index ee3c7dd..81a8c67 100644 --- a/README.md +++ b/README.md @@ -86,8 +86,8 @@ Requirements: **Latest (CLI):** -- `Mac`: arm64: [fluentci_v0.14.4_aarch64-apple-darwin.tar.gz](https://github.com/fluentci-io/fluentci/releases/download/v0.14.4/fluentci_v0.14.4_aarch64-apple-darwin.tar.gz) intel: [fluentci_v0.14.4_x86_64-apple-darwin.tar.gz](https://github.com/fluentci-io/fluentci/releases/download/v0.14.4/fluentci_v0.14.4_x86_64-apple-darwin.tar.gz) -- `Linux`: intel: [fluentci_v0.14.4_x86_64-unknown-linux-gnu.tar.gz](https://github.com/fluentci-io/fluentci/releases/download/v0.14.4/fluentci_v0.14.4_x86_64-unknown-linux-gnu.tar.gz) arm64: [fluentci_v0.14.4_aarch64-unknown-linux-gnu.tar.gz](https://github.com/fluentci-io/fluentci/releases/download/v0.14.4/fluentci_v0.14.4_aarch64-unknown-linux-gnu.tar.gz) +- `Mac`: arm64: [fluentci_v0.14.6_aarch64-apple-darwin.tar.gz](https://github.com/fluentci-io/fluentci/releases/download/v0.14.6/fluentci_v0.14.6_aarch64-apple-darwin.tar.gz) intel: [fluentci_v0.14.6_x86_64-apple-darwin.tar.gz](https://github.com/fluentci-io/fluentci/releases/download/v0.14.6/fluentci_v0.14.6_x86_64-apple-darwin.tar.gz) +- `Linux`: intel: [fluentci_v0.14.6_x86_64-unknown-linux-gnu.tar.gz](https://github.com/fluentci-io/fluentci/releases/download/v0.14.6/fluentci_v0.14.6_x86_64-unknown-linux-gnu.tar.gz) arm64: [fluentci_v0.14.6_aarch64-unknown-linux-gnu.tar.gz](https://github.com/fluentci-io/fluentci/releases/download/v0.14.6/fluentci_v0.14.6_aarch64-unknown-linux-gnu.tar.gz) ## ✨ Quick Start From ed1f2ea3345e8321c2096b32b1d5fcd0faa0fd00 Mon Sep 17 00:00:00 2001 From: Tsiry Sandratraina Date: Thu, 6 Jun 2024 10:47:18 +0000 Subject: [PATCH 02/18] setup remote exec add log add log --- src/cmd/agent.ts | 143 +++++++++++++++--- src/cmd/run.ts | 8 +- .../graphql/resolvers/action/mutations.ts | 2 +- src/types.ts | 53 +++++++ 4 files changed, 184 insertions(+), 22 deletions(-) diff --git a/src/cmd/agent.ts b/src/cmd/agent.ts index dc65577..30d14f1 100644 --- a/src/cmd/agent.ts +++ b/src/cmd/agent.ts @@ -7,6 +7,7 @@ import { BlobReader, Table, dayjs, + createId, } from "../../deps.ts"; import { FLUENTCI_WS_URL, @@ -22,7 +23,7 @@ import { isLogged, } from "../utils.ts"; import { hostname, release, cpus, arch, totalmem, platform } from "node:os"; -import { Agent } from "../types.ts"; +import { Action, Agent, Log, Run } from "../types.ts"; // import O from "https://esm.sh/v133/mimic-fn@4.0.0/denonext/mimic-fn.mjs"; async function startAgent() { @@ -72,8 +73,10 @@ async function startAgent() { try { logger.info(`Message from server ${event.data}`); const data = JSON.parse(event.data); - const { action, src, query, buildId } = JSON.parse(data.event); - const { jobs, pipeline } = JSON.parse(query); + const { action, src, query, buildId, actions, run } = JSON.parse( + data.event + ); + const { jobs, pipeline, wasm } = JSON.parse(query); if (action === "build") { const project_id = src.split("/")[0]; @@ -91,7 +94,10 @@ async function startAgent() { sha256, pipeline, jobs, - buildId + buildId, + wasm, + actions, + run ); return; } @@ -110,7 +116,10 @@ async function startAgent() { sha256, pipeline, jobs, - buildId + buildId, + wasm, + actions, + run ); } } catch (e) { @@ -156,24 +165,46 @@ async function spawnFluentCI( sha256: string, pipeline: string, jobs: [string, ...Array], - clientId: string + clientId: string, + wasm: boolean = false, + actions: Action[] = [], + run?: Run ) { + if (actions.length > 0) { + await executeActions(actions, project_id, sha256, run!); + return; + } const accessToken = await getAccessToken(); const headers = { Authorization: `Bearer ${accessToken}`, }; - const command = new Deno.Command("dagger", { - args: ["--progress", "plain", "run", "fluentci", "run", pipeline, ...jobs], - cwd: `${dir("home")}/.fluentci/builds/${project_id}/${sha256}`, - stdout: "piped", - stderr: "piped", - env: { - _EXPERIMENTAL_DAGGER_CLOUD_URL: `https://events.fluentci.io?id=${ - "build-" + clientId - }&project_id=${project_id}`, - DAGGER_CLOUD_TOKEN: Deno.env.get("DAGGER_CLOUD_TOKEN") || "123", - }, - }); + const command = wasm + ? new Deno.Command("fluentci", { + args: [ + "run", + "--wasm", + pipeline, + ...jobs.filter((x) => x !== "--remote-exec"), + ], + cwd: `${dir("home")}/.fluentci/builds/${project_id}/${sha256}`, + stdout: "piped", + stderr: "piped", + }) + : new Deno.Command("dagger", { + args: [ + "--progress", + "plain", + "run", + "fluentci", + "run", + pipeline, + ...jobs, + ], + cwd: `${dir("home")}/.fluentci/builds/${project_id}/${sha256}`, + stdout: "piped", + stderr: "piped", + }); + console.log(">> command", command); const process = command.spawn(); let logs = ""; const writable = new WritableStream({ @@ -206,6 +237,82 @@ async function spawnFluentCI( ]).catch((e) => logger.error(e.message)); } +async function executeActions( + actions: Action[], + project_id: string, + sha256: string, + run: Run +) { + let currentActionIndex = 0; + const runStart = dayjs(); + const jobs = [...run.jobs]; + + for (const action of actions) { + if (!action.enabled) { + currentActionIndex += 1; + continue; + } + + for (const cmd of action.commands.split("\n")) { + const result = await spawn( + `fluentci run ${action.useWasm ? "--wasm" : ""} ${ + action.plugin + } ${cmd}`, + `${dir("home")}/.fluentci/builds/${project_id}/${sha256}`, + jobs[currentActionIndex].id + ); + } + } +} + +async function spawn(cmd: string, cwd = Deno.cwd(), jobId?: string) { + const logs: Log[] = []; + const child = new Deno.Command("bash", { + args: ["-c", cmd], + stdout: "piped", + stderr: "piped", + cwd, + }).spawn(); + + const writableStdoutStream = new WritableStream({ + write: (chunk) => { + const text = new TextDecoder().decode(chunk); + console.log(text); + const log: Log = { + id: createId(), + jobId: jobId!, + message: text, + createdAt: new Date().toISOString(), + }; + logs.push(log); + // TODO: send logs to the client + }, + }); + + const writableStderrStream = new WritableStream({ + write: (chunk) => { + const text = new TextDecoder().decode(chunk); + console.log(text); + const log: Log = { + id: createId(), + jobId: jobId!, + message: text, + createdAt: new Date().toISOString(), + }; + logs.push(log); + // TODO: send logs to the client + }, + }); + + const [_stdout, _stderr, { code }] = await Promise.all([ + child.stdout.pipeTo(writableStdoutStream), + child.stderr.pipeTo(writableStderrStream), + child.status, + ]); + + return { logs, code }; +} + async function getWebSocketUuid(agentId: string) { const accessToken = await getAccessToken(); const daggerVersion = await getDaggerVersion(); diff --git a/src/cmd/run.ts b/src/cmd/run.ts index 7a1f71e..969bd1c 100644 --- a/src/cmd/run.ts +++ b/src/cmd/run.ts @@ -48,7 +48,7 @@ async function run( export: true, }); - if (options.wasm) { + if (options.wasm && !options.remoteExec) { Deno.env.set("WASM_ENABLED", "1"); await runWasmPlugin(pipeline, jobs); Deno.exit(0); @@ -68,7 +68,7 @@ async function run( if (Deno.env.get("FLUENTCI_PROJECT_ID")) { if (options.remoteExec) { - await runPipelineRemotely(pipeline, jobs); + await runPipelineRemotely(pipeline, jobs, options); return; } await detect("."); @@ -246,7 +246,7 @@ async function run( } if (Deno.env.get("FLUENTCI_PROJECT_ID")) { - await runPipelineRemotely(pipeline, jobs, denoModule); + await runPipelineRemotely(pipeline, jobs, options, denoModule); return; } @@ -289,6 +289,7 @@ const spawnCommand = async (command: Deno.Command) => { const runPipelineRemotely = async ( pipeline: string, jobs: [string, ...Array] | string[], + options: Record = {}, denoModule?: string[] ) => { if (!(await isLogged())) { @@ -345,6 +346,7 @@ const runPipelineRemotely = async ( const query = JSON.stringify({ pipeline, jobs, + wasm: options.wasm, denoModule, }); diff --git a/src/server/graphql/resolvers/action/mutations.ts b/src/server/graphql/resolvers/action/mutations.ts index a1cbfce..c55316e 100644 --- a/src/server/graphql/resolvers/action/mutations.ts +++ b/src/server/graphql/resolvers/action/mutations.ts @@ -13,7 +13,7 @@ export async function saveActions( args.actions.map((x) => ({ ...x, id: createId(), - plugin: x.plugin.replace("_pipeline", ""), + plugin: x.useWasm ? x.plugin.replace("_pipeline", "") : x.plugin, })) ); return ctx.kv.actions.get(args.projectId.toString()); diff --git a/src/types.ts b/src/types.ts index 40d1d8e..ebb3253 100644 --- a/src/types.ts +++ b/src/types.ts @@ -43,6 +43,51 @@ export const AgentSchema = z.object({ startedAt: z.string(), }); +export const ActionSchema = z.object({ + id: z.string().optional().nullable(), + name: z.string(), + commands: z.string(), + enabled: z.boolean(), + plugin: z.string(), + useWasm: z.boolean(), + logo: z.string().optional().nullable(), + githubUrl: z.string().optional().nullable(), +}); + +export const LogSchema = z.object({ + id: z.string(), + jobId: z.string(), + message: z.string(), + createdAt: z.string(), +}); + +export const JobSchema = z.object({ + id: z.string(), + name: z.string(), + status: z.string(), + createdAt: z.string(), + startedAt: z.string().optional().nullable(), + duration: z.number().optional().nullable(), + logs: z.array(LogSchema).optional().nullable(), +}); + +export const RunSchema = z.object({ + id: z.string(), + name: z.string(), + project: z.string(), + projectId: z.string(), + title: z.string(), + message: z.string().optional().nullable(), + commit: z.string().optional().nullable(), + author: z.string().optional().nullable(), + branch: z.string().optional().nullable(), + duration: z.number(), + date: z.string(), + jobs: z.array(JobSchema), + cursor: z.string().optional().nullable(), + status: z.string().optional().nullable(), +}); + export type Pipeline = z.infer; export type Label = z.infer; @@ -50,3 +95,11 @@ export type Label = z.infer; export type LogEvent = z.infer; export type Agent = z.infer; + +export type Action = z.infer; + +export type Log = z.infer; + +export type Run = z.infer; + +export type Job = z.infer; From 2562ac46fe8eac6d2d3e10fcad2e02dcda1c02e0 Mon Sep 17 00:00:00 2001 From: Tsiry Sandratraina Date: Thu, 6 Jun 2024 12:24:24 +0000 Subject: [PATCH 03/18] stream stdout and stderr of a remote exec --- src/cmd/agent.ts | 22 ++++++++++++++++++---- 1 file changed, 18 insertions(+), 4 deletions(-) diff --git a/src/cmd/agent.ts b/src/cmd/agent.ts index 30d14f1..6a4fe34 100644 --- a/src/cmd/agent.ts +++ b/src/cmd/agent.ts @@ -204,10 +204,21 @@ async function spawnFluentCI( stdout: "piped", stderr: "piped", }); - console.log(">> command", command); const process = command.spawn(); let logs = ""; - const writable = new WritableStream({ + const writableStdoutStream = new WritableStream({ + write: (chunk) => { + const text = new TextDecoder().decode(chunk); + logger.info(text); + logs = logs.concat(text); + fetch(`${FLUENTCI_EVENTS_URL}?client_id=${clientId}`, { + method: "POST", + body: text, + headers, + }).catch((e) => logger.error(e.message)); + }, + }); + const writableStderrStream = new WritableStream({ write: (chunk) => { const text = new TextDecoder().decode(chunk); logger.info(text); @@ -220,8 +231,11 @@ async function spawnFluentCI( }, }); - await process.stderr?.pipeTo(writable); - const { code } = await process.status; + const [_stdout, _stderr, { code }] = await Promise.all([ + process.stdout?.pipeTo(writableStdoutStream), + process.stderr?.pipeTo(writableStderrStream), + process.status, + ]); Promise.all([ fetch(`${FLUENTCI_EVENTS_URL}?client_id=${clientId}`, { From 835f60be059ed9d3fdfe910308986dbadc6d75f5 Mon Sep 17 00:00:00 2001 From: Tsiry Sandratraina Date: Thu, 6 Jun 2024 15:45:09 +0000 Subject: [PATCH 04/18] work in progress --- src/cmd/agent.ts | 27 +++++++++++++++++++++++++++ 1 file changed, 27 insertions(+) diff --git a/src/cmd/agent.ts b/src/cmd/agent.ts index 6a4fe34..6abc281 100644 --- a/src/cmd/agent.ts +++ b/src/cmd/agent.ts @@ -261,12 +261,19 @@ async function executeActions( const runStart = dayjs(); const jobs = [...run.jobs]; + // TODO: send update run status "RUNNING" + date + for (const action of actions) { if (!action.enabled) { currentActionIndex += 1; continue; } + const start = dayjs(); + const logs: Log[] = []; + + // TODO: send update job status "RUNNING" + date + for (const cmd of action.commands.split("\n")) { const result = await spawn( `fluentci run ${action.useWasm ? "--wasm" : ""} ${ @@ -275,8 +282,28 @@ async function executeActions( `${dir("home")}/.fluentci/builds/${project_id}/${sha256}`, jobs[currentActionIndex].id ); + + logs.push(...result.logs); + + if (result.code !== 0) { + dayjs().diff(start, "milliseconds"); + // TODO: send update job status "FAILURE" + duration + logs + // TODO: send update run status "FAILURE" + duration + // TODO: send update project stats + return; + } } + + dayjs().diff(start, "milliseconds"); + // TODO: send update job status "SUCCESS" + duration + logs + // TODO: send update run jobs + currentActionIndex += 1; } + + // deno-lint-ignore no-unused-vars + const duration = dayjs().diff(runStart, "milliseconds"); + // TODO: update run status "SUCCESS" + duration + // TODO: update project stats } async function spawn(cmd: string, cwd = Deno.cwd(), jobId?: string) { From 98870f8ea74f6b4d0e943e0df82d7ae1da713bba Mon Sep 17 00:00:00 2001 From: Tsiry Sandratraina Date: Sat, 8 Jun 2024 16:07:09 +0000 Subject: [PATCH 05/18] [remote-exec] run actions --- src/cmd/agent.ts | 178 +++++++++++++++++++++++++++++++++++++------- src/cmd/run.ts | 15 ++-- src/detect.ts | 77 ++++++++++++++++++- src/server/icons.ts | 2 +- 4 files changed, 232 insertions(+), 40 deletions(-) diff --git a/src/cmd/agent.ts b/src/cmd/agent.ts index 6abc281..52f524a 100644 --- a/src/cmd/agent.ts +++ b/src/cmd/agent.ts @@ -24,7 +24,11 @@ import { } from "../utils.ts"; import { hostname, release, cpus, arch, totalmem, platform } from "node:os"; import { Action, Agent, Log, Run } from "../types.ts"; -// import O from "https://esm.sh/v133/mimic-fn@4.0.0/denonext/mimic-fn.mjs"; + +const accessToken = await getAccessToken(); +const headers = { + Authorization: `Bearer ${accessToken}`, +}; async function startAgent() { console.log(` @@ -73,7 +77,7 @@ async function startAgent() { try { logger.info(`Message from server ${event.data}`); const data = JSON.parse(event.data); - const { action, src, query, buildId, actions, run } = JSON.parse( + const { action, src, query, runId, actions, run } = JSON.parse( data.event ); const { jobs, pipeline, wasm } = JSON.parse(query); @@ -94,7 +98,7 @@ async function startAgent() { sha256, pipeline, jobs, - buildId, + runId, wasm, actions, run @@ -116,7 +120,7 @@ async function startAgent() { sha256, pipeline, jobs, - buildId, + runId, wasm, actions, run @@ -171,13 +175,10 @@ async function spawnFluentCI( run?: Run ) { if (actions.length > 0) { - await executeActions(actions, project_id, sha256, run!); + await executeActions(actions, project_id, sha256, run!, logger, clientId); return; } - const accessToken = await getAccessToken(); - const headers = { - Authorization: `Bearer ${accessToken}`, - }; + const command = wasm ? new Deno.Command("fluentci", { args: [ @@ -198,7 +199,7 @@ async function spawnFluentCI( "fluentci", "run", pipeline, - ...jobs, + ...jobs.filter((x) => x !== "--remote-exec"), ], cwd: `${dir("home")}/.fluentci/builds/${project_id}/${sha256}`, stdout: "piped", @@ -255,13 +256,20 @@ async function executeActions( actions: Action[], project_id: string, sha256: string, - run: Run + run: Run, + logger: Logger, + clientId: string ) { let currentActionIndex = 0; const runStart = dayjs(); - const jobs = [...run.jobs]; + let jobs = [...run.jobs]; - // TODO: send update run status "RUNNING" + date + // send update run status "RUNNING" + date + sendEvent(clientId, "run", { + ...run, + status: "RUNNING", + date: new Date().toISOString(), + }).catch((e) => logger.error(e.message)); for (const action of actions) { if (!action.enabled) { @@ -272,7 +280,18 @@ async function executeActions( const start = dayjs(); const logs: Log[] = []; - // TODO: send update job status "RUNNING" + date + jobs = jobs.map((job, j) => ({ + ...job, + startedAt: currentActionIndex === j ? start.toISOString() : job.startedAt, + status: currentActionIndex === j ? "RUNNING" : job.status, + })); + + // send update job status "RUNNING" + date + sendEvent(clientId, "job", { + ...jobs[currentActionIndex], + startedAt: start.toISOString(), + status: "RUNNING", + }).catch((e) => logger.error(e.message)); for (const cmd of action.commands.split("\n")) { const result = await spawn( @@ -280,33 +299,102 @@ async function executeActions( action.plugin } ${cmd}`, `${dir("home")}/.fluentci/builds/${project_id}/${sha256}`, - jobs[currentActionIndex].id + jobs[currentActionIndex].id, + logger, + clientId ); logs.push(...result.logs); if (result.code !== 0) { - dayjs().diff(start, "milliseconds"); - // TODO: send update job status "FAILURE" + duration + logs - // TODO: send update run status "FAILURE" + duration - // TODO: send update project stats + // send update job status "FAILURE" + duration + logs + sendEvent(clientId, "job", { + ...jobs[currentActionIndex], + status: "FAILURE", + duration: dayjs().diff(start, "milliseconds"), + logs: [...(jobs[currentActionIndex].logs || []), ...logs], + }).catch((e) => logger.error(e.message)); + + jobs = jobs.map((job, j) => ({ + ...job, + status: currentActionIndex === j ? "FAILURE" : job.status, + duration: + currentActionIndex === j + ? dayjs().diff(start, "milliseconds") + : job.duration, + logs: + currentActionIndex === j + ? [...(job.logs || []), ...result.logs] + : job.logs, + })); + const duration = dayjs().diff(runStart, "milliseconds"); + // send update run status "FAILURE" + duration + sendEvent(clientId, "run", { + ...run!, + jobs, + status: "FAILURE", + duration, + }).catch((e) => logger.error(e.message)); + + // send update project stats + sendEvent(clientId, "update-stats", {}).catch((e) => + logger.error(e.message) + ); + + fetch(`${FLUENTCI_EVENTS_URL}?client_id=${clientId}`, { + method: "POST", + body: `fluentci_exit=1`, + headers, + }).catch((e) => logger.error(e.message)); return; } } + jobs = jobs.map((job, j) => ({ + ...job, + status: currentActionIndex === j ? "SUCCESS" : job.status, + duration: + currentActionIndex === j + ? dayjs().diff(start, "milliseconds") + : job.duration, + logs: + currentActionIndex === j ? [...(job.logs || []), ...logs] : job.logs, + })); + dayjs().diff(start, "milliseconds"); - // TODO: send update job status "SUCCESS" + duration + logs - // TODO: send update run jobs + // send update run jobs + sendEvent(clientId, "run", { + ...run!, + jobs, + }); currentActionIndex += 1; } - // deno-lint-ignore no-unused-vars + // update run status "SUCCESS" + duration const duration = dayjs().diff(runStart, "milliseconds"); - // TODO: update run status "SUCCESS" + duration - // TODO: update project stats + sendEvent(clientId, "run", { + ...run!, + status: "SUCCESS", + duration, + }).catch((e) => logger.error(e.message)); + + // update project stats + sendEvent(clientId, "update-stats", {}).catch((e) => logger.error(e.message)); + + fetch(`${FLUENTCI_EVENTS_URL}?client_id=${clientId}`, { + method: "POST", + body: `fluentci_exit=0`, + headers, + }).catch((e) => logger.error(e.message)); } -async function spawn(cmd: string, cwd = Deno.cwd(), jobId?: string) { +async function spawn( + cmd: string, + cwd = Deno.cwd(), + jobId: string, + logger: Logger, + clientId: string +) { const logs: Log[] = []; const child = new Deno.Command("bash", { args: ["-c", cmd], @@ -319,14 +407,24 @@ async function spawn(cmd: string, cwd = Deno.cwd(), jobId?: string) { write: (chunk) => { const text = new TextDecoder().decode(chunk); console.log(text); + + fetch(`${FLUENTCI_EVENTS_URL}?client_id=${clientId}`, { + method: "POST", + body: text, + headers, + }).catch((e) => logger.error(e.message)); + const log: Log = { id: createId(), - jobId: jobId!, + jobId, message: text, createdAt: new Date().toISOString(), }; logs.push(log); - // TODO: send logs to the client + // send logs to the client + sendEvent(clientId, "logs", { text, jobId }).catch((e) => + logger.error(e.message) + ); }, }); @@ -334,6 +432,13 @@ async function spawn(cmd: string, cwd = Deno.cwd(), jobId?: string) { write: (chunk) => { const text = new TextDecoder().decode(chunk); console.log(text); + + fetch(`${FLUENTCI_EVENTS_URL}?client_id=${clientId}`, { + method: "POST", + body: text, + headers, + }).catch((e) => logger.error(e.message)); + const log: Log = { id: createId(), jobId: jobId!, @@ -341,7 +446,10 @@ async function spawn(cmd: string, cwd = Deno.cwd(), jobId?: string) { createdAt: new Date().toISOString(), }; logs.push(log); - // TODO: send logs to the client + // send logs to the client + sendEvent(clientId, "logs", { text, jobId }).catch((e) => + logger.error(e.message) + ); }, }); @@ -419,4 +527,18 @@ export async function listAgents() { table.render(); } +// deno-lint-ignore no-explicit-any +const sendEvent = (clientId: string, channel: string, data: any) => + fetch(`${FLUENTCI_EVENTS_URL}?client_id=${clientId}`, { + method: "POST", + body: JSON.stringify({ + channel, + data, + }), + headers: { + ...headers, + "Content-Type": "application/json", + }, + }); + export default startAgent; diff --git a/src/cmd/run.ts b/src/cmd/run.ts index 969bd1c..9295c6b 100644 --- a/src/cmd/run.ts +++ b/src/cmd/run.ts @@ -10,7 +10,7 @@ import { resolve, } from "../../deps.ts"; import { BASE_URL, FLUENTCI_WS_URL, RUNNER_URL } from "../consts.ts"; -import detect from "../detect.ts"; +import detect, { detectProjectType } from "../detect.ts"; import { getCommitInfos } from "../git.ts"; import { setupFluentCIengine, @@ -322,6 +322,8 @@ const runPipelineRemotely = async ( Deno.exit(1); } + const projectType = await detectProjectType("."); + console.log("📦 Creating zip file ..."); const blobWriter = new BlobWriter("application/zip"); @@ -348,6 +350,7 @@ const runPipelineRemotely = async ( jobs, wasm: options.wasm, denoModule, + projectType, }); const blob = await blobWriter.getData(); @@ -370,11 +373,9 @@ const runPipelineRemotely = async ( Deno.exit(1); } else { console.log("✅ Context uploaded successfully"); - const { buildId } = JSON.parse(xhr.response); - saveRepositoryMetadata(buildId); - const websocket = new WebSocket( - `${FLUENTCI_WS_URL}?client_id=${buildId}` - ); + const { runId } = JSON.parse(xhr.response); + saveRepositoryMetadata(runId); + const websocket = new WebSocket(`${FLUENTCI_WS_URL}?client_id=${runId}`); websocket.addEventListener("message", (event) => { if (event.data === "fluentci_exit=0") { @@ -421,7 +422,7 @@ const saveRepositoryMetadata = async (id: string) => { try { const { commit, sha, author, branch } = await getCommitInfos(); console.log("📝 Saving repository metadata ..."); - const status = await fetch(`${BASE_URL}/build/${id}/metadata`, { + const status = await fetch(`${BASE_URL}/run/${id}/metadata`, { method: "PUT", headers: { "Content-Type": "application/json", diff --git a/src/detect.ts b/src/detect.ts index db229ac..0d8a4e2 100644 --- a/src/detect.ts +++ b/src/detect.ts @@ -2,7 +2,7 @@ import * as projects from "./server/kv/projects.ts"; import * as actions from "./server/kv/actions.ts"; import { createId } from "../deps.ts"; -async function fileExists(path: string): Promise { +export async function fileExists(path: string): Promise { try { const { isFile } = await Deno.stat(path); return isFile; @@ -11,7 +11,7 @@ async function fileExists(path: string): Promise { } } -async function dirExists(path: string): Promise { +export async function dirExists(path: string): Promise { try { const { isDirectory } = await Deno.stat(path); return isDirectory; @@ -106,8 +106,8 @@ export default async function detect(src: string): Promise { enabled: true, plugin: "bun", useWasm: true, - logo: "https://cdn.jsdelivr.net/gh/fluent-ci-templates/.github/assets/bun.svg", - githubUrl: "https://github.com/fluent-ci-templates/bun-pipeline", + logo: "https://cdn.jsdelivr.net/gh/fluent-ci-templates/.github/assets/nodejs.svg", + githubUrl: "https://github.com/fluent-ci-templates/nodejs-pipeline", }, ]); return; @@ -398,3 +398,72 @@ export default async function detect(src: string): Promise { }, ]); } + +export async function detectProjectType(src: string): Promise { + if (await fileExists(`${src}/Cargo.toml`)) { + return "rust"; + } + + if (await fileExists(`${src}/go.mod`)) { + return "go"; + } + + if ( + (await fileExists(`${src}/package.json`)) && + (await fileExists(`${src}/bun.lockb`)) + ) { + return "bun"; + } + + if (await fileExists(`${src}/package.json`)) { + return "node"; + } + + if ( + (await fileExists(`${src}/deno.json`)) || + (await fileExists(`${src}/deno.jsonc`)) + ) { + return "deno"; + } + + if (await fileExists(`${src}/gleam.toml`)) { + return "gleam"; + } + + if (await fileExists(`${src}/build.zig`)) { + return "zig"; + } + + if (await fileExists(`${src}/mix.exs`)) { + return "elixir"; + } + + if (await fileExists(`${src}/composer.json`)) { + return "php"; + } + + if ( + (await fileExists(`${src}/pubspec.yaml`)) && + (await dirExists(`${src}/android`)) + ) { + return "flutter"; + } + + if (await fileExists(`${src}/Gemfile`)) { + return "ruby"; + } + + if (await fileExists(`${src}/project.clj`)) { + return "clojure"; + } + + if (await fileExists(`${src}/pom.xml`)) { + return "maven"; + } + + if (await fileExists(`${src}/build.sbt`)) { + return "sbt"; + } + + return "base"; +} diff --git a/src/server/icons.ts b/src/server/icons.ts index ea836a1..ee59012 100644 --- a/src/server/icons.ts +++ b/src/server/icons.ts @@ -224,7 +224,7 @@ export default [ "first-quarter", "full-moon", "last-quarter", - "moon-phase", + "moon", "moon-stars", "new-moon", "waning-crescent", From 7c5098c468330dfdea6a155f3e4c8c726c49f0d8 Mon Sep 17 00:00:00 2001 From: Tsiry Sandratraina Date: Sat, 8 Jun 2024 17:31:17 +0000 Subject: [PATCH 06/18] [remote-exec] fix types --- src/cmd/agent.ts | 5 +++-- src/types.ts | 8 ++++---- 2 files changed, 7 insertions(+), 6 deletions(-) diff --git a/src/cmd/agent.ts b/src/cmd/agent.ts index 52f524a..045ad7d 100644 --- a/src/cmd/agent.ts +++ b/src/cmd/agent.ts @@ -282,7 +282,8 @@ async function executeActions( jobs = jobs.map((job, j) => ({ ...job, - startedAt: currentActionIndex === j ? start.toISOString() : job.startedAt, + startedAt: + currentActionIndex === j ? start.toISOString() : job.started_at, status: currentActionIndex === j ? "RUNNING" : job.status, })); @@ -295,7 +296,7 @@ async function executeActions( for (const cmd of action.commands.split("\n")) { const result = await spawn( - `fluentci run ${action.useWasm ? "--wasm" : ""} ${ + `fluentci run ${action.use_wasm ? "--wasm" : ""} ${ action.plugin } ${cmd}`, `${dir("home")}/.fluentci/builds/${project_id}/${sha256}`, diff --git a/src/types.ts b/src/types.ts index ebb3253..20779df 100644 --- a/src/types.ts +++ b/src/types.ts @@ -49,9 +49,9 @@ export const ActionSchema = z.object({ commands: z.string(), enabled: z.boolean(), plugin: z.string(), - useWasm: z.boolean(), + use_wasm: z.boolean(), logo: z.string().optional().nullable(), - githubUrl: z.string().optional().nullable(), + github_url: z.string().optional().nullable(), }); export const LogSchema = z.object({ @@ -65,8 +65,8 @@ export const JobSchema = z.object({ id: z.string(), name: z.string(), status: z.string(), - createdAt: z.string(), - startedAt: z.string().optional().nullable(), + created_at: z.string(), + started_at: z.string().optional().nullable(), duration: z.number().optional().nullable(), logs: z.array(LogSchema).optional().nullable(), }); From 202b845fa8402d49cbdea50b2514dd72ffdac939 Mon Sep 17 00:00:00 2001 From: Tsiry Sandratraina Date: Sun, 9 Jun 2024 06:56:55 +0000 Subject: [PATCH 07/18] add --work-dir option --- main.ts | 2 ++ src/cmd/run.ts | 12 +++++++++++- 2 files changed, 13 insertions(+), 1 deletion(-) diff --git a/main.ts b/main.ts index 09cf779..36b27a8 100644 --- a/main.ts +++ b/main.ts @@ -44,6 +44,7 @@ export async function main() { .option("-r, --reload", "Reload pipeline source cache") .option("-w, --wasm", "Run pipeline as WebAssembly Module") .option("--remote-exec", "Run pipeline on remote agent") + .option("--work-dir ", "Set working directory") .option("-*, --* [args:string]", "Pass arguments to pipeline") .action(function (options, pipeline, ...jobs: [string, ...Array]) { if (options.wasm) { @@ -61,6 +62,7 @@ export async function main() { .option("-r, --reload", "Reload pipeline source cache") .option("-w, --wasm", "Run pipeline as WebAssembly Module") .option("--remote-exec", "Run pipeline on remote agent") + .option("--work-dir ", "Set working directory") .option("-*, --* [args:string]", "Pass arguments to pipeline") .action(function (options, pipeline, ...jobs: [string, ...Array]) { if (pipeline.endsWith(".wasm") || pipeline.endsWith("?wasm=1")) { diff --git a/src/cmd/run.ts b/src/cmd/run.ts index 9295c6b..13c85a5 100644 --- a/src/cmd/run.ts +++ b/src/cmd/run.ts @@ -10,7 +10,7 @@ import { resolve, } from "../../deps.ts"; import { BASE_URL, FLUENTCI_WS_URL, RUNNER_URL } from "../consts.ts"; -import detect, { detectProjectType } from "../detect.ts"; +import detect, { detectProjectType, dirExists } from "../detect.ts"; import { getCommitInfos } from "../git.ts"; import { setupFluentCIengine, @@ -48,6 +48,16 @@ async function run( export: true, }); + if (options.workDir) { + if (!(await dirExists(options.workDir as string))) { + console.error( + `Directory ${green(options.workDir as string)} does not exist` + ); + Deno.exit(1); + } + Deno.chdir(options.workDir as string); + } + if (options.wasm && !options.remoteExec) { Deno.env.set("WASM_ENABLED", "1"); await runWasmPlugin(pipeline, jobs); From 1fbd817ddd7ccf7464215e91beb1e69b8e9eb09a Mon Sep 17 00:00:00 2001 From: Tsiry Sandratraina Date: Sun, 9 Jun 2024 07:18:37 +0000 Subject: [PATCH 08/18] correctly set cwd option --- src/cmd/agent.ts | 19 +++++++++++++------ src/cmd/run.ts | 43 +++++++++++++++++++++++++++++-------------- 2 files changed, 42 insertions(+), 20 deletions(-) diff --git a/src/cmd/agent.ts b/src/cmd/agent.ts index 045ad7d..a410cf1 100644 --- a/src/cmd/agent.ts +++ b/src/cmd/agent.ts @@ -80,7 +80,7 @@ async function startAgent() { const { action, src, query, runId, actions, run } = JSON.parse( data.event ); - const { jobs, pipeline, wasm } = JSON.parse(query); + const { jobs, pipeline, wasm, workDir } = JSON.parse(query); if (action === "build") { const project_id = src.split("/")[0]; @@ -101,7 +101,8 @@ async function startAgent() { runId, wasm, actions, - run + run, + workDir ); return; } @@ -123,7 +124,8 @@ async function startAgent() { runId, wasm, actions, - run + run, + workDir ); } } catch (e) { @@ -172,7 +174,8 @@ async function spawnFluentCI( clientId: string, wasm: boolean = false, actions: Action[] = [], - run?: Run + run?: Run, + workDir = "." ) { if (actions.length > 0) { await executeActions(actions, project_id, sha256, run!, logger, clientId); @@ -187,7 +190,9 @@ async function spawnFluentCI( pipeline, ...jobs.filter((x) => x !== "--remote-exec"), ], - cwd: `${dir("home")}/.fluentci/builds/${project_id}/${sha256}`, + cwd: `${dir( + "home" + )}/.fluentci/builds/${project_id}/${sha256}/${workDir}`, stdout: "piped", stderr: "piped", }) @@ -201,7 +206,9 @@ async function spawnFluentCI( pipeline, ...jobs.filter((x) => x !== "--remote-exec"), ], - cwd: `${dir("home")}/.fluentci/builds/${project_id}/${sha256}`, + cwd: `${dir( + "home" + )}/.fluentci/builds/${project_id}/${sha256}/${workDir}`, stdout: "piped", stderr: "piped", }); diff --git a/src/cmd/run.ts b/src/cmd/run.ts index 13c85a5..c66097c 100644 --- a/src/cmd/run.ts +++ b/src/cmd/run.ts @@ -55,25 +55,26 @@ async function run( ); Deno.exit(1); } - Deno.chdir(options.workDir as string); } if (options.wasm && !options.remoteExec) { Deno.env.set("WASM_ENABLED", "1"); - await runWasmPlugin(pipeline, jobs); + await runWasmPlugin(pipeline, jobs, options.workDir as string); Deno.exit(0); } if (pipeline === ".") { - try { - // verify if .fluentci directory exists - const fluentciDir = await Deno.stat(".fluentci"); - await Deno.stat(".fluentci/mod.ts"); - if (!fluentciDir.isDirectory) { + if (!Deno.env.has("FLUENTCI_PROJECT_ID")) { + try { + // verify if .fluentci directory exists + const fluentciDir = await Deno.stat(".fluentci"); + await Deno.stat(".fluentci/mod.ts"); + if (!fluentciDir.isDirectory) { + displayErrorMessage(); + } + } catch (_) { displayErrorMessage(); } - } catch (_) { - displayErrorMessage(); } if (Deno.env.get("FLUENTCI_PROJECT_ID")) { @@ -81,7 +82,7 @@ async function run( await runPipelineRemotely(pipeline, jobs, options); return; } - await detect("."); + await detect((options.workDir as string) || "."); const query = gql` mutation Run($projectId: ID!, $wait: Boolean) { @@ -103,7 +104,7 @@ async function run( await projects.save({ ...project, - path: resolve("."), + path: resolve((options.workDir as string) || "."), }); await projects.deleteAt("empty"); @@ -143,6 +144,7 @@ async function run( ], stdout: "inherit", stderr: "inherit", + cwd: (options.workDir as string) || ".", }); if ( @@ -164,6 +166,7 @@ async function run( ], stdout: "inherit", stderr: "inherit", + cwd: (options.workDir as string) || ".", }); } @@ -171,7 +174,9 @@ async function run( const commands = []; for (const job of jobs) { try { - const jobFile = await Deno.stat(`.fluentci/${job}.ts`); + const jobFile = await Deno.stat( + `${(options.workDir as string) || "."}/.fluentci/${job}.ts` + ); if (jobFile.isFile) { jobFileExists = true; commands.push( @@ -188,6 +193,7 @@ async function run( ], stdout: "inherit", stderr: "inherit", + cwd: (options.workDir as string) || ".", }) ); break; @@ -211,6 +217,7 @@ async function run( ], stdout: "inherit", stderr: "inherit", + cwd: (options.workDir as string) || ".", }) ); } @@ -264,6 +271,7 @@ async function run( args: ["run", "-A", ...denoModule], stdout: "inherit", stderr: "inherit", + cwd: (options.workDir as string) || ".", }); if ( @@ -274,6 +282,7 @@ async function run( args: ["run", "deno", "run", "-A", ...denoModule], stdout: "inherit", stderr: "inherit", + cwd: (options.workDir as string) || ".", }); } @@ -332,7 +341,9 @@ const runPipelineRemotely = async ( Deno.exit(1); } - const projectType = await detectProjectType("."); + const projectType = await detectProjectType( + (options.workDir as string) || "." + ); console.log("📦 Creating zip file ..."); @@ -361,6 +372,7 @@ const runPipelineRemotely = async ( wasm: options.wasm, denoModule, projectType, + workDir: options.workDir, }); const blob = await blobWriter.getData(); @@ -454,12 +466,13 @@ const saveRepositoryMetadata = async (id: string) => { } }; -const runWasmPlugin = async (pipeline: string, job: string[]) => { +const runWasmPlugin = async (pipeline: string, job: string[], cwd = ".") => { if (pipeline.endsWith(".wasm") || pipeline.endsWith("?wasm=1")) { const command = new Deno.Command("bash", { args: ["-c", `fluentci-engine call -m ${pipeline} -- ` + job.join(" ")], stdout: "inherit", stderr: "inherit", + cwd, }); await spawnCommand(command); return; @@ -511,6 +524,7 @@ const runWasmPlugin = async (pipeline: string, job: string[]) => { ], stdout: "inherit", stderr: "inherit", + cwd, }); await spawnCommand(command); return; @@ -539,6 +553,7 @@ const runWasmPlugin = async (pipeline: string, job: string[]) => { args: ["-c", `fluentci-engine call -m ${url} -- ` + job.join(" ")], stdout: "inherit", stderr: "inherit", + cwd, }); await spawnCommand(command); }; From 9b09093379c131691de29ab7745e32b3f4c0b17a Mon Sep 17 00:00:00 2001 From: Tsiry Sandratraina Date: Sun, 9 Jun 2024 07:43:55 +0000 Subject: [PATCH 09/18] correctly set cwd option --- src/cmd/run.ts | 48 ++++++++++++++++++++++++++++++++++++++---------- src/utils.ts | 6 +++--- 2 files changed, 41 insertions(+), 13 deletions(-) diff --git a/src/cmd/run.ts b/src/cmd/run.ts index c66097c..bf9a557 100644 --- a/src/cmd/run.ts +++ b/src/cmd/run.ts @@ -67,8 +67,10 @@ async function run( if (!Deno.env.has("FLUENTCI_PROJECT_ID")) { try { // verify if .fluentci directory exists - const fluentciDir = await Deno.stat(".fluentci"); - await Deno.stat(".fluentci/mod.ts"); + const fluentciDir = await Deno.stat( + `${options.workDir || "."}/.fluentci` + ); + await Deno.stat(`${options.workDir || "."}/.fluentci/mod.ts`); if (!fluentciDir.isDirectory) { displayErrorMessage(); } @@ -139,7 +141,13 @@ async function run( ".fluentci/src/dagger/runner.ts", ...jobs, ...Object.keys(options) - .filter((key) => key !== "reload" && key !== "wasm") + .filter( + (key) => + key !== "reload" && + key !== "wasm" && + key !== "remoteExec" && + key !== "workDir" + ) .map((key) => `--${key}=${options[key]}`), ], stdout: "inherit", @@ -161,7 +169,13 @@ async function run( ".fluentci/src/dagger/runner.ts", ...jobs, ...Object.keys(options) - .filter((key) => key !== "reload" && key !== "wasm") + .filter( + (key) => + key !== "reload" && + key !== "wasm" && + key !== "remoteExec" && + key !== "workDir" + ) .map((key) => `--${key}=${options[key]}`), ], stdout: "inherit", @@ -188,7 +202,13 @@ async function run( "-A", `.fluentci/${job}.ts`, ...Object.keys(options) - .filter((key) => key !== "reload" && key !== "wasm") + .filter( + (key) => + key !== "reload" && + key !== "wasm" && + key !== "remoteExec" && + key !== "workDir" + ) .map((key) => `--${key}=${options[key]}`), ], stdout: "inherit", @@ -211,7 +231,13 @@ async function run( "-A", `.fluentci/${job}.ts`, ...Object.keys(options) - .filter((key) => key !== "reload" && key !== "wasm") + .filter( + (key) => + key !== "reload" && + key !== "wasm" && + key !== "remoteExec" && + key !== "workDir" + ) .map((key) => `--${key}=${options[key]}`) .join(" "), ], @@ -478,10 +504,10 @@ const runWasmPlugin = async (pipeline: string, job: string[], cwd = ".") => { return; } - const pluginDirExists = await fluentciPluginDirExists(); + const pluginDirExists = await fluentciPluginDirExists(cwd); if (!pluginDirExists && pipeline === ".") { try { - await Deno.stat(".fluentci/Cargo.toml"); + await Deno.stat(`${cwd}/.fluentci/Cargo.toml`); } catch (_) { console.log("This directory does not contain a FluentCI plugin"); Deno.exit(1); @@ -501,13 +527,15 @@ const runWasmPlugin = async (pipeline: string, job: string[], cwd = ".") => { args: ["build", "--release", "--target", "wasm32-unknown-unknown"], stdout: "inherit", stderr: "inherit", - cwd: pluginDirExists ? ".fluentci/plugin" : ".fluentci", + cwd: pluginDirExists ? `${cwd}/.fluentci/plugin` : `${cwd}/.fluentci`, }); await spawnCommand(build); const cargoToml = toml.parse( Deno.readTextFileSync( - pluginDirExists ? ".fluentci/plugin/Cargo.toml" : ".fluentci/Cargo.toml" + pluginDirExists + ? `${cwd}/.fluentci/plugin/Cargo.toml` + : `${cwd}/.fluentci/Cargo.toml` ) // deno-lint-ignore no-explicit-any ) as Record; diff --git a/src/utils.ts b/src/utils.ts index 8fc367c..8f02feb 100644 --- a/src/utils.ts +++ b/src/utils.ts @@ -325,10 +325,10 @@ export async function setupFluentCIStudio() { } } -export async function fluentciPluginDirExists(): Promise { +export async function fluentciPluginDirExists(cwd = "."): Promise { try { - const fluentciDir = await Deno.stat(".fluentci/plugin"); - await Deno.stat(".fluentci/plugin/Cargo.toml"); + const fluentciDir = await Deno.stat(`${cwd}/.fluentci/plugin`); + await Deno.stat(`${cwd}/.fluentci/plugin/Cargo.toml`); return fluentciDir.isDirectory; } catch (_) { return false; From f8bc124dddaba5ecfbda1629c456c8569e3d0e85 Mon Sep 17 00:00:00 2001 From: Tsiry Sandratraina Date: Sun, 9 Jun 2024 08:08:07 +0000 Subject: [PATCH 10/18] fix .fluentciignore issue --- src/cmd/run.ts | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/src/cmd/run.ts b/src/cmd/run.ts index bf9a557..f877ed5 100644 --- a/src/cmd/run.ts +++ b/src/cmd/run.ts @@ -350,6 +350,8 @@ const runPipelineRemotely = async ( const accessToken = getAccessToken(); + console.log(parseIgnoredFiles()); + const entries = walk(".", { skip: parseIgnoredFiles(), }); @@ -440,7 +442,7 @@ const runPipelineRemotely = async ( xhr.send(blob); }; -const parseIgnoredFiles = () => { +const parseIgnoredFiles = (): RegExp[] => { let ignoredFilesArray: RegExp[] = []; try { // verify if .fluentciignore exists @@ -449,6 +451,7 @@ const parseIgnoredFiles = () => { ignoredFilesArray = ignoredFilesArray.concat( ignoredFiles .split("\n") + .filter((x) => x.trim().length) .map((file) => new RegExp(file.replace(".", "\\."))) ); } @@ -459,7 +462,10 @@ const parseIgnoredFiles = () => { try { const ignoredFiles = Deno.readTextFileSync(".gitignore"); return ignoredFilesArray.concat( - ignoredFiles.split("\n").map((file: string) => new RegExp(file)) + ignoredFiles + .split("\n") + .filter((x) => x.trim().length) + .map((file: string) => new RegExp(file)) ); } catch (_e) { return ignoredFilesArray; From fdc46d9f5385d7a15eb9815eabd381334f42f635 Mon Sep 17 00:00:00 2001 From: Tsiry Sandratraina Date: Sun, 9 Jun 2024 08:10:40 +0000 Subject: [PATCH 11/18] correctly set cwd option --- src/cmd/agent.ts | 15 ++++++++++++--- 1 file changed, 12 insertions(+), 3 deletions(-) diff --git a/src/cmd/agent.ts b/src/cmd/agent.ts index a410cf1..bb51354 100644 --- a/src/cmd/agent.ts +++ b/src/cmd/agent.ts @@ -178,7 +178,15 @@ async function spawnFluentCI( workDir = "." ) { if (actions.length > 0) { - await executeActions(actions, project_id, sha256, run!, logger, clientId); + await executeActions( + actions, + project_id, + sha256, + run!, + logger, + clientId, + workDir + ); return; } @@ -265,7 +273,8 @@ async function executeActions( sha256: string, run: Run, logger: Logger, - clientId: string + clientId: string, + workDir = "." ) { let currentActionIndex = 0; const runStart = dayjs(); @@ -306,7 +315,7 @@ async function executeActions( `fluentci run ${action.use_wasm ? "--wasm" : ""} ${ action.plugin } ${cmd}`, - `${dir("home")}/.fluentci/builds/${project_id}/${sha256}`, + `${dir("home")}/.fluentci/builds/${project_id}/${sha256}/${workDir}`, jobs[currentActionIndex].id, logger, clientId From e23b816098f323cbef3d7c79c137f4f2cff950cb Mon Sep 17 00:00:00 2001 From: Tsiry Sandratraina Date: Sun, 9 Jun 2024 15:29:45 +0000 Subject: [PATCH 12/18] rename startedAt -> started_at --- src/cmd/agent.ts | 4 ++-- src/cmd/run.ts | 2 -- 2 files changed, 2 insertions(+), 4 deletions(-) diff --git a/src/cmd/agent.ts b/src/cmd/agent.ts index bb51354..0fb1473 100644 --- a/src/cmd/agent.ts +++ b/src/cmd/agent.ts @@ -298,7 +298,7 @@ async function executeActions( jobs = jobs.map((job, j) => ({ ...job, - startedAt: + started_at: currentActionIndex === j ? start.toISOString() : job.started_at, status: currentActionIndex === j ? "RUNNING" : job.status, })); @@ -306,7 +306,7 @@ async function executeActions( // send update job status "RUNNING" + date sendEvent(clientId, "job", { ...jobs[currentActionIndex], - startedAt: start.toISOString(), + started_at: start.toISOString(), status: "RUNNING", }).catch((e) => logger.error(e.message)); diff --git a/src/cmd/run.ts b/src/cmd/run.ts index f877ed5..09abca7 100644 --- a/src/cmd/run.ts +++ b/src/cmd/run.ts @@ -350,8 +350,6 @@ const runPipelineRemotely = async ( const accessToken = getAccessToken(); - console.log(parseIgnoredFiles()); - const entries = walk(".", { skip: parseIgnoredFiles(), }); From 2e8bc760cf76ef59937bf9b550dac52ec6b0a039 Mon Sep 17 00:00:00 2001 From: Tsiry Sandratraina Date: Sun, 9 Jun 2024 18:41:58 +0000 Subject: [PATCH 13/18] use iso string date --- src/cmd/agent.ts | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/src/cmd/agent.ts b/src/cmd/agent.ts index 0fb1473..cb2d6cb 100644 --- a/src/cmd/agent.ts +++ b/src/cmd/agent.ts @@ -279,12 +279,13 @@ async function executeActions( let currentActionIndex = 0; const runStart = dayjs(); let jobs = [...run.jobs]; + const date = new Date().toISOString(); // send update run status "RUNNING" + date sendEvent(clientId, "run", { ...run, status: "RUNNING", - date: new Date().toISOString(), + date, }).catch((e) => logger.error(e.message)); for (const action of actions) { @@ -351,6 +352,7 @@ async function executeActions( jobs, status: "FAILURE", duration, + date, }).catch((e) => logger.error(e.message)); // send update project stats @@ -383,6 +385,7 @@ async function executeActions( sendEvent(clientId, "run", { ...run!, jobs, + date, }); currentActionIndex += 1; } @@ -393,6 +396,7 @@ async function executeActions( ...run!, status: "SUCCESS", duration, + date, }).catch((e) => logger.error(e.message)); // update project stats From 6aa98918bc5a0901abe29a2d789a1c5c74b926ea Mon Sep 17 00:00:00 2001 From: Tsiry Sandratraina Date: Sun, 9 Jun 2024 20:44:01 +0000 Subject: [PATCH 14/18] use job_id fix job status fix job status fix job status fix job status fix job status --- src/cmd/agent.ts | 18 ++++++++++++++---- src/types.ts | 1 + 2 files changed, 15 insertions(+), 4 deletions(-) diff --git a/src/cmd/agent.ts b/src/cmd/agent.ts index cb2d6cb..b7278dc 100644 --- a/src/cmd/agent.ts +++ b/src/cmd/agent.ts @@ -8,6 +8,7 @@ import { Table, dayjs, createId, + sleep, } from "../../deps.ts"; import { FLUENTCI_WS_URL, @@ -317,7 +318,7 @@ async function executeActions( action.plugin } ${cmd}`, `${dir("home")}/.fluentci/builds/${project_id}/${sha256}/${workDir}`, - jobs[currentActionIndex].id, + jobs[currentActionIndex].job_id, logger, clientId ); @@ -369,6 +370,13 @@ async function executeActions( } } + sendEvent(clientId, "job", { + ...jobs[currentActionIndex], + status: "SUCCESS", + duration: dayjs().diff(start, "milliseconds"), + logs: [...(jobs[currentActionIndex].logs || []), ...logs], + }).catch((e) => logger.error(e.message)); + jobs = jobs.map((job, j) => ({ ...job, status: currentActionIndex === j ? "SUCCESS" : job.status, @@ -386,18 +394,20 @@ async function executeActions( ...run!, jobs, date, - }); + }).catch((e) => logger.error(e.message)); currentActionIndex += 1; } + await sleep(500); + // update run status "SUCCESS" + duration const duration = dayjs().diff(runStart, "milliseconds"); - sendEvent(clientId, "run", { + await sendEvent(clientId, "run", { ...run!, status: "SUCCESS", duration, date, - }).catch((e) => logger.error(e.message)); + }); // update project stats sendEvent(clientId, "update-stats", {}).catch((e) => logger.error(e.message)); diff --git a/src/types.ts b/src/types.ts index 20779df..6dbcac2 100644 --- a/src/types.ts +++ b/src/types.ts @@ -63,6 +63,7 @@ export const LogSchema = z.object({ export const JobSchema = z.object({ id: z.string(), + job_id: z.string(), name: z.string(), status: z.string(), created_at: z.string(), From 9672b89e9ceb0e165f8fc044c0f393874d7948f9 Mon Sep 17 00:00:00 2001 From: Tsiry Sandratraina Date: Sun, 9 Jun 2024 23:06:24 +0000 Subject: [PATCH 15/18] fix job status --- src/cmd/agent.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/cmd/agent.ts b/src/cmd/agent.ts index b7278dc..6b916d9 100644 --- a/src/cmd/agent.ts +++ b/src/cmd/agent.ts @@ -398,7 +398,7 @@ async function executeActions( currentActionIndex += 1; } - await sleep(500); + await sleep(100); // update run status "SUCCESS" + duration const duration = dayjs().diff(runStart, "milliseconds"); From 2d10b835ca466a306cf399e556fd0b93cedac5b4 Mon Sep 17 00:00:00 2001 From: Tsiry Sandratraina Date: Sun, 9 Jun 2024 23:10:15 +0000 Subject: [PATCH 16/18] fix run duration --- src/cmd/agent.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/cmd/agent.ts b/src/cmd/agent.ts index 6b916d9..4a473dd 100644 --- a/src/cmd/agent.ts +++ b/src/cmd/agent.ts @@ -401,7 +401,7 @@ async function executeActions( await sleep(100); // update run status "SUCCESS" + duration - const duration = dayjs().diff(runStart, "milliseconds"); + const duration = dayjs().diff(runStart, "milliseconds") - 100; await sendEvent(clientId, "run", { ...run!, status: "SUCCESS", From 837ebf164c49937ec63fc23a699a6a8464faa38a Mon Sep 17 00:00:00 2001 From: Tsiry Sandratraina Date: Tue, 11 Jun 2024 06:09:31 +0000 Subject: [PATCH 17/18] set projectId value in update-stats event --- src/cmd/agent.ts | 10 ++++++---- src/types.ts | 3 ++- 2 files changed, 8 insertions(+), 5 deletions(-) diff --git a/src/cmd/agent.ts b/src/cmd/agent.ts index 4a473dd..534113b 100644 --- a/src/cmd/agent.ts +++ b/src/cmd/agent.ts @@ -357,9 +357,9 @@ async function executeActions( }).catch((e) => logger.error(e.message)); // send update project stats - sendEvent(clientId, "update-stats", {}).catch((e) => - logger.error(e.message) - ); + sendEvent(clientId, "update-stats", { + projectId: run.project_id, + }).catch((e) => logger.error(e.message)); fetch(`${FLUENTCI_EVENTS_URL}?client_id=${clientId}`, { method: "POST", @@ -410,7 +410,9 @@ async function executeActions( }); // update project stats - sendEvent(clientId, "update-stats", {}).catch((e) => logger.error(e.message)); + sendEvent(clientId, "update-stats", { projectId: run.project_id }).catch( + (e) => logger.error(e.message) + ); fetch(`${FLUENTCI_EVENTS_URL}?client_id=${clientId}`, { method: "POST", diff --git a/src/types.ts b/src/types.ts index 6dbcac2..787e69c 100644 --- a/src/types.ts +++ b/src/types.ts @@ -76,7 +76,8 @@ export const RunSchema = z.object({ id: z.string(), name: z.string(), project: z.string(), - projectId: z.string(), + project_id: z.string(), + run_id: z.string(), title: z.string(), message: z.string().optional().nullable(), commit: z.string().optional().nullable(), From 790f4ab8dbcdc1135ef9807029a7c7822d8ff5bc Mon Sep 17 00:00:00 2001 From: Tsiry Sandratraina Date: Tue, 11 Jun 2024 10:34:05 +0000 Subject: [PATCH 18/18] bump version code --- README.md | 10 +++++----- src/consts.ts | 2 +- src/utils.ts | 2 +- 3 files changed, 7 insertions(+), 7 deletions(-) diff --git a/README.md b/README.md index 81a8c67..fb58b92 100644 --- a/README.md +++ b/README.md @@ -81,13 +81,13 @@ Requirements: **Latest (Desktop):** -- `Mac`: arm64: [fluentci-studio_v0.1.0_arm64.dmg](https://github.com/fluentci-io/fluentci-studio/releases/download/v0.1.0/fluentci-studio_v0.1.0_arm64.dmg) intel: [fluentci-studio_v0.1.0_x64.dmg](https://github.com/fluentci-io/fluentci-studio/releases/download/v0.1.0/fluentci-studio_v0.1.0_x64.dmg) -- `Linux`: [fluentci-studio_v0.1.0.AppImage](https://github.com/fluentci-io/fluentci-studio/releases/download/v0.1.0/fluentci-studio_v0.1.0.AppImage) +- `Mac`: arm64: [fluentci-studio_v0.1.1_arm64.dmg](https://github.com/fluentci-io/fluentci-studio/releases/download/v0.1.1/fluentci-studio_v0.1.1_arm64.dmg) intel: [fluentci-studio_v0.1.1_x64.dmg](https://github.com/fluentci-io/fluentci-studio/releases/download/v0.1.1/fluentci-studio_v0.1.1_x64.dmg) +- `Linux`: [fluentci-studio_v0.1.1.AppImage](https://github.com/fluentci-io/fluentci-studio/releases/download/v0.1.1/fluentci-studio_v0.1.1.AppImage) **Latest (CLI):** -- `Mac`: arm64: [fluentci_v0.14.6_aarch64-apple-darwin.tar.gz](https://github.com/fluentci-io/fluentci/releases/download/v0.14.6/fluentci_v0.14.6_aarch64-apple-darwin.tar.gz) intel: [fluentci_v0.14.6_x86_64-apple-darwin.tar.gz](https://github.com/fluentci-io/fluentci/releases/download/v0.14.6/fluentci_v0.14.6_x86_64-apple-darwin.tar.gz) -- `Linux`: intel: [fluentci_v0.14.6_x86_64-unknown-linux-gnu.tar.gz](https://github.com/fluentci-io/fluentci/releases/download/v0.14.6/fluentci_v0.14.6_x86_64-unknown-linux-gnu.tar.gz) arm64: [fluentci_v0.14.6_aarch64-unknown-linux-gnu.tar.gz](https://github.com/fluentci-io/fluentci/releases/download/v0.14.6/fluentci_v0.14.6_aarch64-unknown-linux-gnu.tar.gz) +- `Mac`: arm64: [fluentci_v0.14.7_aarch64-apple-darwin.tar.gz](https://github.com/fluentci-io/fluentci/releases/download/v0.14.7/fluentci_v0.14.7_aarch64-apple-darwin.tar.gz) intel: [fluentci_v0.14.7_x86_64-apple-darwin.tar.gz](https://github.com/fluentci-io/fluentci/releases/download/v0.14.7/fluentci_v0.14.7_x86_64-apple-darwin.tar.gz) +- `Linux`: intel: [fluentci_v0.14.7_x86_64-unknown-linux-gnu.tar.gz](https://github.com/fluentci-io/fluentci/releases/download/v0.14.7/fluentci_v0.14.7_x86_64-unknown-linux-gnu.tar.gz) arm64: [fluentci_v0.14.7_aarch64-unknown-linux-gnu.tar.gz](https://github.com/fluentci-io/fluentci/releases/download/v0.14.7/fluentci_v0.14.7_aarch64-unknown-linux-gnu.tar.gz) ## ✨ Quick Start @@ -110,7 +110,7 @@ fluentci studio fluentci --help Usage: fluentci [pipeline] [jobs...] -Version: 0.14.6 +Version: 0.14.7 Description: diff --git a/src/consts.ts b/src/consts.ts index 9790f7f..ebc128d 100644 --- a/src/consts.ts +++ b/src/consts.ts @@ -1,6 +1,6 @@ import { dir } from "../deps.ts"; -export const VERSION = "0.14.6"; +export const VERSION = "0.14.7"; export const BASE_URL = "https://api.fluentci.io/v1"; diff --git a/src/utils.ts b/src/utils.ts index 8f02feb..3f12930 100644 --- a/src/utils.ts +++ b/src/utils.ts @@ -293,7 +293,7 @@ export async function setupFluentCIengine() { export async function setupFluentCIStudio() { await setupPkgx(); let FLUENTCI_STUDIO_VERSION = - Deno.env.get("FLUENTCI_STUDIO_VERSION") || "v0.1.0"; + Deno.env.get("FLUENTCI_STUDIO_VERSION") || "v0.1.1"; if (!FLUENTCI_STUDIO_VERSION.startsWith("v")) { FLUENTCI_STUDIO_VERSION = `v${FLUENTCI_STUDIO_VERSION}`;