diff --git a/.github/workflows/checks.yml b/.github/workflows/checks.yml index 7360ce227..4bb08374f 100644 --- a/.github/workflows/checks.yml +++ b/.github/workflows/checks.yml @@ -37,10 +37,11 @@ jobs: run: yarn install --immutable - name: Run tests and generate coverage run: make test_unit_codecov - - uses: codecov/codecov-action@v1 + - uses: codecov/codecov-action@v3 with: + token: ${{ secrets.CODECOV_TOKEN }} files: .coverage/coverage-final.json - fail_ci_if_error: true + fail_ci_if_error: false lint_project: runs-on: ubuntu-latest @@ -68,13 +69,7 @@ jobs: release: name: Generate Release if: ${{ (github.event_name != 'pull_request') && (needs.extract_branch.outputs.branch == 'master') }} - needs: - [ - unit_tests_with_coverage, - lint_project, - build_docker_image, - extract_branch, - ] + needs: [unit_tests_with_coverage, lint_project, build_docker_image, extract_branch] runs-on: ubuntu-latest steps: - name: Checkout diff --git a/packages/oss-console/src/components/Executions/contextProvider/NodeExecutionDetails/createExecutionArray.tsx b/packages/oss-console/src/components/Executions/contextProvider/NodeExecutionDetails/createExecutionArray.tsx index 0dbbc321b..549b8ecf6 100644 --- a/packages/oss-console/src/components/Executions/contextProvider/NodeExecutionDetails/createExecutionArray.tsx +++ b/packages/oss-console/src/components/Executions/contextProvider/NodeExecutionDetails/createExecutionArray.tsx @@ -7,6 +7,7 @@ import { Identifier } from '../../../../models/Common/types'; import { CompiledTask } from '../../../../models/Task/types'; import { dNode } from '../../../../models/Graph/types'; import { isEndNode, isStartNode } from '../../../../models/Node/utils'; +import { getTaskTypeFromCompiledNode } from '../../../WorkflowGraph/utils'; interface NodeExecutionInfo extends NodeExecutionDetails { scopedId?: string; @@ -104,6 +105,16 @@ export const getNodeDetails = ( }; } + if (compiledNode?.arrayNode) { + returnVal = { + ...returnVal, + displayType: + returnVal.displayType !== NodeExecutionDisplayType.Unknown + ? returnVal.displayType + : NodeExecutionDisplayType.ArrayNode, + }; + } + return returnVal; }; @@ -119,6 +130,13 @@ export const getNodeDetailsFromTask = (node: dNode, task?: CompiledTask): NodeEx displayType: taskType ?? NodeExecutionDisplayType.Unknown, }; + if (node.value?.arrayNode) { + returnVal = { + ...returnVal, + displayType: NodeExecutionDisplayType.ArrayNode, + }; + } + if (node.value?.workflowNode) { const { workflowNode } = node.value; const info = workflowNode.launchplanRef ?? workflowNode.subWorkflowRef; @@ -160,7 +178,7 @@ export const getNodeExecutionDetails = ( node: dNode, tasks: CompiledTask[] = [], ): NodeExecutionInfo => { - const templateName = node?.value?.taskNode?.referenceId?.name ?? node.name; - const task = tasks.find((t) => t.template.id.name === templateName); + const taskNode = node?.value?.arrayNode?.node?.taskNode || node?.value?.taskNode; + const task = getTaskTypeFromCompiledNode(taskNode!, tasks); return getNodeDetailsFromTask(node, task); }; diff --git a/packages/oss-console/src/components/Executions/types.ts b/packages/oss-console/src/components/Executions/types.ts index 4256a359d..535167d8d 100644 --- a/packages/oss-console/src/components/Executions/types.ts +++ b/packages/oss-console/src/components/Executions/types.ts @@ -1,9 +1,4 @@ -import { - NodeExecution, - NodeExecutionClosure, - NodeExecutionMetadata, - WorkflowNodeMetadata, -} from '../../models/Execution/types'; +import { NodeExecution, NodeExecutionMetadata } from '../../models/Execution/types'; import { TaskTemplate } from '../../models/Task/types'; export interface ExecutionPhaseConstants { @@ -15,6 +10,7 @@ export interface ExecutionPhaseConstants { } export enum NodeExecutionDisplayType { + ArrayNode = 'Array Node', MapTask = 'Map Task', BatchHiveTask = 'Hive Batch Task', BranchNode = 'Branch Node', diff --git a/packages/oss-console/src/components/WorkflowGraph/transformerWorkflowToDag.tsx b/packages/oss-console/src/components/WorkflowGraph/transformerWorkflowToDag.tsx index c71bdbe75..b45b213bd 100644 --- a/packages/oss-console/src/components/WorkflowGraph/transformerWorkflowToDag.tsx +++ b/packages/oss-console/src/components/WorkflowGraph/transformerWorkflowToDag.tsx @@ -6,7 +6,7 @@ import { DISPLAY_NAME_END, DISPLAY_NAME_START } from '../flytegraph/ReactFlow/ut import { createDebugLogger } from '../../common/log'; import { dTypes, dEdge, dNode } from '../../models/Graph/types'; import { startNodeId, endNodeId } from '../../models/Node/constants'; -import { CompiledNode, ConnectionSet, TaskNode } from '../../models/Node/types'; +import { ArrayNode, CompiledNode, ConnectionSet, TaskNode } from '../../models/Node/types'; import { CompiledTask } from '../../models/Task/types'; import { CompiledWorkflow, CompiledWorkflowClosure } from '../../models/Workflow/types'; import { isStartOrEndNode } from '../../models/Node/utils'; @@ -81,6 +81,7 @@ const createDNode = ({ gateNode: compiledNode.gateNode, level: parentDNode?.level !== undefined ? parentDNode.level + 1 : 0, ...nodeMetadata, + ...(compiledNode.arrayNode ? { arrayNode: compiledNode.arrayNode } : {}), ...(compiledNode.workflowNode ? { workflowNode: compiledNode.workflowNode } : {}), ...(compiledNode.gateNode ? { gateNode: compiledNode.gateNode } : {}), ...(compiledNode.branchNode ? { taskNode: compiledNode.taskNode } : {}), @@ -247,6 +248,21 @@ const parseNode = ({ compiledWorkflowClosure, }); } + } else if (node?.arrayNode) { + const arrayNode = (node.arrayNode as ArrayNode).node; + const taskNode = arrayNode.taskNode as TaskNode; + const taskType: CompiledTask = getTaskTypeFromCompiledNode( + taskNode, + compiledWorkflowClosure.tasks, + ) as CompiledTask; + dNode = createDNode({ + compiledNode: node, + parentDNode: root, + taskTemplate: taskType, + nodeMetadataMap, + staticExecutionIdsMap, + compiledWorkflowClosure, + }); } else if (node.taskNode) { const taskNode = node.taskNode as TaskNode; const taskType: CompiledTask = getTaskTypeFromCompiledNode( diff --git a/packages/oss-console/src/components/WorkflowGraph/utils.ts b/packages/oss-console/src/components/WorkflowGraph/utils.ts index cc6c3a68c..4bf9e9193 100644 --- a/packages/oss-console/src/components/WorkflowGraph/utils.ts +++ b/packages/oss-console/src/components/WorkflowGraph/utils.ts @@ -100,6 +100,9 @@ export const getSubWorkflowFromId = ( }; export const getTaskTypeFromCompiledNode = (taskNode: TaskNode, tasks: CompiledTask[]) => { + if (!taskNode?.referenceId) { + return undefined; + } for (let i = 0; i < tasks.length; i++) { const compiledTask: CompiledTask = tasks[i]; const taskTemplate: TaskTemplate = compiledTask.template; @@ -108,7 +111,7 @@ export const getTaskTypeFromCompiledNode = (taskNode: TaskNode, tasks: CompiledT return compiledTask; } } - return null; + return undefined; }; export const getNodeNameFromDag = (dagData: dNode, nodeId: string) => { diff --git a/packages/oss-console/src/models/Node/types.ts b/packages/oss-console/src/models/Node/types.ts index 22c6529c8..f8c85887d 100644 --- a/packages/oss-console/src/models/Node/types.ts +++ b/packages/oss-console/src/models/Node/types.ts @@ -6,6 +6,14 @@ export type WorkflowNode = Core.IWorkflowNode; /** A graph node indicating a branching decision. */ export type BranchNode = Core.IBranchNode; +/** A graph node indicating a Array Node. */ +export interface ArrayNode extends Core.IArrayNode { + node: CompiledNode; + parallelism?: number; + minSuccesses?: number; + minSuccessRatio?: number; +} + /** A graph node indicating a task to be executed. This is the most common * node type in a Flyte graph. */ @@ -30,6 +38,7 @@ export interface CompiledNode extends Core.INode { inputs?: Binding[]; metadata?: CompiledNodeMetadata; outputAliases?: Alias[]; + arrayNode?: ArrayNode; taskNode?: TaskNode; upstreamNodeIds?: string[]; workflowNode?: WorkflowNode;