From 33a6ccfc9f9f8bc12b2e314e99bb9eefaedd7d45 Mon Sep 17 00:00:00 2001 From: Nathan Vieira Marcelino Date: Tue, 24 Oct 2023 07:54:58 -0300 Subject: [PATCH 01/16] feat: add additional language support on code editor input - close: [frontend] Add new language options to CodeEditorItem #51 --- .../src/components/CodeEditorInput/index.tsx | 31 +++++++++++++---- .../PieceForm/PieceFormItem/index.tsx | 34 +++++++++++++++++-- 2 files changed, 56 insertions(+), 9 deletions(-) diff --git a/frontend/src/components/CodeEditorInput/index.tsx b/frontend/src/components/CodeEditorInput/index.tsx index d787245d..e7f6cb20 100644 --- a/frontend/src/components/CodeEditorInput/index.tsx +++ b/frontend/src/components/CodeEditorInput/index.tsx @@ -7,11 +7,16 @@ import { useFormContext, } from "react-hook-form"; -const CodeEditorItem = React.forwardRef( - ({ ...register }, ref) => ( +interface Props { + language?: string; + placeholder?: string; +} + +const CodeEditorItem = React.forwardRef( + ({ language = "python", placeholder, ...register }, ref) => ( ( CodeEditorItem.displayName = "CodeEditorItem"; -interface Props { +interface CodeEditorInputProps { name: Path; + language?: string; + placeholder?: string; } -function CodeEditorInput({ name }: Props) { +function CodeEditorInput({ + name, + language, + placeholder, +}: CodeEditorInputProps) { const { control } = useFormContext(); return ( } + render={({ field }) => ( + + )} /> ); } diff --git a/frontend/src/features/workflowEditor/components/SidebarForm/PieceForm/PieceFormItem/index.tsx b/frontend/src/features/workflowEditor/components/SidebarForm/PieceForm/PieceFormItem/index.tsx index 8dfff74c..0046121d 100644 --- a/frontend/src/features/workflowEditor/components/SidebarForm/PieceForm/PieceFormItem/index.tsx +++ b/frontend/src/features/workflowEditor/components/SidebarForm/PieceForm/PieceFormItem/index.tsx @@ -155,10 +155,40 @@ const PieceFormItem: React.FC = ({ "type" in schema && "widget" in schema && schema.type === "string" && - schema.widget === "codeeditor" + (schema.widget === "codeeditor" || schema.widget === "codeeditor-python") ) { inputElement = ( - name={`inputs.${itemKey}.value`} /> + + name={`inputs.${itemKey}.value`} + language="python" + placeholder="Enter Python code." + /> + ); + } else if ( + "type" in schema && + "widget" in schema && + schema.type === "string" && + schema.widget === "codeeditor-json" + ) { + inputElement = ( + + name={`inputs.${itemKey}.value`} + language="json" + placeholder="Enter JSON code." + /> + ); + } else if ( + "type" in schema && + "widget" in schema && + schema.type === "string" && + schema.widget === "codeeditor-sql" + ) { + inputElement = ( + + name={`inputs.${itemKey}.value`} + language="sql" + placeholder="Enter SQL code." + /> ); } else if ( "type" in schema && From f76c052ed1e459dcc8e05a5cd184e29038a879f9 Mon Sep 17 00:00:00 2001 From: Nathan Vieira Marcelino Date: Tue, 24 Oct 2023 09:00:45 -0300 Subject: [PATCH 02/16] fix: icons styles and result styles --- frontend/src/components/WorkflowPanel/DefaultNode/index.tsx | 4 ++-- frontend/src/components/WorkflowPanel/RunNode/index.tsx | 2 +- .../components/WorkflowDetail/CustomTabMenu/TaskResult.tsx | 1 + 3 files changed, 4 insertions(+), 3 deletions(-) diff --git a/frontend/src/components/WorkflowPanel/DefaultNode/index.tsx b/frontend/src/components/WorkflowPanel/DefaultNode/index.tsx index 6f351d66..e30c310a 100644 --- a/frontend/src/components/WorkflowPanel/DefaultNode/index.tsx +++ b/frontend/src/components/WorkflowPanel/DefaultNode/index.tsx @@ -71,7 +71,7 @@ export const CustomNode = memo(({ id, data, selected }) => { ...data.style.nodeStyle, display: "flex", flexDirection: "row", - justifyContent: "center", + justifyContent: "space-evenly", alignItems: "center", position: "relative", @@ -88,7 +88,7 @@ export const CustomNode = memo(({ id, data, selected }) => { backgroundColor: theme.palette.error.light, color: theme.palette.error.contrastText, }), - }; + } satisfies CSSProperties; }, [data, selected]); const { sourcePosition, targetPosition } = useMemo( diff --git a/frontend/src/components/WorkflowPanel/RunNode/index.tsx b/frontend/src/components/WorkflowPanel/RunNode/index.tsx index 6ee470b7..d3f844a4 100644 --- a/frontend/src/components/WorkflowPanel/RunNode/index.tsx +++ b/frontend/src/components/WorkflowPanel/RunNode/index.tsx @@ -88,7 +88,7 @@ const RunNode = memo(({ id, data, selected }) => { ...data.style.nodeStyle, display: "flex", flexDirection: "row", - justifyContent: "center", + justifyContent: "space-evenly", alignItems: "center", textAlign: "center", position: "relative", diff --git a/frontend/src/features/workflows/components/WorkflowDetail/CustomTabMenu/TaskResult.tsx b/frontend/src/features/workflows/components/WorkflowDetail/CustomTabMenu/TaskResult.tsx index 85f1f785..35314721 100644 --- a/frontend/src/features/workflows/components/WorkflowDetail/CustomTabMenu/TaskResult.tsx +++ b/frontend/src/features/workflows/components/WorkflowDetail/CustomTabMenu/TaskResult.tsx @@ -14,6 +14,7 @@ export const TaskResult = (props: ITaskResultProps) => { const style: CSSProperties = { height: "100%", + width: "100%", overflowY: "scroll", overflowX: "hidden", wordWrap: "break-word", From 1c2d2c30fdffdfcf30ce9c90a86d3e44eff1f900 Mon Sep 17 00:00:00 2001 From: Nathan Vieira Marcelino Date: Tue, 24 Oct 2023 11:23:42 -0300 Subject: [PATCH 03/16] feat: export workflow --- .../components/WorkflowEditor.tsx | 108 +++++++- .../context/workflowPiecesData.tsx | 2 +- .../context/workflowsEditor.tsx | 262 ++++++++++-------- frontend/src/utils/downloadJson.ts | 26 ++ frontend/src/utils/index.ts | 1 + frontend/tsconfig.json | 2 +- 6 files changed, 278 insertions(+), 123 deletions(-) create mode 100644 frontend/src/utils/downloadJson.ts diff --git a/frontend/src/features/workflowEditor/components/WorkflowEditor.tsx b/frontend/src/features/workflowEditor/components/WorkflowEditor.tsx index 58c21f8e..94855076 100644 --- a/frontend/src/features/workflowEditor/components/WorkflowEditor.tsx +++ b/frontend/src/features/workflowEditor/components/WorkflowEditor.tsx @@ -1,8 +1,9 @@ import { Settings as SettingsSuggestIcon } from "@mui/icons-material"; import ClearIcon from "@mui/icons-material/Clear"; import DownloadIcon from "@mui/icons-material/Download"; +import IosShareIcon from "@mui/icons-material/IosShare"; import SaveIcon from "@mui/icons-material/Save"; -import { Button, Grid, Paper } from "@mui/material"; +import { Button, Grid, Paper, styled } from "@mui/material"; import { AxiosError } from "axios"; import Loading from "components/Loading"; import { @@ -15,11 +16,12 @@ import { useWorkflowsEditor } from "features/workflowEditor/context"; import { type DragEvent, useCallback, useRef, useState } from "react"; import { toast } from "react-toastify"; import { type Edge, type Node, type XYPosition } from "reactflow"; -import { yupResolver, useInterval } from "utils"; +import { yupResolver, useInterval, exportToJson } from "utils"; import { v4 as uuidv4 } from "uuid"; import * as yup from "yup"; import { type IWorkflowPieceData, storageAccessModes } from "../context/types"; +import { type DominoWorkflowForage } from "../context/workflowsEditor"; import { containerResourcesSchema } from "../schemas/containerResourcesSchemas"; import { extractDefaultInputValues, extractDefaultValues } from "../utils"; @@ -42,6 +44,18 @@ const getId = (module_name: string) => { return `${module_name}_${uuidv4()}`; }; +const VisuallyHiddenInput = styled("input")({ + clip: "rect(0 0 0 0)", + clipPath: "inset(50%)", + height: 1, + overflow: "hidden", + position: "absolute", + bottom: 0, + left: 0, + whiteSpace: "nowrap", + width: 1, +}); + export const WorkflowsEditorComponent: React.FC = () => { const workflowPanelRef = useRef(null); const [sidebarSettingsDrawer, setSidebarSettingsDrawer] = useState(false); @@ -70,7 +84,7 @@ export const WorkflowsEditorComponent: React.FC = () => { const { clearForageData, - workflowsEditorBodyFromFlowchart, + generateWorkflowsEditorBodyParams, fetchWorkflowForage, handleCreateWorkflow, fetchForagePieceById, @@ -153,7 +167,7 @@ export const WorkflowsEditorComponent: React.FC = () => { await validateWorkflowPiecesData(payload); await validateWorkflowSettings(payload); - const data = await workflowsEditorBodyFromFlowchart(); + const data = await generateWorkflowsEditorBodyParams(payload); await handleCreateWorkflow({ workspace_id: workspace?.id, ...data }); @@ -175,7 +189,7 @@ export const WorkflowsEditorComponent: React.FC = () => { handleCreateWorkflow, validateWorkflowPiecesData, validateWorkflowSettings, - workflowsEditorBodyFromFlowchart, + generateWorkflowsEditorBodyParams, workspace?.id, ]); @@ -185,6 +199,76 @@ export const WorkflowsEditorComponent: React.FC = () => { workflowPanelRef.current?.setNodes([]); }, [clearForageData]); + const handleExport = useCallback(async () => { + await saveDataToLocalForage(); + const payload = await fetchWorkflowForage(); + exportToJson(payload, payload.workflowSettingsData?.config?.name); + }, []); + + const validateJsonImported = useCallback( + async (json: DominoWorkflowForage) => { + const getRepositories = function ( + workflowPieces: DominoWorkflowForage["workflowPieces"], + ) { + return [ + ...new Set( + Object.values(workflowPieces) + .reduce>((acc, next) => { + acc.push(next.source_image); + return acc; + }, []) + .filter((su) => !!su) as string[], + ), + ]; + }; + + const { workflowPieces } = await fetchWorkflowForage(); + + const currentRepositories = getRepositories(workflowPieces); + const incomeRepositories = getRepositories(json.workflowPieces); + const differences = currentRepositories.filter( + (x) => !incomeRepositories.includes(x), + ); + + return differences.length ? differences : null; + }, + [fetchWorkflowForage], + ); + + const handleImport = useCallback((e: React.ChangeEvent) => { + const file = e.target.files?.[0]; + + if (file) { + const reader = new FileReader(); + + reader.onload = (e) => { + try { + const jsonData = JSON.parse( + e.target?.result as string, + ) as DominoWorkflowForage; + + validateJsonImported(jsonData) + .then((diferences) => { + if (diferences) { + alert( + `Missing some repositories: ${JSON.stringify(diferences)}`, + ); + } else { + alert("Same itens"); + } + }) + .catch((e) => { + alert(e); + }); + } catch (error) { + console.error("Error parsing JSON file:", error); + } + }; + + reader.readAsText(file); + } + }, []); + const onNodesDelete = useCallback( async (nodes: any) => { for (const node of nodes) { @@ -344,11 +428,23 @@ export const WorkflowsEditorComponent: React.FC = () => { + + + + + + )} + + + + + + + ); + }, +); + +Modal.displayName = "Modal"; diff --git a/frontend/src/components/NewFeatureDialog/index.tsx b/frontend/src/components/NewFeatureDialog/index.tsx deleted file mode 100644 index 449507f7..00000000 --- a/frontend/src/components/NewFeatureDialog/index.tsx +++ /dev/null @@ -1,43 +0,0 @@ -import { - Button, - Dialog, - DialogActions, - DialogContent, - DialogContentText, - DialogTitle, - Grid, -} from "@mui/material"; -import React from "react"; - -interface Props { - isOpen: boolean; - confirmFn: () => void; -} - -export const NewFeatureDialog: React.FC = ({ isOpen, confirmFn }) => { - return ( - - New feature - - - This feature is not ready yet! We launch new versions every time, - check out our changelog for more information ! - - - - - - - - - - - ); -}; diff --git a/frontend/src/features/workflowEditor/components/SidebarForm/index.tsx b/frontend/src/features/workflowEditor/components/SidebarForm/index.tsx index 936a57f5..dae41c48 100644 --- a/frontend/src/features/workflowEditor/components/SidebarForm/index.tsx +++ b/frontend/src/features/workflowEditor/components/SidebarForm/index.tsx @@ -33,7 +33,7 @@ const SidebarPieceForm: React.FC = (props) => { const { schema, formId, open, onClose, title } = props; const { - setForageWorkflowPiecesData, + setForageWorkflowPiecesDataById, fetchForageWorkflowPiecesDataById, setForageWorkflowPiecesOutputSchema, clearDownstreamDataById, @@ -141,10 +141,10 @@ const SidebarPieceForm: React.FC = (props) => { const saveData = useCallback(async () => { if (formId && open) { - await setForageWorkflowPiecesData(formId, data as IWorkflowPieceData); + await setForageWorkflowPiecesDataById(formId, data as IWorkflowPieceData); await updateOutputSchema(); } - }, [formId, open, setForageWorkflowPiecesData, data, updateOutputSchema]); + }, [formId, open, setForageWorkflowPiecesDataById, data, updateOutputSchema]); // load forage useEffect(() => { diff --git a/frontend/src/features/workflowEditor/components/WorkflowEditor.tsx b/frontend/src/features/workflowEditor/components/WorkflowEditor.tsx index 94855076..599dffc6 100644 --- a/frontend/src/features/workflowEditor/components/WorkflowEditor.tsx +++ b/frontend/src/features/workflowEditor/components/WorkflowEditor.tsx @@ -6,6 +6,7 @@ import SaveIcon from "@mui/icons-material/Save"; import { Button, Grid, Paper, styled } from "@mui/material"; import { AxiosError } from "axios"; import Loading from "components/Loading"; +import { Modal, type ModalRef } from "components/Modal"; import { type WorkflowPanelRef, WorkflowPanel, @@ -16,6 +17,7 @@ import { useWorkflowsEditor } from "features/workflowEditor/context"; import { type DragEvent, useCallback, useRef, useState } from "react"; import { toast } from "react-toastify"; import { type Edge, type Node, type XYPosition } from "reactflow"; +import localForage from "services/config/localForage.config"; import { yupResolver, useInterval, exportToJson } from "utils"; import { v4 as uuidv4 } from "uuid"; import * as yup from "yup"; @@ -69,6 +71,9 @@ export const WorkflowsEditorComponent: React.FC = () => { "horizontal", ); + const incompatiblePiecesModalRef = useRef(null); + const [incompatiblesPieces, setIncompatiblesPieces] = useState([]); + const { workspace } = useWorkspaces(); const saveDataToLocalForage = useCallback(async () => { @@ -95,7 +100,8 @@ export const WorkflowsEditorComponent: React.FC = () => { removeForageWorkflowPiecesById, removeForageWorkflowPieceDataById, fetchWorkflowPieceById, - setForageWorkflowPiecesData, + setForageWorkflowPiecesDataById, + importWorkflowToForage, clearDownstreamDataById, setWorkflowEdges, setWorkflowNodes, @@ -222,12 +228,17 @@ export const WorkflowsEditorComponent: React.FC = () => { ]; }; - const { workflowPieces } = await fetchWorkflowForage(); - - const currentRepositories = getRepositories(workflowPieces); + const currentRepositories = [ + ...new Set( + Object.values((await localForage.getItem("pieces")) as any)?.map( + (p: any) => p?.source_image, + ), + ), + ]; const incomeRepositories = getRepositories(json.workflowPieces); - const differences = currentRepositories.filter( - (x) => !incomeRepositories.includes(x), + + const differences = incomeRepositories.filter( + (x) => !currentRepositories.includes(x), ); return differences.length ? differences : null; @@ -235,39 +246,53 @@ export const WorkflowsEditorComponent: React.FC = () => { [fetchWorkflowForage], ); - const handleImport = useCallback((e: React.ChangeEvent) => { - const file = e.target.files?.[0]; - - if (file) { - const reader = new FileReader(); - - reader.onload = (e) => { - try { - const jsonData = JSON.parse( - e.target?.result as string, - ) as DominoWorkflowForage; - - validateJsonImported(jsonData) - .then((diferences) => { - if (diferences) { - alert( - `Missing some repositories: ${JSON.stringify(diferences)}`, - ); - } else { - alert("Same itens"); - } - }) - .catch((e) => { - alert(e); - }); - } catch (error) { - console.error("Error parsing JSON file:", error); - } - }; - - reader.readAsText(file); - } - }, []); + const handleImport = useCallback( + (e: React.ChangeEvent) => { + const file = e.target.files?.[0]; + + if (file) { + const reader = new FileReader(); + + reader.onload = (e) => { + try { + const jsonData = JSON.parse( + e.target?.result as string, + ) as DominoWorkflowForage; + + validateJsonImported(jsonData) + .then((diferences) => { + if (diferences) { + toast.error( + "Some repositories are missing or incompatible version", + ); + setIncompatiblesPieces(diferences); + incompatiblePiecesModalRef.current?.open(); + return; + } + + workflowPanelRef?.current?.setNodes(jsonData.workflowNodes); + workflowPanelRef?.current?.setEdges(jsonData.workflowEdges); + void importWorkflowToForage(jsonData); + }) + .catch((e) => { + console.log(e); + }); + } catch (error) { + console.error("Error parsing JSON file:", error); + } + }; + + reader.readAsText(file); + } + }, + [ + validateJsonImported, + workflowPanelRef, + importWorkflowToForage, + setIncompatiblesPieces, + incompatiblePiecesModalRef, + ], + ); const onNodesDelete = useCallback( async (nodes: any) => { @@ -351,7 +376,10 @@ export const WorkflowsEditorComponent: React.FC = () => { inputs: defaultInputs, }; - await setForageWorkflowPiecesData(newNode.id, defaultWorkflowPieceData); + await setForageWorkflowPiecesDataById( + newNode.id, + defaultWorkflowPieceData, + ); return newNode; }, [ @@ -359,7 +387,7 @@ export const WorkflowsEditorComponent: React.FC = () => { fetchForagePieceById, setForageWorkflowPieces, getForageWorkflowPieces, - setForageWorkflowPiecesData, + setForageWorkflowPiecesDataById, ], ); @@ -442,6 +470,17 @@ export const WorkflowsEditorComponent: React.FC = () => { > Import + + {incompatiblesPieces.map((item) => ( +
  • {item}
    • + ))} + + } + ref={incompatiblePiecesModalRef} + /> diff --git a/frontend/src/features/workflowEditor/context/workflowPiecesData.tsx b/frontend/src/features/workflowEditor/context/workflowPiecesData.tsx index c3e7e035..578ab218 100644 --- a/frontend/src/features/workflowEditor/context/workflowPiecesData.tsx +++ b/frontend/src/features/workflowEditor/context/workflowPiecesData.tsx @@ -7,10 +7,11 @@ import { type IWorkflowPieceData } from "./types"; export type ForagePiecesData = Record; export interface IWorkflowPiecesDataContext { - setForageWorkflowPiecesData: ( + setForageWorkflowPiecesDataById: ( id: string, pieceData: IWorkflowPieceData, ) => Promise; + setForageWorkflowPiecesData: (pieceData: ForagePiecesData) => Promise; fetchForageWorkflowPiecesData: () => Promise; fetchForageWorkflowPiecesDataById: ( id: string, @@ -26,7 +27,7 @@ export const [WorkflowPiecesDataContext, useWorkflowPiecesData] = const WorkflowPiecesDataProvider: React.FC<{ children: React.ReactNode }> = ({ children, }) => { - const setForageWorkflowPiecesData = useCallback( + const setForageWorkflowPiecesDataById = useCallback( async (id: string, pieceData: IWorkflowPieceData) => { let currentData = await localForage.getItem("workflowPiecesData"); @@ -38,6 +39,12 @@ const WorkflowPiecesDataProvider: React.FC<{ children: React.ReactNode }> = ({ }, [], ); + const setForageWorkflowPiecesData = useCallback( + async (pieceData: ForagePiecesData) => { + await localForage.setItem("workflowPiecesData", pieceData); + }, + [], + ); const fetchForageWorkflowPiecesData = useCallback(async () => { const workflowPiecesData = @@ -120,6 +127,7 @@ const WorkflowPiecesDataProvider: React.FC<{ children: React.ReactNode }> = ({ }, []); const value: IWorkflowPiecesDataContext = { + setForageWorkflowPiecesDataById, setForageWorkflowPiecesData, fetchForageWorkflowPiecesData, fetchForageWorkflowPiecesDataById, diff --git a/frontend/src/features/workflowEditor/context/workflowsEditor.tsx b/frontend/src/features/workflowEditor/context/workflowsEditor.tsx index 02e2f8a0..bc87266e 100644 --- a/frontend/src/features/workflowEditor/context/workflowsEditor.tsx +++ b/frontend/src/features/workflowEditor/context/workflowsEditor.tsx @@ -50,10 +50,13 @@ interface IWorkflowsEditorContext IWorkflowSettingsContext, IWorkflowPieceContext, IWorkflowPiecesDataContext { - fetchWorkflowForage: () => Promise; // TODO add type + fetchWorkflowForage: () => Promise; + importWorkflowToForage: ( + importedWorkflow: DominoWorkflowForage, + ) => Promise; generateWorkflowsEditorBodyParams: ( p: GenerateWorkflowsParams, - ) => Promise; // TODO add type + ) => Promise; handleCreateWorkflow: ( params: IPostWorkflowParams, ) => Promise; @@ -101,6 +104,7 @@ const WorkflowsEditorProvider: FC<{ children?: React.ReactNode }> = ({ fetchForageWorkflowPiecesData, fetchForageWorkflowPiecesDataById, setForageWorkflowPiecesData, + setForageWorkflowPiecesDataById, clearForageWorkflowPiecesData, removeForageWorkflowPieceDataById, clearDownstreamDataById, @@ -142,6 +146,23 @@ const WorkflowsEditorProvider: FC<{ children?: React.ReactNode }> = ({ getForageWorkflowPieces, ]); + const importWorkflowToForage = useCallback( + async (dominoWorkflow: DominoWorkflowForage) => { + await setForageWorkflowPieces(dominoWorkflow.workflowPieces); + await setForageWorkflowPiecesData(dominoWorkflow.workflowPiecesData); + await setWorkflowSettingsData(dominoWorkflow.workflowSettingsData); + await setWorkflowNodes(dominoWorkflow.workflowNodes); + await setWorkflowEdges(dominoWorkflow.workflowEdges); + }, + [ + setForageWorkflowPieces, + setForageWorkflowPiecesData, + setWorkflowSettingsData, + setWorkflowNodes, + setWorkflowEdges, + ], + ); + const generateWorkflowsEditorBodyParams = useCallback( async ({ workflowPiecesData, @@ -303,6 +324,8 @@ const WorkflowsEditorProvider: FC<{ children?: React.ReactNode }> = ({ search, handleSearch, + importWorkflowToForage, + setWorkflowEdges, setWorkflowNodes, fetchForageWorkflowEdges, @@ -317,6 +340,7 @@ const WorkflowsEditorProvider: FC<{ children?: React.ReactNode }> = ({ clearForageWorkflowPieces, setForageWorkflowPiecesData, + setForageWorkflowPiecesDataById, fetchForageWorkflowPiecesData, fetchForageWorkflowPiecesDataById, removeForageWorkflowPieceDataById, diff --git a/frontend/src/features/workflows/components/WorkflowsList/Actions.tsx b/frontend/src/features/workflows/components/WorkflowsList/Actions.tsx index 830f2ce5..883e01b4 100644 --- a/frontend/src/features/workflows/components/WorkflowsList/Actions.tsx +++ b/frontend/src/features/workflows/components/WorkflowsList/Actions.tsx @@ -3,10 +3,10 @@ import PauseCircleOutlineIcon from "@mui/icons-material/PauseCircleOutline"; import PlayCircleOutlineIcon from "@mui/icons-material/PlayCircleOutline"; import { IconButton } from "@mui/material"; import { type CommonProps } from "@mui/material/OverridableComponent"; -import { NewFeatureDialog } from "components/NewFeatureDialog"; +import { Modal, type ModalRef } from "components/Modal"; import { type IWorkflow } from "features/workflows/types"; import theme from "providers/theme.config"; -import React, { useState } from "react"; +import React, { useRef, useState } from "react"; import { ConfirmDeleteModal } from "./ConfirmDeleteModal"; @@ -19,7 +19,7 @@ interface Props extends CommonProps { export const Actions: React.FC = ({ runFn, deleteFn, className }) => { const [deleteModalOpen, setDeleteModalOpen] = useState(false); - const [newFeatureModal, setNewFeatureModal] = useState(false); + const newFeatureModal = useRef(null); return ( <> @@ -31,7 +31,7 @@ export const Actions: React.FC = ({ runFn, deleteFn, className }) => { { - setNewFeatureModal(true); + newFeatureModal.current?.open(); }} > = ({ runFn, deleteFn, className }) => { style={{ pointerEvents: "none", color: theme.palette.error.main }} /> - { - setNewFeatureModal(false); - }} + { ); const handleRowClick = useCallback>( - (row, event) => { + (params: GridRowParams, event) => { const isActionButtonClick = event.target instanceof Element && event.target.classList.contains(".action-button"); if (!isActionButtonClick) { - // Handle row click logic only if it's not an action button click - navigate(`/workflows/${row.id}`); + if (params.row.status !== "failed" && params.row.status !== "creating") + navigate(`/workflows/${params.id}`); } }, [navigate], @@ -166,6 +167,9 @@ export const WorkflowList: React.FC = () => { density="comfortable" columns={columns} rows={rows} + isRowSelectable={(params) => + params.row.status !== "failed" && params.row.status !== "creating" + } onRowClick={handleRowClick} pagination paginationMode="server" From 017f8bd68a36bb8cfe0163d66cee9024469e3a2e Mon Sep 17 00:00:00 2001 From: Nathan Vieira Marcelino Date: Fri, 27 Oct 2023 12:54:38 -0300 Subject: [PATCH 05/16] fix: clear state to trigger onChange if import same file --- .../workflowEditor/components/WorkflowEditor.tsx | 13 ++++++++++++- 1 file changed, 12 insertions(+), 1 deletion(-) diff --git a/frontend/src/features/workflowEditor/components/WorkflowEditor.tsx b/frontend/src/features/workflowEditor/components/WorkflowEditor.tsx index 599dffc6..c9e90e3f 100644 --- a/frontend/src/features/workflowEditor/components/WorkflowEditor.tsx +++ b/frontend/src/features/workflowEditor/components/WorkflowEditor.tsx @@ -246,6 +246,8 @@ export const WorkflowsEditorComponent: React.FC = () => { [fetchWorkflowForage], ); + const fileInputRef = useRef(null); + const handleImport = useCallback( (e: React.ChangeEvent) => { const file = e.target.files?.[0]; @@ -277,6 +279,10 @@ export const WorkflowsEditorComponent: React.FC = () => { .catch((e) => { console.log(e); }); + + if (fileInputRef.current) { + fileInputRef.current.value = ""; + } } catch (error) { console.error("Error parsing JSON file:", error); } @@ -291,6 +297,7 @@ export const WorkflowsEditorComponent: React.FC = () => { importWorkflowToForage, setIncompatiblesPieces, incompatiblePiecesModalRef, + fileInputRef, ], ); @@ -469,7 +476,11 @@ export const WorkflowsEditorComponent: React.FC = () => { startIcon={} > Import - + Date: Mon, 30 Oct 2023 10:04:59 -0300 Subject: [PATCH 06/16] fix(workflow settings): settings onLoad load workflow settings data from/to forage without the need to open sidebar --- .../components/SidebarSettingsForm/index.tsx | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/frontend/src/features/workflowEditor/components/SidebarSettingsForm/index.tsx b/frontend/src/features/workflowEditor/components/SidebarSettingsForm/index.tsx index 1c6a29b5..0fa4131b 100644 --- a/frontend/src/features/workflowEditor/components/SidebarSettingsForm/index.tsx +++ b/frontend/src/features/workflowEditor/components/SidebarSettingsForm/index.tsx @@ -123,13 +123,15 @@ const SidebarSettingsForm = (props: ISidebarSettingsFormProps) => { const loadData = useCallback(async () => { const data = await fetchWorkflowSettingsData(); + console.log("aq"); if (Object.keys(data).length === 0) { reset(defaultSettingsData); + await setWorkflowSettingsData(defaultSettingsData); } else { reset(data); } setLoaded(true); - }, [reset, fetchWorkflowSettingsData]); + }, [reset, fetchWorkflowSettingsData, setWorkflowSettingsData]); const saveData = useCallback(async () => { if (open) { @@ -138,10 +140,8 @@ const SidebarSettingsForm = (props: ISidebarSettingsFormProps) => { }, [formData, open, setWorkflowSettingsData]); useEffect(() => { - if (open) { - void loadData(); - } - }, [open, loadData]); + void loadData(); + }, [loadData]); useEffect(() => { void saveData(); From 931142d08c2ac49cececdf7fa2aeced0755d6315 Mon Sep 17 00:00:00 2001 From: Vinicius Vaz Date: Mon, 30 Oct 2023 10:59:40 -0300 Subject: [PATCH 07/16] using forward ref in sidebar settings to use onClear --- .../components/SidebarSettingsForm/index.tsx | 345 ++++++++++-------- .../components/WorkflowEditor.tsx | 7 +- 2 files changed, 193 insertions(+), 159 deletions(-) diff --git a/frontend/src/features/workflowEditor/components/SidebarSettingsForm/index.tsx b/frontend/src/features/workflowEditor/components/SidebarSettingsForm/index.tsx index 0fa4131b..f90879a2 100644 --- a/frontend/src/features/workflowEditor/components/SidebarSettingsForm/index.tsx +++ b/frontend/src/features/workflowEditor/components/SidebarSettingsForm/index.tsx @@ -15,7 +15,14 @@ import { storageSourcesAWS, storageSourcesLocal, } from "features/workflowEditor/context/types"; -import { useCallback, useEffect, useState } from "react"; +import { + useCallback, + useEffect, + useState, + forwardRef, + useImperativeHandle, + type ForwardedRef, +} from "react"; import { FormProvider, useForm } from "react-hook-form"; import { yupResolver } from "utils"; import * as yup from "yup"; @@ -100,181 +107,203 @@ export const WorkflowSettingsFormSchema: ValidationSchema = yup.object().shape({ }), }); -const SidebarSettingsForm = (props: ISidebarSettingsFormProps) => { - const { open, onClose } = props; +export interface SidebarSettingsFormRef { + loadData: () => Promise; +} - const { fetchWorkflowSettingsData, setWorkflowSettingsData } = - useWorkflowsEditor(); +const SidebarSettingsForm = forwardRef< + SidebarSettingsFormRef, + ISidebarSettingsFormProps +>( + ( + props: ISidebarSettingsFormProps, + ref: ForwardedRef, + ) => { + const { open, onClose } = props; - const resolver = yupResolver(WorkflowSettingsFormSchema); - const methods = useForm({ mode: "onChange", resolver }); - const { register, watch, reset, trigger, getValues } = methods; - const formData = watch(); + const { fetchWorkflowSettingsData, setWorkflowSettingsData } = + useWorkflowsEditor(); - const [loaded, setLoaded] = useState(false); + const resolver = yupResolver(WorkflowSettingsFormSchema); + const methods = useForm({ mode: "onChange", resolver }); + const { register, watch, reset, trigger, getValues } = methods; + const formData = watch(); - const validate = useCallback(() => { - if (loaded) void trigger(); - }, [loaded, trigger]); + const [loaded, setLoaded] = useState(false); - useEffect(() => { - validate(); - }, [validate]); + const validate = useCallback(() => { + if (loaded) void trigger(); + }, [loaded, trigger]); - const loadData = useCallback(async () => { - const data = await fetchWorkflowSettingsData(); - console.log("aq"); - if (Object.keys(data).length === 0) { - reset(defaultSettingsData); - await setWorkflowSettingsData(defaultSettingsData); - } else { - reset(data); - } - setLoaded(true); - }, [reset, fetchWorkflowSettingsData, setWorkflowSettingsData]); + useEffect(() => { + validate(); + }, [validate]); - const saveData = useCallback(async () => { - if (open) { - await setWorkflowSettingsData(formData); - } - }, [formData, open, setWorkflowSettingsData]); + const loadData = useCallback(async () => { + const data = await fetchWorkflowSettingsData(); + if (Object.keys(data).length === 0) { + reset(defaultSettingsData); + await setWorkflowSettingsData(defaultSettingsData); + } else { + reset(data); + } + setLoaded(true); + }, [reset, fetchWorkflowSettingsData, setWorkflowSettingsData]); - useEffect(() => { - void loadData(); - }, [loadData]); + const saveData = useCallback(async () => { + if (open) { + await setWorkflowSettingsData(formData); + } + }, [formData, open, setWorkflowSettingsData]); - useEffect(() => { - void saveData(); - }, [saveData]); + useEffect(() => { + void loadData(); + }, [loadData]); - if (Object.keys(formData).length === 0) { - return null; - } + useEffect(() => { + void saveData(); + }, [saveData]); - return ( - - - - - Settings - - - - - - - - - - - - - - - - - {getValues().config.endDateType === - endDateTypes.UserDefined && ( - - - - )} - - - - - - + useImperativeHandle( + ref, + () => { + return { + loadData, + }; + }, + [loadData], + ); + + if (Object.keys(formData).length === 0) { + return null; + } + + return ( + + + - Storage + Settings - - - - - - - {formData.storage.storageSource === storageSourcesAWS.AWS_S3 ? ( - <> - - - - - + + + + + + + + + + + + + - - ) : null} + {getValues().config.endDateType === + endDateTypes.UserDefined && ( + + + + )} + + + + + + + + Storage + - + + + + + + {formData.storage.storageSource === storageSourcesAWS.AWS_S3 ? ( + <> + + + + + + + + ) : null} + + + - - - ); -}; -export default SidebarSettingsForm; + + ); + }, +); +SidebarSettingsForm.displayName = "SidebarSettingsForm"; +export { SidebarSettingsForm }; diff --git a/frontend/src/features/workflowEditor/components/WorkflowEditor.tsx b/frontend/src/features/workflowEditor/components/WorkflowEditor.tsx index c9e90e3f..02b4a55b 100644 --- a/frontend/src/features/workflowEditor/components/WorkflowEditor.tsx +++ b/frontend/src/features/workflowEditor/components/WorkflowEditor.tsx @@ -32,8 +32,10 @@ import SidebarPieceForm from "./SidebarForm"; import { ContainerResourceFormSchema } from "./SidebarForm/ContainerResourceForm"; import { createInputsSchemaValidation } from "./SidebarForm/PieceForm/validation"; import { storageFormSchema } from "./SidebarForm/StorageForm"; -import SidebarSettingsForm, { +import { + SidebarSettingsForm, WorkflowSettingsFormSchema, + type SidebarSettingsFormRef, } from "./SidebarSettingsForm"; /** @@ -60,6 +62,7 @@ const VisuallyHiddenInput = styled("input")({ export const WorkflowsEditorComponent: React.FC = () => { const workflowPanelRef = useRef(null); + const sidebarSettingsRef = useRef(null); const [sidebarSettingsDrawer, setSidebarSettingsDrawer] = useState(false); const [sidebarPieceDrawer, setSidebarPieceDrawer] = useState(false); const [formId, setFormId] = useState(""); @@ -203,6 +206,7 @@ export const WorkflowsEditorComponent: React.FC = () => { await clearForageData(); workflowPanelRef.current?.setEdges([]); workflowPanelRef.current?.setNodes([]); + await sidebarSettingsRef.current?.loadData(); }, [clearForageData]); const handleExport = useCallback(async () => { @@ -538,6 +542,7 @@ export const WorkflowsEditorComponent: React.FC = () => { ); From 7fa36109bb0799bb612369757a7b947b3f253b22 Mon Sep 17 00:00:00 2001 From: Vinicius Vaz Date: Mon, 30 Oct 2023 11:25:25 -0300 Subject: [PATCH 08/16] error message on export empty workflow --- .../src/features/workflowEditor/components/WorkflowEditor.tsx | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/frontend/src/features/workflowEditor/components/WorkflowEditor.tsx b/frontend/src/features/workflowEditor/components/WorkflowEditor.tsx index 02b4a55b..0e6c4fa6 100644 --- a/frontend/src/features/workflowEditor/components/WorkflowEditor.tsx +++ b/frontend/src/features/workflowEditor/components/WorkflowEditor.tsx @@ -212,6 +212,10 @@ export const WorkflowsEditorComponent: React.FC = () => { const handleExport = useCallback(async () => { await saveDataToLocalForage(); const payload = await fetchWorkflowForage(); + if (Object.keys(payload.workflowPieces).length === 0) { + toast.error("Workflow must have at least one piece to be exported."); + return; + } exportToJson(payload, payload.workflowSettingsData?.config?.name); }, []); From 3b3c6d6a23f572e9be332c6bcb8a8bae5755756a Mon Sep 17 00:00:00 2001 From: Vinicius Vaz Date: Mon, 30 Oct 2023 11:33:33 -0300 Subject: [PATCH 09/16] update warning msg for missing pieces repos --- .../workflowEditor/components/WorkflowEditor.tsx | 11 +++++++++-- 1 file changed, 9 insertions(+), 2 deletions(-) diff --git a/frontend/src/features/workflowEditor/components/WorkflowEditor.tsx b/frontend/src/features/workflowEditor/components/WorkflowEditor.tsx index 0e6c4fa6..c96cd57e 100644 --- a/frontend/src/features/workflowEditor/components/WorkflowEditor.tsx +++ b/frontend/src/features/workflowEditor/components/WorkflowEditor.tsx @@ -228,7 +228,14 @@ export const WorkflowsEditorComponent: React.FC = () => { ...new Set( Object.values(workflowPieces) .reduce>((acc, next) => { - acc.push(next.source_image); + acc.push( + `${next.source_image.split("ghcr.io/")[1].split(":")[0]}:${ + next.source_image + .split("ghcr.io/")[1] + .split(":")[1] + .split("-")[0] + }`, + ); return acc; }, []) .filter((su) => !!su) as string[], @@ -490,7 +497,7 @@ export const WorkflowsEditorComponent: React.FC = () => { ref={fileInputRef} /> {incompatiblesPieces.map((item) => ( From 27b2f49d7c13421c85c176e02ad7d962248b4116 Mon Sep 17 00:00:00 2001 From: Vinicius Vaz Date: Mon, 30 Oct 2023 13:52:41 -0300 Subject: [PATCH 10/16] improve missing secrets message --- rest/services/secret_service.py | 1 + rest/services/workflow_service.py | 2 +- 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/rest/services/secret_service.py b/rest/services/secret_service.py index 9bf276c3..c43f9bb4 100644 --- a/rest/services/secret_service.py +++ b/rest/services/secret_service.py @@ -131,4 +131,5 @@ def get_piece_secrets( value=decoded_value ) ) + return response \ No newline at end of file diff --git a/rest/services/workflow_service.py b/rest/services/workflow_service.py index 9706ba0b..76fda390 100644 --- a/rest/services/workflow_service.py +++ b/rest/services/workflow_service.py @@ -349,7 +349,7 @@ def _validate_workflow_tasks(self, tasks_dict: dict, workspace_id: int): ) for secret in piece_secrets: if not secret.value: - raise ResourceNotFoundException(f"Piece {piece.name} have missing secrets.") + raise ResourceNotFoundException(f"Secret {secret.name} missing for {piece.name}.") return True def _get_storage_repository_from_tasks(self, tasks: list, workspace_id: int): From e03597ad8e46fc6e6da3c6390170bbd9cde00835 Mon Sep 17 00:00:00 2001 From: Vinicius Vaz Date: Mon, 30 Oct 2023 14:36:30 -0300 Subject: [PATCH 11/16] update schedule_interval to schedule and force all datetime objects to use utc tz --- .../components/SidebarSettingsForm/index.tsx | 2 +- .../context/types/workflowPieceData.ts | 2 +- .../context/workflowsEditor.tsx | 2 +- .../components/WorkflowsList/index.tsx | 15 +++++- .../src/features/workflows/types/workflow.ts | 4 +- .../alembic/versions/758034ba1a89_.py | 30 ++++++++++++ rest/database/models/workflow.py | 2 +- rest/populate_db.py | 20 -------- rest/repository/workflow_repository.py | 2 +- rest/schemas/requests/workflow.py | 4 +- rest/schemas/responses/workflow.py | 46 +++++++++++++------ rest/services/workflow_service.py | 34 ++++++++------ .../workflow/create_workflow_request_model.py | 2 +- rest/tests/workflow/test_workflow_router.py | 2 +- 14 files changed, 106 insertions(+), 61 deletions(-) create mode 100644 rest/database/alembic/versions/758034ba1a89_.py delete mode 100644 rest/populate_db.py diff --git a/frontend/src/features/workflowEditor/components/SidebarSettingsForm/index.tsx b/frontend/src/features/workflowEditor/components/SidebarSettingsForm/index.tsx index f90879a2..b31cab6d 100644 --- a/frontend/src/features/workflowEditor/components/SidebarSettingsForm/index.tsx +++ b/frontend/src/features/workflowEditor/components/SidebarSettingsForm/index.tsx @@ -217,7 +217,7 @@ const SidebarSettingsForm = forwardRef< name="config.scheduleInterval" defaultValue={defaultSettingsData.config.scheduleInterval} options={Object.values(scheduleIntervals)} - label="Schedule Interval" + label="Schedule" /> diff --git a/frontend/src/features/workflowEditor/context/types/workflowPieceData.ts b/frontend/src/features/workflowEditor/context/types/workflowPieceData.ts index 0423aebc..73e76448 100644 --- a/frontend/src/features/workflowEditor/context/types/workflowPieceData.ts +++ b/frontend/src/features/workflowEditor/context/types/workflowPieceData.ts @@ -21,7 +21,7 @@ interface WorkflowBaseSettings { name: string; start_date: string; // ISOFormat select_end_date: EndDateTypes; - schedule_interval: ScheduleIntervals; + schedule: ScheduleIntervals; end_date?: string; // ISOFormat catchup?: boolean; diff --git a/frontend/src/features/workflowEditor/context/workflowsEditor.tsx b/frontend/src/features/workflowEditor/context/workflowsEditor.tsx index bc87266e..2a430b6f 100644 --- a/frontend/src/features/workflowEditor/context/workflowsEditor.tsx +++ b/frontend/src/features/workflowEditor/context/workflowsEditor.tsx @@ -172,7 +172,7 @@ const WorkflowsEditorProvider: FC<{ children?: React.ReactNode }> = ({ }: GenerateWorkflowsParams) => { const workflow: CreateWorkflowRequest["workflow"] = { name: workflowSettingsData.config.name, - schedule_interval: workflowSettingsData.config.scheduleInterval, + schedule: workflowSettingsData.config.scheduleInterval, select_end_date: workflowSettingsData.config.endDateType, start_date: workflowSettingsData.config.startDate, end_date: workflowSettingsData.config.endDate, diff --git a/frontend/src/features/workflows/components/WorkflowsList/index.tsx b/frontend/src/features/workflows/components/WorkflowsList/index.tsx index 1b80754b..50fa55f9 100644 --- a/frontend/src/features/workflows/components/WorkflowsList/index.tsx +++ b/frontend/src/features/workflows/components/WorkflowsList/index.tsx @@ -40,6 +40,7 @@ export const WorkflowList: React.FC = () => { paginationModel.page, paginationModel.pageSize, ); + const handleDeleteWorkflow = useAuthenticatedDeleteWorkflowId(); const handleRunWorkflow = useAuthenticatedPostWorkflowRunId(); @@ -107,12 +108,22 @@ export const WorkflowList: React.FC = () => { headerAlign: "center", }, { - field: "schedule_interval", - headerName: "Schedule Interval", + field: "schedule", + headerName: "Schedule", + flex: 1, + align: "center", + headerAlign: "center", + sortable: false, + }, + { + field: "next_dagrun", + headerName: "Next Run", flex: 1, align: "center", headerAlign: "center", sortable: false, + valueFormatter: ({ value }) => + value ? new Date(value).toLocaleString() : "none", }, { field: "actions", diff --git a/frontend/src/features/workflows/types/workflow.ts b/frontend/src/features/workflows/types/workflow.ts index 0065ac5e..d6f52690 100644 --- a/frontend/src/features/workflows/types/workflow.ts +++ b/frontend/src/features/workflows/types/workflow.ts @@ -30,7 +30,7 @@ export interface IWorkflow { status: workflowStatus; is_subdag: boolean; last_pickled: string; - schedule_interval: string; + schedule: string; max_active_tasks: number; max_active_runs: number; has_task_concurrency_limits: boolean; @@ -61,7 +61,7 @@ export interface IWorkflowConfig { name: string; start_date: string; end_date: string; - schedule_interval: string; + schedule: string; catchup: boolean; generate_report: string; description: string; diff --git a/rest/database/alembic/versions/758034ba1a89_.py b/rest/database/alembic/versions/758034ba1a89_.py new file mode 100644 index 00000000..6a2f312b --- /dev/null +++ b/rest/database/alembic/versions/758034ba1a89_.py @@ -0,0 +1,30 @@ +"""empty message + +Revision ID: 758034ba1a89 +Revises: c25e53090da7 +Create Date: 2023-10-30 13:58:20.061180 + +""" +from alembic import op +import sqlalchemy as sa +from sqlalchemy.dialects import postgresql + +# revision identifiers, used by Alembic. +revision = '758034ba1a89' +down_revision = 'c25e53090da7' +branch_labels = None +depends_on = None + + +def upgrade(): + # ### commands auto generated by Alembic - please adjust! ### + op.add_column('workflow', sa.Column('schedule', sa.Enum('none', 'once', 'hourly', 'daily', 'weekly', 'monthly', 'yearly', 'Config', name='workflowscheduleinterval'), nullable=True)) + op.drop_column('workflow', 'schedule_interval') + # ### end Alembic commands ### + + +def downgrade(): + # ### commands auto generated by Alembic - please adjust! ### + op.add_column('workflow', sa.Column('schedule_interval', postgresql.ENUM('none', 'once', 'hourly', 'daily', 'weekly', 'monthly', 'yearly', 'Config', name='workflowscheduleinterval'), autoincrement=False, nullable=True)) + op.drop_column('workflow', 'schedule') + # ### end Alembic commands ### diff --git a/rest/database/models/workflow.py b/rest/database/models/workflow.py index 1b8069bc..7da4620b 100644 --- a/rest/database/models/workflow.py +++ b/rest/database/models/workflow.py @@ -18,7 +18,7 @@ class Workflow(Base): last_changed_at = Column(DateTime, nullable=False, default=datetime.utcnow) start_date = Column(DateTime, nullable=True) end_date = Column(DateTime, nullable=True) - schedule_interval = Column(Enum(WorkflowScheduleInterval), nullable=True, default=WorkflowScheduleInterval.none.value) + schedule = Column(Enum(WorkflowScheduleInterval), nullable=True, default=WorkflowScheduleInterval.none.value) workspace_id = Column(Integer, ForeignKey("workspace.id", ondelete='cascade'), nullable=False) last_changed_by = Column(Integer, ForeignKey("user.id"), nullable=False) diff --git a/rest/populate_db.py b/rest/populate_db.py deleted file mode 100644 index f185b8f2..00000000 --- a/rest/populate_db.py +++ /dev/null @@ -1,20 +0,0 @@ -import argparse -from sqlalchemy import create_engine - - -if __name__ == "__main__": - # Parse command-line arguments - parser = argparse.ArgumentParser() - parser.add_argument("--user", required=True) - parser.add_argument("--password", required=True) - parser.add_argument("--host", required=True) - args = parser.parse_args() - - # Create database engine - engine = create_engine(f"postgresql://{args.user}:{args.password}@{args.host}/mydb") - - # Write initial data to the database using SQLAlchemy - with engine.connect() as conn: - conn.execute("INSERT INTO users (username, password) VALUES ('user1', 'password1')") - conn.execute("INSERT INTO users (username, password) VALUES ('user2', 'password2')") - conn.execute("INSERT INTO users (username, password) VALUES ('user3', 'password3')") diff --git a/rest/repository/workflow_repository.py b/rest/repository/workflow_repository.py index 8083fa9c..1d766977 100644 --- a/rest/repository/workflow_repository.py +++ b/rest/repository/workflow_repository.py @@ -131,7 +131,7 @@ def update(self, workflow: Workflow): saved_workflow.last_changed_at = workflow.last_changed_at saved_workflow.start_date = workflow.start_date saved_workflow.end_date = workflow.end_date - saved_workflow.schedule_interval = workflow.schedule_interval + saved_workflow.schedule = workflow.schedule saved_workflow.last_changed_by = workflow.last_changed_by session.flush() session.expunge(saved_workflow) diff --git a/rest/schemas/requests/workflow.py b/rest/schemas/requests/workflow.py index 671c1cbe..c3dd4fa3 100644 --- a/rest/schemas/requests/workflow.py +++ b/rest/schemas/requests/workflow.py @@ -41,7 +41,7 @@ class WorkflowBaseSettings(BaseModel): start_date: str = Field(alias="startDateTime") select_end_date: Optional[SelectEndDate] = Field(alias="selectEndDate", default=SelectEndDate.never) end_date: Optional[str] = Field(alias='endDateTime') - schedule_interval: ScheduleIntervalType = Field(alias="scheduleInterval") + schedule: ScheduleIntervalType = Field(alias="scheduleInterval") catchup: Optional[bool] = False # TODO add catchup to UI? generate_report: Optional[bool] = Field(alias="generateReport", default=False) # TODO add generate report to UI? description: Optional[str] # TODO add description to UI? @@ -159,5 +159,5 @@ class ListWorkflowsFilters(BaseModel): start_date__gt: Optional[str] end_date: Optional[str] end_date__gt: Optional[str] - schedule_interval: Optional[ScheduleIntervalType] + schedule: Optional[ScheduleIntervalType] diff --git a/rest/schemas/responses/workflow.py b/rest/schemas/responses/workflow.py index 62a4a3a4..1a040591 100644 --- a/rest/schemas/responses/workflow.py +++ b/rest/schemas/responses/workflow.py @@ -1,6 +1,6 @@ from schemas.responses.base import PaginationSet from pydantic import BaseModel, Field, validator -from datetime import datetime +from datetime import datetime, timezone from typing import Dict, Optional, List, Union from enum import Enum @@ -48,15 +48,15 @@ class WorkflowConfigResponse(BaseModel): name: str start_date: str end_date: Optional[str] - schedule_interval: Optional[ScheduleIntervalTypeResponse] + schedule: Optional[ScheduleIntervalTypeResponse] catchup: bool = False generate_report: bool = False description: Optional[str] - @validator('schedule_interval') - def set_schedule_interval(cls, schedule_interval): - return schedule_interval or ScheduleIntervalTypeResponse.none + @validator('schedule') + def set_schedule(cls, schedule): + return schedule or ScheduleIntervalTypeResponse.none class BaseWorkflowModel(BaseModel): workflow: WorkflowConfigResponse @@ -76,11 +76,31 @@ class GetWorkflowsResponseData(BaseModel): is_paused: bool is_active: bool status: WorkflowStatus - schedule_interval: Optional[ScheduleIntervalTypeResponse] + schedule: Optional[ScheduleIntervalTypeResponse] + next_dagrun: Optional[datetime] - @validator('schedule_interval') - def set_schedule_interval(cls, schedule_interval): - return schedule_interval or ScheduleIntervalTypeResponse.none + @validator('schedule') + def set_schedule(cls, schedule): + return schedule or ScheduleIntervalTypeResponse.none + + + @validator('created_at', pre=True, always=True) + def add_utc_timezone_created_at(cls, v): + if isinstance(v, datetime) and v.tzinfo is None: + v = v.replace(tzinfo=timezone.utc) + return v + + @validator('last_changed_at', pre=True, always=True) + def add_utc_timezone_last_changed_at(cls, v): + if isinstance(v, datetime) and v.tzinfo is None: + v = v.replace(tzinfo=timezone.utc) + return v + + @validator('next_dagrun', pre=True, always=True) + def add_utc_timezone_next_dagrun(cls, v): + if isinstance(v, datetime) and v.tzinfo is None: + v = v.replace(tzinfo=timezone.utc) + return v class GetWorkflowsResponse(BaseModel): data: List[GetWorkflowsResponseData] @@ -107,7 +127,7 @@ class GetWorkflowResponse(BaseModel): is_subdag: Optional[Union[bool, WorkflowStatus]] # Whether the DAG is SubDAG. last_pickled: Optional[Union[datetime, WorkflowStatus]] # The last time the DAG was pickled. last_expired: Optional[Union[datetime, WorkflowStatus]] # Time when the DAG last received a refresh signal (e.g. the DAG's "refresh" button was clicked in the web UI) - schedule_interval: Optional[Union[ScheduleIntervalTypeResponse, WorkflowStatus]] # The schedule interval for the DAG. + schedule: Optional[Union[ScheduleIntervalTypeResponse, WorkflowStatus]] # The schedule interval for the DAG. max_active_tasks: Optional[Union[int, WorkflowStatus]] # Maximum number of active tasks that can be run on the DAG max_active_runs: Optional[Union[int, WorkflowStatus]] # Maximum number of active DAG runs for the DAG has_task_concurrency_limits: Optional[Union[bool, WorkflowStatus]] # Whether the DAG has task concurrency limits @@ -116,9 +136,9 @@ class GetWorkflowResponse(BaseModel): next_dagrun_data_interval_start: Optional[Union[datetime, WorkflowStatus]] # The start date of the next dag run. next_dagrun_data_interval_end: Optional[Union[datetime, WorkflowStatus]] # The end date of the next dag run. - @validator('schedule_interval') - def set_schedule_interval(cls, schedule_interval): - return schedule_interval or ScheduleIntervalTypeResponse.none + @validator('schedule') + def set_schedule(cls, schedule): + return schedule or ScheduleIntervalTypeResponse.none class GetWorkflowRunsResponseData(BaseModel): diff --git a/rest/services/workflow_service.py b/rest/services/workflow_service.py index 76fda390..518ab21f 100644 --- a/rest/services/workflow_service.py +++ b/rest/services/workflow_service.py @@ -84,7 +84,7 @@ def create_workflow( last_changed_at=datetime.utcnow(), start_date=body.workflow.start_date, end_date=body.workflow.end_date, - schedule_interval=body.workflow.schedule_interval, + schedule=body.workflow.schedule, last_changed_by=auth_context.user_id, workspace_id=workspace_id ) @@ -192,25 +192,27 @@ async def list_workflows( is_dag_broken = dag_uuid in import_errors_uuids - schedule_interval = 'none' + schedule = 'none' is_active = False is_paused = False + next_dagrun = None status = WorkflowStatus.creating.value if is_dag_broken: status = WorkflowStatus.failed.value - schedule_interval = 'failed' + schedule = 'failed' is_active = False is_paused = False if response and not is_dag_broken: - schedule_interval = response.get("schedule_interval") - if isinstance(schedule_interval, dict): - schedule_interval = schedule_interval.get("value") + schedule = response.get("schedule") + if isinstance(schedule, dict): + schedule = schedule.get("value") status = WorkflowStatus.active.value is_paused = response.get("is_paused") is_active = response.get("is_active") - + next_dagrun = response.get("next_dagrun") + data.append( GetWorkflowsResponseData( id=dag_data.id, @@ -223,7 +225,8 @@ async def list_workflows( is_paused=is_paused, is_active=is_active, status=status, - schedule_interval=schedule_interval + schedule=schedule, + next_dagrun=next_dagrun ) ) @@ -259,7 +262,7 @@ def get_workflow(self, workspace_id: int, workflow_id: str, auth_context: Author 'is_subdag': 'creating', 'last_pickled': 'creating', 'last_expired': 'creating', - 'schedule_interval': 'creating', + 'schedule': 'creating', 'max_active_tasks': 'creating', 'max_active_runs': 'creating', 'has_task_concurrency_limits': 'creating', @@ -271,9 +274,10 @@ def get_workflow(self, workspace_id: int, workflow_id: str, auth_context: Author else: airflow_dag_info = airflow_dag_info.json() - schedule_interval = airflow_dag_info.pop("schedule_interval") - if isinstance(schedule_interval, dict): - schedule_interval = schedule_interval.get("value") + # Airflow 2.4.0 deprecated schedule_interval in dag but the API (2.7.2) still using it + schedule = airflow_dag_info.pop("schedule_interval") + if isinstance(schedule, dict): + schedule = schedule.get("value") response = GetWorkflowResponse( id=workflow.id, @@ -285,7 +289,7 @@ def get_workflow(self, workspace_id: int, workflow_id: str, auth_context: Author last_changed_by=workflow.last_changed_by, created_by=workflow.created_by, workspace_id=workflow.workspace_id, - schedule_interval=schedule_interval, + schedule=schedule, **airflow_dag_info ) @@ -392,13 +396,13 @@ def _create_dag_code_from_raw_json(self, data: dict, workspace_id: int): """ Format workflow kwargs to the formmate required by airflow. It should contain only the following kwargs: - dag_id - - schedule_interval + - schedule - start_date - end_date (not none) """ workflow_kwargs['dag_id'] = workflow_kwargs.pop('id') select_end_date = workflow_kwargs.pop('select_end_date') # TODO define how to use select end date - workflow_kwargs['schedule_interval'] = None if workflow_kwargs['schedule_interval'] == 'none' else f"@{workflow_kwargs['schedule_interval']}" + workflow_kwargs['schedule'] = None if workflow_kwargs['schedule'] == 'none' else f"@{workflow_kwargs['schedule']}" workflow_processed_schema = { 'workflow': deepcopy(workflow_kwargs), diff --git a/rest/tests/workflow/create_workflow_request_model.py b/rest/tests/workflow/create_workflow_request_model.py index 3f190575..e95fbde9 100644 --- a/rest/tests/workflow/create_workflow_request_model.py +++ b/rest/tests/workflow/create_workflow_request_model.py @@ -2,7 +2,7 @@ workflow_request_model = { "workflow": { - "schedule_interval": "none", + "schedule": "none", "name": "WorkflowTest", "select_end_date": "never", "start_date": str(datetime.utcnow().date()) diff --git a/rest/tests/workflow/test_workflow_router.py b/rest/tests/workflow/test_workflow_router.py index 46dc8146..b7d9b47d 100644 --- a/rest/tests/workflow/test_workflow_router.py +++ b/rest/tests/workflow/test_workflow_router.py @@ -33,7 +33,7 @@ def test_create_workflow(patch_piece_secret: Response, create_workflow: Response "name": workflow_request_model["workflow"]["name"], "start_date": workflow_request_model["workflow"]["start_date"], "end_date": None, - "schedule_interval": None, + "schedule": None, "catchup": False, "generate_report": False, "description": None From 5d6931f2a25405441aee3f891ec16ffa516bc68c Mon Sep 17 00:00:00 2001 From: Vinicius Vaz Date: Mon, 30 Oct 2023 15:12:38 -0300 Subject: [PATCH 12/16] update default piece repo version --- docker-compose-dev.yaml | 2 +- rest/core/settings.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/docker-compose-dev.yaml b/docker-compose-dev.yaml index c3038c99..7ef9d7bf 100644 --- a/docker-compose-dev.yaml +++ b/docker-compose-dev.yaml @@ -309,7 +309,7 @@ services: - DOMINO_DB_HOST=domino_postgres - DOMINO_DB_PORT=5432 - DOMINO_DB_NAME=postgres - - DOMINO_DEFAULT_PIECES_REPOSITORY_VERSION=0.4.2 + - DOMINO_DEFAULT_PIECES_REPOSITORY_VERSION=0.4.3 - DOMINO_DEFAULT_PIECES_REPOSITORY_TOKEN=${DOMINO_DEFAULT_PIECES_REPOSITORY_TOKEN} - DOMINO_GITHUB_ACCESS_TOKEN_WORKFLOWS=${DOMINO_GITHUB_ACCESS_TOKEN_WORKFLOWS} - DOMINO_GITHUB_WORKFLOWS_REPOSITORY=${DOMINO_GITHUB_WORKFLOWS_REPOSITORY} diff --git a/rest/core/settings.py b/rest/core/settings.py index b128bf38..e2409914 100644 --- a/rest/core/settings.py +++ b/rest/core/settings.py @@ -50,7 +50,7 @@ class Settings(BaseSettings): # Default domino pieces repository DOMINO_DEFAULT_PIECES_REPOSITORY = os.environ.get('DOMINO_DEFAULT_PIECES_REPOSITORY', "Tauffer-Consulting/default_domino_pieces") - DOMINO_DEFAULT_PIECES_REPOSITORY_VERSION = os.environ.get('DOMINO_DEFAULT_PIECES_REPOSITORY_VERSION', "0.4.2") + DOMINO_DEFAULT_PIECES_REPOSITORY_VERSION = os.environ.get('DOMINO_DEFAULT_PIECES_REPOSITORY_VERSION', "0.4.3") DOMINO_DEFAULT_PIECES_REPOSITORY_SOURCE = os.environ.get('DOMINO_DEFAULT_PIECES_REPOSITORY_SOURCE', "github") DOMINO_DEFAULT_PIECES_REPOSITORY_TOKEN: EmptyStrToNone = os.environ.get('DOMINO_DEFAULT_PIECES_REPOSITORY_TOKEN', "") DOMINO_DEFAULT_PIECES_REPOSITORY_URL: str = os.environ.get('DOMINO_DEFAULT_PIECES_REPOSITORY_URL', 'https://github.com/Tauffer-Consulting/default_domino_pieces') From 082f90a7aec4d9e71652e5a86318bdb52f758391 Mon Sep 17 00:00:00 2001 From: Vinicius Date: Mon, 30 Oct 2023 17:40:06 -0300 Subject: [PATCH 13/16] shutil to copy and pathlib to chmod fix #132 --- src/domino/cli/utils/platform.py | 15 +++++++++------ 1 file changed, 9 insertions(+), 6 deletions(-) diff --git a/src/domino/cli/utils/platform.py b/src/domino/cli/utils/platform.py index 9fe9d157..7238926e 100644 --- a/src/domino/cli/utils/platform.py +++ b/src/domino/cli/utils/platform.py @@ -15,6 +15,7 @@ from kubernetes import client, config from domino.cli.utils.constants import COLOR_PALETTE, DOMINO_HELM_PATH, DOMINO_HELM_VERSION, DOMINO_HELM_REPOSITORY +import shutil class AsLiteral(str): @@ -660,21 +661,23 @@ def run_platform_compose(detached: bool = False, use_config_file: bool = False, local_path = Path(".").resolve() domino_dir = local_path / "domino_data" domino_dir.mkdir(parents=True, exist_ok=True) - subprocess.run(["chmod", "-R", "777", "domino_data"]) - airflow_logs_dir = local_path / "airflow/logs" + domino_dir.chmod(0o777) + + airflow_base = local_path / 'airflow' + airflow_logs_dir = airflow_base / "logs" airflow_logs_dir.mkdir(parents=True, exist_ok=True) - airflow_dags_dir = local_path / "airflow/dags" + airflow_dags_dir = airflow_base / "dags" airflow_dags_dir.mkdir(parents=True, exist_ok=True) - airflow_plugins_dir = local_path / "airflow/plugins" + airflow_plugins_dir = airflow_base / "plugins" airflow_plugins_dir.mkdir(parents=True, exist_ok=True) - subprocess.run(["chmod", "-R", "777", "airflow"]) + airflow_base.chmod(0o777) # Copy docker-compose.yaml file from package to local path if create_database: docker_compose_path = Path(__file__).resolve().parent / "docker-compose.yaml" else: docker_compose_path = Path(__file__).resolve().parent / "docker-compose-without-database.yaml" - subprocess.run(["cp", str(docker_compose_path), "./docker-compose.yaml"]) + shutil.copy(str(docker_compose_path), "./docker-compose.yaml") # Run docker-compose up cmd = [ "docker", From ce6905517bfad7f5be9fe3062b7a308a4dbb710a Mon Sep 17 00:00:00 2001 From: Vinicius Vaz Date: Tue, 31 Oct 2023 07:52:33 -0300 Subject: [PATCH 14/16] fix import error --- .../components/WorkflowEditor.tsx | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/frontend/src/features/workflowEditor/components/WorkflowEditor.tsx b/frontend/src/features/workflowEditor/components/WorkflowEditor.tsx index c96cd57e..b783d81e 100644 --- a/frontend/src/features/workflowEditor/components/WorkflowEditor.tsx +++ b/frontend/src/features/workflowEditor/components/WorkflowEditor.tsx @@ -228,14 +228,7 @@ export const WorkflowsEditorComponent: React.FC = () => { ...new Set( Object.values(workflowPieces) .reduce>((acc, next) => { - acc.push( - `${next.source_image.split("ghcr.io/")[1].split(":")[0]}:${ - next.source_image - .split("ghcr.io/")[1] - .split(":")[1] - .split("-")[0] - }`, - ); + acc.push(next.source_image); return acc; }, []) .filter((su) => !!su) as string[], @@ -501,7 +494,14 @@ export const WorkflowsEditorComponent: React.FC = () => { content={
      {incompatiblesPieces.map((item) => ( -
    • {item}
      • +
    • + {`${item.split("ghcr.io/")[1].split(":")[0]}: ${ + item + .split("ghcr.io/")[1] + .split(":")[1] + .split("-")[0] + }`} +
      • ))}
    } From 0445a2a77c5ce81505dc39990a9b80c679559895 Mon Sep 17 00:00:00 2001 From: Vinicius Vaz Date: Tue, 31 Oct 2023 08:03:22 -0300 Subject: [PATCH 15/16] Fix workflow settings loadData --- .../workflowEditor/components/SidebarSettingsForm/index.tsx | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/frontend/src/features/workflowEditor/components/SidebarSettingsForm/index.tsx b/frontend/src/features/workflowEditor/components/SidebarSettingsForm/index.tsx index b31cab6d..093da384 100644 --- a/frontend/src/features/workflowEditor/components/SidebarSettingsForm/index.tsx +++ b/frontend/src/features/workflowEditor/components/SidebarSettingsForm/index.tsx @@ -158,7 +158,7 @@ const SidebarSettingsForm = forwardRef< useEffect(() => { void loadData(); - }, [loadData]); + }, [open, loadData]); useEffect(() => { void saveData(); From 48fd524327c4919c6d3b5b568c87a2043189074f Mon Sep 17 00:00:00 2001 From: Vinicius Vaz Date: Tue, 31 Oct 2023 08:07:04 -0300 Subject: [PATCH 16/16] update version --- src/domino/VERSION | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/domino/VERSION b/src/domino/VERSION index b35b9dd0..89235c0b 100644 --- a/src/domino/VERSION +++ b/src/domino/VERSION @@ -1 +1 @@ -0.5.8 \ No newline at end of file +0.5.9 \ No newline at end of file