Merge pull request #250 from Tauffer-Consulting/dev

Dev
Tauffer-Consulting · Mar 8, 2024 · 9d21ea7 · 9d21ea7
2 parents 3c337d0 + 9548308
commit 9d21ea7
Show file tree

Hide file tree

Showing 19 changed files with 282 additions and 71 deletions.
diff --git a/docker-compose-dev.yaml b/docker-compose-dev.yaml
@@ -14,8 +14,9 @@ x-airflow-common:
     AIRFLOW__CELERY__RESULT_BACKEND: db+postgresql://airflow:airflow@postgres/airflow
     AIRFLOW__CELERY__BROKER_URL: redis://:@redis:6379/0
     AIRFLOW__CORE__FERNET_KEY: ''
-    AIRFLOW__CORE__DAGS_ARE_PAUSED_AT_CREATION: 'true'
+    AIRFLOW__CORE__DAGS_ARE_PAUSED_AT_CREATION: 'false'
     AIRFLOW__CORE__LOAD_EXAMPLES: 'false'
+    AIRFLOW__WEBSERVER__SHOW_TRIGGER_FORM_IF_NO_PARAMS: 'true'
     AIRFLOW__API__AUTH_BACKENDS: 'airflow.api.auth.backend.basic_auth,airflow.api.auth.backend.session'
     AIRFLOW__SCHEDULER__ENABLE_HEALTH_CHECK: 'true'
     AIRFLOW__SCHEDULER__DAG_DIR_LIST_INTERVAL: 10

diff --git a/frontend/src/components/DatetimeInput/index.tsx b/frontend/src/components/DatetimeInput/index.tsx
@@ -22,13 +22,15 @@ interface Props<T> {
   name: Path<T>;
   type?: "time" | "date" | "date-time";
   defaultValue?: string | null;
+  disablePast?: boolean;
 }
 
 function DatetimeInput<T extends FieldValues>({
   label,
   name,
   type = "date",
   defaultValue = null,
+  disablePast = false,
 }: Props<T>) {
   const {
     control,
@@ -53,6 +55,7 @@ function DatetimeInput<T extends FieldValues>({
             <LocalizationProvider dateAdapter={AdapterDayjs}>
               <DemoContainer components={["DateTimePicker"]}>
                 <DateTimePicker
+                  disablePast={disablePast}
                   label={label}
                   ampm={false}
                   format="DD/MM/YYYY HH:mm"
@@ -91,6 +94,7 @@ function DatetimeInput<T extends FieldValues>({
             <LocalizationProvider dateAdapter={AdapterDayjs}>
               <DemoContainer components={["TimePicker"]} sx={{ width: "100%" }}>
                 <TimePicker
+                  disablePast={disablePast}
                   ampm={false}
                   label={label}
                   format="HH:mm"
@@ -134,6 +138,7 @@ function DatetimeInput<T extends FieldValues>({
             <DemoContainer components={["DatePicker"]} sx={{ width: "100%" }}>
               <LocalizationProvider dateAdapter={AdapterDayjs}>
                 <DatePicker
+                  disablePast={disablePast}
                   label={label}
                   views={["day", "month", "year"]}
                   format="DD/MM/YYYY"

diff --git a/frontend/src/features/myWorkflows/components/WorkflowsList/Actions.tsx b/frontend/src/features/myWorkflows/components/WorkflowsList/Actions.tsx
@@ -1,7 +1,7 @@
 import DeleteOutlineIcon from "@mui/icons-material/DeleteOutline";
 import PauseCircleOutlineIcon from "@mui/icons-material/PauseCircleOutline";
 import PlayCircleOutlineIcon from "@mui/icons-material/PlayCircleOutline";
-import { IconButton } from "@mui/material";
+import { IconButton, Tooltip } from "@mui/material";
 import { type CommonProps } from "@mui/material/OverridableComponent";
 import { Modal, type ModalRef } from "components/Modal";
 import { type IWorkflow } from "features/myWorkflows/types";
@@ -15,19 +15,47 @@ interface Props extends CommonProps {
   deleteFn: () => void;
   runFn: () => void;
   pauseFn: () => void;
+  disabled: boolean;
 }
 
-export const Actions: React.FC<Props> = ({ runFn, deleteFn, className }) => {
+export const Actions: React.FC<Props> = ({
+  runFn,
+  deleteFn,
+  className,
+  disabled = false,
+}) => {
   const [deleteModalOpen, setDeleteModalOpen] = useState(false);
   const newFeatureModal = useRef<ModalRef>(null);
 
   return (
     <>
-      <IconButton className={className} onClick={runFn}>
-        <PlayCircleOutlineIcon
-          style={{ pointerEvents: "none", color: theme.palette.success.main }}
-        />
-      </IconButton>
+      {disabled ? (
+        <Tooltip title="Can't run future workflows." arrow>
+          <span>
+            <IconButton
+              className={className}
+              onClick={runFn}
+              disabled={disabled}
+            >
+              <PlayCircleOutlineIcon
+                style={{
+                  pointerEvents: "none",
+                  color: theme.palette.grey[500],
+                }}
+              />
+            </IconButton>
+          </span>
+        </Tooltip>
+      ) : (
+        <IconButton className={className} onClick={runFn}>
+          <PlayCircleOutlineIcon
+            style={{
+              pointerEvents: "none",
+              color: theme.palette.success.main,
+            }}
+          />
+        </IconButton>
+      )}
       <IconButton
         className={className}
         onClick={() => {

diff --git a/frontend/src/features/myWorkflows/components/WorkflowsList/index.tsx b/frontend/src/features/myWorkflows/components/WorkflowsList/index.tsx
@@ -1,4 +1,5 @@
-import { Paper } from "@mui/material";
+import { InfoOutlined } from "@mui/icons-material";
+import { Paper, Tooltip } from "@mui/material";
 import {
   DataGrid,
   type GridRowParams,
@@ -14,6 +15,7 @@ import {
 import { type IWorkflow } from "features/myWorkflows/types";
 import React, { useCallback, useMemo } from "react";
 import { useNavigate } from "react-router-dom";
+import { toast } from "react-toastify";
 import { useInterval } from "utils";
 
 import { Actions } from "./Actions";
@@ -90,6 +92,22 @@ export const WorkflowList: React.FC = () => {
         sortable: false,
       },
       { field: "name", headerName: "Workflow Name", flex: 2 },
+      {
+        field: "start_date",
+        headerName: (
+          <Tooltip title="Start date is the date when your workflow begins scheduling. Workflows cannot be run before this start date">
+            <span style={{ display: "flex", alignItems: "center" }}>
+              Start Date{" "}
+              <InfoOutlined style={{ marginLeft: "5px" }} fontSize="small" />
+            </span>
+          </Tooltip>
+        ) as any,
+        flex: 1,
+        align: "center",
+
+        valueFormatter: ({ value }) => new Date(value).toLocaleString(),
+        headerAlign: "center",
+      },
       {
         field: "created_at",
         headerName: "Created At",
@@ -129,21 +147,24 @@ export const WorkflowList: React.FC = () => {
         field: "actions",
         headerName: "Actions",
         flex: 1,
-        renderCell: ({ row }) => (
-          <Actions
-            id={row.id}
-            className=".action-button"
-            deleteFn={() => {
-              void deleteWorkflow(row.id);
-            }}
-            runFn={() => {
-              void runWorkflow(row.id);
-            }}
-            pauseFn={() => {
-              pauseWorkflow(row.id);
-            }}
-          />
-        ),
+        renderCell: ({ row }) => {
+          return (
+            <Actions
+              id={row.id}
+              className=".action-button"
+              deleteFn={() => {
+                void deleteWorkflow(row.id);
+              }}
+              runFn={() => {
+                void runWorkflow(row.id);
+              }}
+              pauseFn={() => {
+                pauseWorkflow(row.id);
+              }}
+              disabled={new Date(row.start_date) > new Date()}
+            />
+          );
+        },
         headerAlign: "center",
         align: "center",
         sortable: false,
@@ -158,6 +179,12 @@ export const WorkflowList: React.FC = () => {
         event.target instanceof Element &&
         event.target.classList.contains(".action-button");
       if (!isActionButtonClick) {
+        if (new Date(params.row.start_date) > new Date()) {
+          toast.warning(
+            "Future workflows runs cannot be accessed. Wait until the start date.",
+          );
+          return;
+        }
         if (params.row.status !== "failed" && params.row.status !== "creating")
           navigate(`/my-workflows/${params.id}`);
       }

diff --git a/frontend/src/features/myWorkflows/types/workflow.ts b/frontend/src/features/myWorkflows/types/workflow.ts
@@ -12,6 +12,7 @@ export interface IWorkflow {
   id: number;
   name: string;
   created_at: string;
+  start_date: string;
   schema: IWorkflowSchema;
   ui_schema: IWorkDominoSchema;
   last_changed_at: string;

diff --git a/frontend/src/features/workflowEditor/components/Drawers/SettingsFormDrawer/index.tsx b/frontend/src/features/workflowEditor/components/Drawers/SettingsFormDrawer/index.tsx
@@ -1,4 +1,4 @@
-import { Drawer, Grid, Typography } from "@mui/material";
+import { Drawer, Grid, Typography, Tooltip } from "@mui/material";
 import DatetimeInput from "components/DatetimeInput";
 import SelectInput from "components/SelectInput";
 import TextInput from "components/TextInput";
@@ -10,7 +10,9 @@ import {
   type ScheduleIntervals,
   type StorageSourcesAWS,
   type StorageSourcesLocal,
+  type StartDateTypes,
   endDateTypes,
+  startDateTypes,
   scheduleIntervals,
   storageSourcesAWS,
   storageSourcesLocal,
@@ -37,6 +39,7 @@ const defaultSettingsData: IWorkflowSettings = {
     scheduleInterval: scheduleIntervals.None,
     startDate: dayjs(new Date()).toISOString(),
     endDateType: endDateTypes.Never,
+    startDateType: startDateTypes.Now,
   },
   storage: {
     storageSource: storageSourcesLocal.None,
@@ -81,12 +84,16 @@ export const WorkflowSettingsFormSchema: ValidationSchema = yup.object().shape({
       .mixed<ScheduleIntervals>()
       .oneOf(Object.values(scheduleIntervals))
       .required(),
-    startDate: yup.string().required(),
+    startDate: yup.string(),
     endDate: yup.string(),
     endDateType: yup
       .mixed<EndDateTypes>()
       .oneOf(Object.values(endDateTypes))
       .required(),
+    startDateType: yup
+      .mixed<StartDateTypes>()
+      .oneOf(Object.values(startDateTypes))
+      .required(),
   }),
   storage: yup.object().shape({
     storageSource: yup.lazy((value) => {
@@ -211,12 +218,31 @@ const SettingsFormDrawer = forwardRef<
                     label="Schedule"
                   />
                 </Grid>
-                <Grid item xs={12}>
-                  <DatetimeInput
-                    name="config.startDate"
-                    label="Start Date/Time"
-                    type="date-time"
-                  />
+                <Grid
+                  container
+                  spacing={2}
+                  alignItems="center"
+                  sx={{ margin: "0px" }}
+                >
+                  <Grid item xs={4}>
+                    <SelectInput
+                      name="config.startDateType"
+                      label="Start Date/Time"
+                      options={Object.values(startDateTypes)}
+                      defaultValue={defaultSettingsData.config.startDateType}
+                    />
+                  </Grid>
+                  {formData?.config?.startDateType ===
+                    startDateTypes.UserDefined && (
+                    <Grid item xs={8}>
+                      <DatetimeInput
+                        disablePast={true}
+                        name="config.startDate"
+                        label="Start Date/Time"
+                        type="date-time"
+                      />
+                    </Grid>
+                  )}
                 </Grid>
                 <Grid
                   container
@@ -236,6 +262,7 @@ const SettingsFormDrawer = forwardRef<
                     endDateTypes.UserDefined && (
                     <Grid item xs={8}>
                       <DatetimeInput
+                        disablePast={true}
                         name="config.endDate"
                         label="End Date/Time"
                         type="date-time"

diff --git a/frontend/src/features/workflowEditor/context/types/settings.ts b/frontend/src/features/workflowEditor/context/types/settings.ts
@@ -15,6 +15,13 @@ export enum endDateTypes {
   UserDefined = "User defined",
 }
 
+export enum startDateTypes {
+  Now = "now",
+  UserDefined = "User defined",
+}
+
+export type StartDateTypes = `${startDateTypes}`;
+
 export type EndDateTypes = `${endDateTypes}`;
 
 export enum storageSourcesAWS {
@@ -34,9 +41,10 @@ export type StorageSourcesLocal = `${storageSourcesLocal}`;
 export interface IWorkflowSettingsConfig {
   name: string;
   scheduleInterval: ScheduleIntervals;
-  startDate: string;
+  startDate?: string;
   endDate?: string;
   endDateType: EndDateTypes;
+  startDateType: StartDateTypes;
 }
 
 export interface IWorkflowSettingsStorage {

diff --git a/frontend/src/features/workflowEditor/context/types/workflowPieceData.ts b/frontend/src/features/workflowEditor/context/types/workflowPieceData.ts
@@ -8,6 +8,7 @@ import {
   type ScheduleIntervals,
   type StorageSourcesAWS,
   type StorageSourcesLocal,
+  type StartDateTypes,
 } from "./settings";
 import { type IStorageFormData, type StorageAccessModes } from "./storage";
 
@@ -19,7 +20,8 @@ export interface WorkflowPieceData {
 
 interface WorkflowBaseSettings {
   name: string;
-  start_date: string; // ISOFormat
+  select_start_date: StartDateTypes;
+  start_date?: string; // ISOFormat
   select_end_date: EndDateTypes;
   schedule: ScheduleIntervals;
 

diff --git a/frontend/src/features/workflowEditor/context/workflowsEditor.tsx b/frontend/src/features/workflowEditor/context/workflowsEditor.tsx
@@ -136,10 +136,10 @@ const WorkflowsEditorProvider: FC<{ children?: React.ReactNode }> = ({
         name: workflowSettingsData.config.name,
         schedule: workflowSettingsData.config.scheduleInterval,
         select_end_date: workflowSettingsData.config.endDateType,
+        select_start_date: workflowSettingsData.config.startDateType,
         start_date: workflowSettingsData.config.startDate,
         end_date: workflowSettingsData.config.endDate,
       };
-
       const ui_schema: CreateWorkflowRequest["ui_schema"] = {
         nodes: {},
         edges: workflowEdges,

diff --git a/rest/clients/airflow_client.py b/rest/clients/airflow_client.py
@@ -110,7 +110,7 @@ def get_all_dag_tasks(self, dag_id):
             resource=resource,
         )
         return response
-    
+
     def list_import_errors(self, limit: int = 100, offset: int = 0):
         resource = "api/v1/importErrors"
         response = self.request(