diff --git a/FlowCron-template_definition.json b/FlowCron-template_definition.json new file mode 100644 index 0000000..27d6cc6 --- /dev/null +++ b/FlowCron-template_definition.json @@ -0,0 +1,748 @@ +{ + "Comment": "Transfer files using Globus Transfer", + "StartAt": "GetSourceCollection", + "States": { + "GetSourceCollection": { + "Next": "CheckSourceCollectionId", + "Type": "Action", + "Comment": "Get information about the source Collection", + "WaitTime": 172800, + "ActionUrl": "https://actions.globus.org/transfer/collection_info", + "Parameters": { + "endpoint_id.$": "$.input.source.id" + }, + "ResultPath": "$.SourceEpInfo" + }, + "CheckSourceCollectionId": { + "Type": "Choice", + "Choices": [ + { + "Or": [ + { + "And": [ + { + "Variable": "$.SourceEpInfo.details.subscription_id", + "IsPresent": true + }, + { + "IsNull": false, + "Variable": "$.SourceEpInfo.details.subscription_id" + } + ] + }, + { + "And": [ + { + "Variable": "$.SourceEpInfo.details.entity_type", + "IsPresent": true + }, + { + "Variable": "$.SourceEpInfo.details.entity_type", + "StringEquals": "GCP_guest_collection" + } + ] + } + ], + "Next": "LookupSourcePath" + } + ], + "Comment": "Check that the source collection is managed", + "Default": "JobDirNameLength" + }, + "LookupSourcePath": { + "Next": "CheckSourcePathInfo", + "Type": "Action", + "Catch": [ + { + "Next": "SourcePathMissing", + "ResultPath": "$.SourcePathLookupErrorResult", + "ErrorEquals": [ + "ActionUnableToRun", + "ActionFailedException", + "ActionTimeout" + ] + } + ], + "Comment": "Lookup the source path to determine it is a folder", + "WaitTime": 172800, + "ActionUrl": "https://actions.globus.org/transfer/ls", + "Parameters": { + "path.$": "$.input.source.path", + "path_only": true, + "endpoint_id.$": "$.input.source.id" + }, + "ResultPath": "$.SourcePathInfo" + }, + "SourcePathMissing": { + "Next": "SourcePathMissingFail", + "Type": "ExpressionEval", + "Parameters": { + "error.=": "'Missing source path ' + input.source.path + ' on collection ' + input.source.id" + }, + "ResultPath": "$.FlowResult" + }, + "SourcePathMissingFail": { + "Type": "Fail", + "Cause": "SourcePathMissing", + "Error": "See state in $.FlowResult of the run output", + "Comment": "Report the error and end the flow execution" + }, + "CheckSourcePathInfo": { + "Type": "Choice", + "Choices": [ + { + "Next": "SourcePathMissing", + "Variable": "$.SourcePathInfo.details.DATA[0]", + "IsPresent": false + } + ], + "Comment": "Determine the type of the source path", + "Default": "CheckIfInputPathIsFolder" + }, + "CheckIfInputPathIsFolder": { + "Type": "Choice", + "Choices": [ + { + "Next": "SourcePathNotAFolder", + "Variable": "$.SourcePathInfo.details.DATA[0].is_folder", + "BooleanEquals": false + } + ], + "Comment": "Determine if the source path is a folder", + "Default": "JobDirNameLength" + }, + "SourcePathNotAFolder": { + "Next": "SourcePathNotAFolderFail", + "Type": "ExpressionEval", + "Parameters": { + "error.=": "'Source path ' + input.source.path + ' is not a folder, and it should be a job folder.'" + }, + "ResultPath": "$.FlowResult" + }, + "SourcePathNotAFolderFail": { + "Type": "Fail", + "Cause": "SourcePathNotAFolder", + "Error": "See state in $.FlowResult of the run output", + "Comment": "Report the error and end the flow execution" + }, + "JobDirNameLength": { + "Next": "ReduceJobDirNameForLabelsOrNot", + "Type": "ExpressionEval", + "Parameters": { + "JobDirName.=": "pathsplit(pathsplit(input.source.path)[0])[1]", + "JobDirNameLength.=": "len(pathsplit(pathsplit(input.source.path)[0])[1])" + }, + "ResultPath": "$.jobdir" + }, + "ReduceJobDirNameForLabelsOrNot": { + "Type": "Choice", + "Choices": [ + { + "Next": "ReduceJobDirNameForLabels", + "Variable": "$.jobdir.JobDirNameLength", + "NumericGreaterThan": 29 + } + ], + "Comment": "Reduce the job directory name if it is more than 30 characters", + "Default": "PathStringsCreation" + }, + "ReduceJobDirNameForLabels": { + "Next": "PathStringsCreation", + "Type": "ExpressionEval", + "Parameters": { + "JobDirName.=": "jobdir.JobDirName[:13] + '...' + jobdir.JobDirName[-13:]" + }, + "ResultPath": "$.jobdir" + }, + "PathStringsCreation": { + "Next": "TransferFiles", + "Type": "ExpressionEval", + "Parameters": { + "UploadedFilesLocation.=": "'FLOWCRON_BASE_PATH/UploadedFiles/' + pathsplit(pathsplit(input.source.path)[0])[1] + '_' + _context.run_id + '/'", + "UploadedFilesSentinelsLocation.=": "'FLOWCRON_BASE_PATH/UploadedFiles/' + pathsplit(pathsplit(input.source.path)[0])[1] + '_' + _context.run_id + '/sentinels'", + "AnalysedFilesLocation.=": "'FLOWCRON_BASE_PATH/AnalysedFiles/' + pathsplit(pathsplit(input.source.path)[0])[1] + '_' + _context.run_id + '/'", + "AnalysedFilesSentinelsLocation.=": "'FLOWCRON_BASE_PATH/AnalysedFiles/' + pathsplit(pathsplit(input.source.path)[0])[1] + '_' + _context.run_id + '/sentinels/'", + "FailedJobsLocation.=": "'FLOWCRON_BASE_PATH/FailedJobs/' + pathsplit(pathsplit(input.source.path)[0])[1] + '_' + _context.run_id + '/'", + "FailedJobsSentinelsLocation.=": "'FLOWCRON_BASE_PATH/FailedJobs/' + pathsplit(pathsplit(input.source.path)[0])[1] + '_' + _context.run_id + '/sentinels/'", + "CodeToRunLocation.=": "'FLOWCRON_BASE_PATH/CodeToRun/slurm/' + pathsplit(pathsplit(input.source.path)[0])[1] + '_' + _context.run_id + '/'", + "TransferToHPCNAMELabel.=": "'Transferring ' + jobdir.JobDirName + ' to HPCNAME, flow run ' + _context.run_id", + "AnalysedFilesTransferFromHPCNAMELabel.=": "'Transferring ' + jobdir.JobDirName + ' back from AnalysedFiles in HPCNAME, flow run ' + _context.run_id", + "FailedJobsTransferFromHPCNAMELabel.=": "'Transferring ' + jobdir.JobDirName + ' back from FailedJobs in HPCNAME, flow run ' + _context.run_id", + "AnalysedFilesDeleteFromHPCNAMELabel.=": "'Deleting ' + jobdir.JobDirName + ' from AnalysedFiles in HPCNAME, flow run ' + _context.run_id", + "FailedJobsDeleteFromHPCNAMELabel.=": "'Deleting ' + jobdir.JobDirName + ' from FailedJobs in HPCNAME, flow run ' + _context.run_id" + }, + "ResultPath": "$.path_strings" + }, + "TransferFiles": { + "Next": "CreateSentinels", + "Comment": "Transfer job directory to the UploadedFiles directory of the cron service", + "Type": "Action", + "Catch": [ + { + "Next": "TransferFilesFail", + "ResultPath": "$.TransferFilesErrorResult", + "ErrorEquals": [ + "ActionUnableToRun", + "ActionFailedException", + "ActionTimeout" + ] + } + ], + "ActionUrl": "https://actions.automate.globus.org/transfer/transfer", + "Parameters": { + "source_endpoint_id.$": "$.input.source.id", + "destination_endpoint_id": "SOURCE_ENDPOINT_UUID", + "preserve_timestamp": true, + "verify_checksum": true, + "label.$": "$.path_strings.TransferToHPCNAMELabel", + "transfer_items": [ + { + "source_path.$": "$.input.source.path", + "destination_path.$": "$.path_strings.UploadedFilesLocation", + "recursive": true + } + ] + }, + "ResultPath": "$.TransferFilesResult", + "WaitTime": 172800 + }, + "TransferFilesFail": { + "Type": "Fail", + "Cause": "TransferFilesFail", + "Error": "See state in $.TransferFilesErrorResult of the run output", + "Comment": "Report the error and end the flow execution" + }, + "CreateSentinels": { + "Next": "Wait", + "Type": "Action", + "Catch": [ + { + "Next": "CreateSentinelsFail", + "ResultPath": "$.CreateSentinelsErrorResult", + "ErrorEquals": [ + "ActionUnableToRun", + "ActionFailedException", + "ActionTimeout" + ] + } + ], + "Comment": "Create 'sentinels' sub-directory in the job directory that has been upload to the UploadedFiles directory of the cron service", + "WaitTime": 172800, + "ActionUrl": "https://transfer.actions.globus.org/mkdir", + "Parameters": { + "endpoint_id": "SOURCE_ENDPOINT_UUID", + "path.$": "$.path_strings.UploadedFilesSentinelsLocation" + }, + "ResultPath": "$.CreateSentinelsResult" + }, + "CreateSentinelsFail": { + "Type": "Fail", + "Cause": "CreateSentinelsFail", + "Error": "See state in $.CreateSentinelsErrorResult of the run output", + "Comment": "Report the error and end the flow execution" + }, + "Wait": { + "Type": "Wait", + "Seconds": FLOW_WAIT_MIN, + "Next": "CheckInUploadedFiles" + }, + "CheckInUploadedFiles": { + "Next": "IsItInUploadedFiles", + "Type": "Action", + "Catch": [ + { + "Next": "CheckInUploadedFilesFail", + "ResultPath": "$.CheckInUploadedFilesErrorResult", + "ErrorEquals": [ + "ActionUnableToRun", + "ActionFailedException", + "ActionTimeout" + ] + } + ], + "Comment": "Check if the job directory is in the UploadedFiles directory of the cron service", + "WaitTime": 172800, + "ActionUrl": "https://transfer.actions.globus.org/stat", + "Parameters": { + "endpoint_id": "SOURCE_ENDPOINT_UUID", + "path.$": "$.path_strings.UploadedFilesLocation" + }, + "ResultPath": "$.CheckInUploadedFilesResult" + }, + "CheckInUploadedFilesFail": { + "Type": "Fail", + "Cause": "CheckInUploadedFilesFail", + "Error": "See state in $.CheckInUploadedFilesErrorResult of the run output", + "Comment": "Report the error and end the flow execution" + }, + "IsItInUploadedFiles": { + "Type" : "Choice", + "Choices": [ + { + "And": [ + { + "Variable": "$.CheckInUploadedFilesResult.details.code", + "IsPresent": true + }, + { + "Variable": "$.CheckInUploadedFilesResult.details.code", + "StringEquals": "NotFound" + } + ], + "Next": "CheckInCodeToRun" + } + ], + "Default": "Wait" + }, + "CheckInCodeToRun": { + "Next": "IsItInCodeToRun", + "Type": "Action", + "Catch": [ + { + "Next": "CheckInCodeToRunFail", + "ResultPath": "$.CheckInCodeToRunErrorResult", + "ErrorEquals": [ + "ActionUnableToRun", + "ActionFailedException", + "ActionTimeout" + ] + } + ], + "Comment": "Check if the job directory is in the CodeToRun directory of the cron service", + "WaitTime": 172800, + "ActionUrl": "https://transfer.actions.globus.org/stat", + "Parameters": { + "endpoint_id": "SOURCE_ENDPOINT_UUID", + "path.$": "$.path_strings.CodeToRunLocation" + }, + "ResultPath": "$.CheckInCodeToRunResult" + }, + "CheckInCodeToRunFail": { + "Type": "Fail", + "Cause": "CheckInCodeToRunFail", + "Error": "See state in $.CheckInCodeToRunErrorResult of the run output", + "Comment": "Report the error and end the flow execution" + }, + "IsItInCodeToRun": { + "Type" : "Choice", + "Choices": [ + { + "And": [ + { + "Variable": "$.CheckInCodeToRunResult.details.code", + "IsPresent": true + }, + { + "Variable": "$.CheckInCodeToRunResult.details.code", + "StringEquals": "NotFound" + } + ], + "Next": "CheckInAnalysedFiles" + } + ], + "Default": "Wait" + }, + "CheckInAnalysedFiles": { + "Next": "IsItInAnalysedFiles", + "Type": "Action", + "Catch": [ + { + "Next": "CheckInAnalysedFilesFail", + "ResultPath": "$.CheckInAnalysedFilesErrorResult", + "ErrorEquals": [ + "ActionUnableToRun", + "ActionFailedException", + "ActionTimeout" + ] + } + ], + "Comment": "Check if the job directory is in the AnalysedFiles directory of the cron service", + "WaitTime": 172800, + "ActionUrl": "https://transfer.actions.globus.org/stat", + "Parameters": { + "endpoint_id": "SOURCE_ENDPOINT_UUID", + "path.$": "$.path_strings.AnalysedFilesLocation" + }, + "ResultPath": "$.CheckInAnalysedFilesResult" + }, + "CheckInAnalysedFilesFail": { + "Type": "Fail", + "Cause": "CheckInAnalysedFilesFail", + "Error": "See state in $.CheckInAnalysedFilesErrorResult of the run output", + "Comment": "Report the error and end the flow execution" + }, + "IsItInAnalysedFiles": { + "Type": "Choice", + "Choices": [ + { + "And": [ + { + "Variable": "$.CheckInAnalysedFilesResult.details.code", + "IsPresent": true + }, + { + "Variable": "$.CheckInAnalysedFilesResult.details.code", + "StringEquals": "NotFound" + } + ], + "Next": "CheckInFailedJobs" + } + ], + "Default": "CheckInAnalysedFilesSentinels" + }, + "CheckInAnalysedFilesSentinels": { + "Next": "IsItInAnalysedFilesSentinels", + "Type": "Action", + "Catch": [ + { + "Next": "CheckInAnalysedFilesSentinelsFail", + "ResultPath": "$.CheckInAnalysedFilesSentinelsErrorResult", + "ErrorEquals": [ + "ActionUnableToRun", + "ActionFailedException", + "ActionTimeout" + ] + } + ], + "Comment": "Check if the 'sentinels' sub-directory has been deleted from the job directory which now in the AnalysedFiles directory of the cron service", + "WaitTime": 172800, + "ActionUrl": "https://transfer.actions.globus.org/stat", + "Parameters": { + "endpoint_id": "SOURCE_ENDPOINT_UUID", + "path.$": "$.path_strings.AnalysedFilesSentinelsLocation" + }, + "ResultPath": "$.CheckInAnalysedFilesSentinelsResult" + }, + "CheckInAnalysedFilesSentinelsFail": { + "Type": "Fail", + "Cause": "CheckInAnalysedFilesSentinelsFail", + "Error": "See state in $.CheckInAnalysedFilesSentinelsErrorResult of the run output", + "Comment": "Report the error and end the flow execution" + }, + "IsItInAnalysedFilesSentinels": { + "Type": "Choice", + "Choices": [ + { + "And": [ + { + "Variable": "$.CheckInAnalysedFilesSentinelsResult.details.code", + "IsPresent": true + }, + { + "Variable": "$.CheckInAnalysedFilesSentinelsResult.details.code", + "StringEquals": "NotFound" + } + ], + "Next": "TransferFilesBack" + } + ], + "Default": "Wait" + }, + "TransferFilesBack": { + "Next": "ShouldDeleteFromAnalysedFiles", + "Comment": "Transfer job directory back to the source path with sync option enabled", + "Type": "Action", + "Catch": [ + { + "Next": "TransferFilesBackFail", + "ResultPath": "$.TransferFilesBackErrorResult", + "ErrorEquals": [ + "ActionUnableToRun", + "ActionFailedException", + "ActionTimeout" + ] + } + ], + "ActionUrl": "https://actions.automate.globus.org/transfer/transfer", + "Parameters": { + "source_endpoint_id": "SOURCE_ENDPOINT_UUID", + "destination_endpoint_id.$": "$.input.source.id", + "preserve_timestamp": true, + "sync_level": 2, + "verify_checksum": true, + "label.$": "$.path_strings.AnalysedFilesTransferFromHPCNAMELabel", + "transfer_items": [ + { + "source_path.$": "$.path_strings.AnalysedFilesLocation", + "destination_path.$": "$.input.source.path", + "recursive": true + } + ] + }, + "ResultPath": "$.TransferFilesBackResult", + "WaitTime": 172800 + }, + "TransferFilesBackFail": { + "Type": "Fail", + "Cause": "TransferFilesBackFail", + "Error": "See state in $.TransferFilesBackErrorResult of the run output", + "Comment": "Report the error and end the flow execution" + }, + "ShouldDeleteFromAnalysedFiles": { + "Type": "Choice", + "Choices": [ + { + "Variable": "$.input.delete", + "BooleanEquals": false, + "Next": "PathToJobDir" + } + ], + "Default": "DeleteJobFromAnalysedFiles" + }, + "DeleteJobFromAnalysedFiles": { + "Next": "SuccessfulEndOfFlowWithDelete", + "Comment": "Delete job directory from AnalysedFiles directory in the cron service", + "Type": "Action", + "Catch": [ + { + "Next": "DeleteJobFromAnalysedFilesFail", + "ResultPath": "$.DeleteJobFromAnalysedFilesErrorResult", + "ErrorEquals": [ + "ActionUnableToRun", + "ActionFailedException", + "ActionTimeout" + ] + } + ], + "ActionUrl": "https://transfer.actions.globus.org/delete", + "Parameters": { + "endpoint": "SOURCE_ENDPOINT_UUID", + "recursive": true, + "label.$": "$.path_strings.AnalysedFilesDeleteFromHPCNAMELabel", + "DATA": [ + { + "path.$": "$.path_strings.AnalysedFilesLocation" + } + ] + }, + "ResultPath": "$.DeleteJobFromAnalysedFilesResult", + "WaitTime": 172800 + }, + "DeleteJobFromAnalysedFilesFail": { + "Type": "Fail", + "Cause": "DeleteJobFromAnalysedFilesFail", + "Error": "See state in $.DeleteJobFromAnalysedFilesErrorResult of the run output", + "Comment": "Report the error and end the flow execution" + }, + "CheckInFailedJobs": { + "Next": "IsItInFailedJobs", + "Type": "Action", + "Catch": [ + { + "Next": "CheckInFailedJobsFail", + "ResultPath": "$.CheckInFailedJobsErrorResult", + "ErrorEquals": [ + "ActionUnableToRun", + "ActionFailedException", + "ActionTimeout" + ] + } + ], + "Comment": "Check if the job directory is in the FailedJobs directory of the cron service", + "WaitTime": 172800, + "ActionUrl": "https://transfer.actions.globus.org/stat", + "Parameters": { + "endpoint_id": "SOURCE_ENDPOINT_UUID", + "path.$": "$.path_strings.FailedJobsLocation" + }, + "ResultPath": "$.CheckInFailedJobsResult" + }, + "CheckInFailedJobsFail": { + "Type": "Fail", + "Cause": "CheckInFailedJobsFail", + "Error": "See state in $.CheckInFailedJobsErrorResult of the run output", + "Comment": "Report the error and end the flow execution" + }, + "IsItInFailedJobs": { + "Type": "Choice", + "Choices": [ + { + "And": [ + { + "Variable": "$.CheckInFailedJobsResult.details.code", + "IsPresent": true + }, + { + "Variable": "$.CheckInFailedJobsResult.details.code", + "StringEquals": "NotFound" + } + ], + "Next": "ErrorFilesCannotBeFound" + } + ], + "Default": "CheckInFailedJobsSentinels" + }, + "CheckInFailedJobsSentinels": { + "Next": "IsItInFailedJobsSentinels", + "Type": "Action", + "Catch": [ + { + "Next": "CheckInFailedJobsSentinelsFail", + "ResultPath": "$.CheckInFailedJobsSentinelsErrorResult", + "ErrorEquals": [ + "ActionUnableToRun", + "ActionFailedException", + "ActionTimeout" + ] + } + ], + "Comment": "Check if the 'sentinels' sub-directory has been deleted from the job directory which now in the FailedJobs directory of the cron service", + "WaitTime": 172800, + "ActionUrl": "https://transfer.actions.globus.org/stat", + "Parameters": { + "endpoint_id": "SOURCE_ENDPOINT_UUID", + "path.$": "$.path_strings.AnalysedFilesSentinelsLocation" + }, + "ResultPath": "$.CheckInFailedJobsSentinelsResult" + }, + "CheckInFailedJobsSentinelsFail": { + "Type": "Fail", + "Cause": "CheckInFailedJobsSentinelsFail", + "Error": "See state in $.CheckInFailedJobsSentinelsErrorResult of the run output", + "Comment": "Report the error and end the flow execution" + }, + "IsItInFailedJobsSentinels": { + "Type": "Choice", + "Choices": [ + { + "And": [ + { + "Variable": "$.CheckInFailedJobsSentinelsResult.details.code", + "IsPresent": true + }, + { + "Variable": "$.CheckInFailedJobsSentinelsResult.details.code", + "StringEquals": "NotFound" + } + ], + "Next": "TransferFailedJobBack" + } + ], + "Default": "Wait" + }, + "ErrorFilesCannotBeFound": { + "Type": "Fail", + "Comment": "The job destination directory could not be found in either the UploadedFiles, CodeToRun, AnalysedFiles or FailedJobs of the cron service.", + "Error": "Cron service error. Please contact your Flowcron support team.", + "Cause": "MissingDirectory" + }, + "TransferFailedJobBack": { + "Next": "ShouldDeleteFromFailedJobs", + "Comment": "Transfer failed job's error logs back or directory back", + "Type": "Action", + "Catch": [ + { + "Next": "TransferFailedJobBackFail", + "ResultPath": "$.TransferFailedJobBackErrorResult", + "ErrorEquals": [ + "ActionUnableToRun", + "ActionFailedException", + "ActionTimeout" + ] + } + ], + "ActionUrl": "https://actions.automate.globus.org/transfer/transfer", + "Parameters": { + "source_endpoint_id": "SOURCE_ENDPOINT_UUID", + "destination_endpoint_id.$": "$.input.source.id", + "preserve_timestamp": true, + "sync_level": 2, + "verify_checksum": true, + "label.$": "$.path_strings.FailedJobsTransferFromHPCNAMELabel", + "transfer_items": [ + { + "source_path.$": "$.path_strings.FailedJobsLocation", + "destination_path.$": "$.input.source.path", + "recursive": true + } + ] + }, + "ResultPath": "$.TransferFailedJobBackResult", + "WaitTime": 172800 + }, + "TransferFailedJobBackFail": { + "Type": "Fail", + "Cause": "TransferFailedJobBackFail", + "Error": "See state in $.TransferFailedJobBackErrorResult of the run output", + "Comment": "Report the error and end the flow execution" + }, + "ShouldDeleteFromFailedJobs": { + "Type": "Choice", + "Choices": [ + { + "Variable": "$.input.delete", + "BooleanEquals": false, + "Next": "JobFailed" + } + ], + "Default": "DeleteJobFromFailedJobs" + }, + "DeleteJobFromFailedJobs": { + "Next": "JobFailed", + "Comment": "Delete job directory from FailedJob directory in the cron service", + "Type": "Action", + "Catch": [ + { + "Next": "DeleteJobFromFailedJobsFail", + "ResultPath": "$.DeleteJobFromFailedJobsErrorResult", + "ErrorEquals": [ + "ActionUnableToRun", + "ActionFailedException", + "ActionTimeout" + ] + } + ], + "ActionUrl": "https://transfer.actions.globus.org/delete", + "Parameters": { + "endpoint": "SOURCE_ENDPOINT_UUID", + "recursive": true, + "label.$": "$.path_strings.FailedJobsDeleteFromHPCNAMELabel", + "DATA": [ + { + "path.$": "$.path_strings.FailedJobsLocation" + } + ] + }, + "ResultPath": "$.DeleteJobFromFailedJobsResult", + "WaitTime": 172800 + }, + "DeleteJobFromFailedJobsFail": { + "Type": "Fail", + "Cause": "DeleteJobFromFailedJobsFail", + "Error": "See state in $.DeleteJobFromFailedJobsErrorResult of the run output", + "Comment": "Report the error and end the flow execution" + }, + "JobFailed": { + "Next": "ErrorJobFailed", + "Type": "ExpressionEval", + "Parameters": { + "error.=": "'The failed job is on HPCNAME and is not deleted. It is in the ' + path_strings.FailedJobsLocation + ' path location. Please inspect the failed job dir to find what was wrong, and then delete it from HPCNAME, before resubmitting the job.'" + }, + "ResultPath": "$.FlowResult" + }, + "ErrorJobFailed": { + "Type": "Fail", + "Comment": "The job failed due to errors in the submitted script or job in general.", + "Error": "See state in $.FlowResult of the run output", + "Cause": "JobFailureDueToUserError" + }, + "SuccessfulEndOfFlowWithDelete": { + "Type": "Pass", + "ResultPath": "$.FlowResult", + "End": true + }, + "PathToJobDir": { + "Next": "SuccessfulEndOfFlowWithoutDelete", + "Type": "ExpressionEval", + "Parameters": { + "path_in_HPCNAME.=": "'The job directory is on HPCNAME and is not deleted. It is in the ' + path_strings.AnalysedFilesLocation + ' path location. If you want to submit future jobs and utilise the processed data of this job, the initial commands in the scripts of the future jobs should begin with the creation of symlinks to the absolute path of this job.'" + }, + "ResultPath": "$.FlowResult" + }, + "SuccessfulEndOfFlowWithoutDelete": { + "Type": "Pass", + "ResultPath": "$.FlowResult", + "End": true + } + } +} diff --git a/FlowCron-template_input_schema.json b/FlowCron-template_input_schema.json new file mode 100644 index 0000000..e74bc69 --- /dev/null +++ b/FlowCron-template_input_schema.json @@ -0,0 +1,44 @@ +{ + "required": [ + "input" + ], + "properties": { + "input": { + "type": "object", + "required": [ + "source", + "delete" + ], + "properties": { + "source": { + "type": "object", + "title": "Select source collection and path", + "description": "The source collection and path (path MUST end with a slash)", + "format": "globus-collection", + "required": [ + "id", + "path" + ], + "properties": { + "id": { + "type": "string", + "format": "uuid" + }, + "path": { + "type": "string" + } + }, + "additionalProperties": false + }, + "delete": { + "type": "boolean", + "title": "Delete job directory in HPCNAME after transferring the results back?", + "description": "Whether or not to delete the job directory and all its files in HPCNAME, once the job processing is completed.", + "default": false + } + }, + "additionalProperties": false + } + }, + "additionalProperties": false +} diff --git a/construct_new_instance_FlowCron_configs.sh b/construct_new_instance_FlowCron_configs.sh new file mode 100644 index 0000000..accb2fe --- /dev/null +++ b/construct_new_instance_FlowCron_configs.sh @@ -0,0 +1,69 @@ +#!/bin/bash + +FLOWCRON_INSTANCE_NAME="" +HPC_NAME="" +SOURCE_ENDPOINT_UUID="" +FLOWCRON_BASE_PATH="" +TIME_OPTS=("1" "2" "5" "10" "15" "30") + +#Get the name of the HPC +while true ; do + read -p "What is the name of this particular FlowCron instance (This is to generate the updated json files as 'FlowCron-_definition.json' and 'FlowCron-_input_schema.json'? " FLOWCRON_INSTANCE_NAME + if [ ! -z $FLOWCRON_INSTANCE_NAME ]; then + break + else + echo "Please enter a value for the name of this FlowCron instance." + fi +done + +#Get the name of the HPC +while true ; do + read -p "What is the name of the HPC? " HPC_NAME + if [ ! -z $HPC_NAME ]; then + break + else + echo "Please enter the name of the HPC (this is to be used in labels to properly address it)." + fi +done + +#Get a name for the script from the user. +while true ; do + read -p "What is the UUID of the HPC's Globus Collection? " SOURCE_ENDPOINT_UUID + if [[ $SOURCE_ENDPOINT_UUID =~ ^\{?[A-F0-9a-f]{8}-[A-F0-9a-f]{4}-[A-F0-9a-f]{4}-[A-F0-9a-f]{4}-[A-F0-9a-f]{12}\}?$ ]]; then + break + else + echo "Please enter valid UUID." + fi +done + +#Get the absolute path to the FlowCron service on the HPC +while true ; do + read -p "What is absolute path to the FlowCron service on the HPC (aka what is the absolute path to the directory that has the 'UploadedFiles', 'CodeToRun', etc. as subdirectories)? " FLOWCRON_BASE_PATH + if [[ $FLOWCRON_BASE_PATH == /* ]] && [[ $FLOWCRON_BASE_PATH != */ ]] ; then + break + else + echo "Please enter the absolute to the FlowCron service directory (please do not add '/' as the last character)" + fi +done + +#Ask how many minutes should this repeat +echo -e "\nEvery how many minutes should the Globus Flow aspect of FlowCron check for the job's status on the HPC? " +select time in "${TIME_OPTS[@]}" +do + for i in "${TIME_OPTS[@]}"; do + if [ "${time}" == "${i}" ]; then + FLOW_WAIT_MIN=$((time*60)); break 2; + fi + done +done + +mkdir "$FLOWCRON_INSTANCE_NAME" + +cp FlowCron-template_definition.json "$FLOWCRON_INSTANCE_NAME"/FlowCron-"$FLOWCRON_INSTANCE_NAME"_definition.json +cp FlowCron-template_input_schema.json "$FLOWCRON_INSTANCE_NAME"/FlowCron-"$FLOWCRON_INSTANCE_NAME"_input_schema.json + +sed -i "s|HPCNAME|$HPC_NAME|g" "$FLOWCRON_INSTANCE_NAME"/FlowCron-"$FLOWCRON_INSTANCE_NAME"_definition.json +sed -i "s|HPCNAME|$HPC_NAME|g" "$FLOWCRON_INSTANCE_NAME"/FlowCron-"$FLOWCRON_INSTANCE_NAME"_input_schema.json +sed -i "s|SOURCE_ENDPOINT_UUID|$SOURCE_ENDPOINT_UUID|g" "$FLOWCRON_INSTANCE_NAME"/FlowCron-"$FLOWCRON_INSTANCE_NAME"_definition.json +sed -i "s|FLOWCRON_BASE_PATH|$FLOWCRON_BASE_PATH|g" "$FLOWCRON_INSTANCE_NAME"/FlowCron-"$FLOWCRON_INSTANCE_NAME"_definition.json +sed -i "s|FLOW_WAIT_MIN|$FLOW_WAIT_MIN|g" "$FLOWCRON_INSTANCE_NAME"/FlowCron-"$FLOWCRON_INSTANCE_NAME"_definition.json