Skip to content

Commit

Permalink
Update roboto module to 0.2.13
Browse files Browse the repository at this point in the history
Add end-to-end test: platform_test.sh
  • Loading branch information
YvesSchoenberg committed Mar 12, 2024
1 parent 257e067 commit 0f873b6
Show file tree
Hide file tree
Showing 4 changed files with 94 additions and 12 deletions.
2 changes: 1 addition & 1 deletion actions/ulog_ingestion/requirements.runtime.txt
Original file line number Diff line number Diff line change
@@ -1,5 +1,5 @@
# Python packages to install within the Docker image associated with this Action.
roboto==0.2.11
roboto==0.2.13
pyulog==1.0.2
mcap==1.1.1
jsonschema>=4.21.1
83 changes: 83 additions & 0 deletions actions/ulog_ingestion/scripts/platform_test.sh
Original file line number Diff line number Diff line change
@@ -0,0 +1,83 @@
#!/bin/bash
set -e

ACTION_NAME=ulog_ingestion
INPUT_FILE_PATH="./test/input/test.ulg"
EXPECTED_OUTPUT_FILE_NAME="battery_status.mcap"
INPUT_DATA="*ulg"
ORG=${ORG}

# Check for command line argument for ORG, fallback to roboto-public if not provided
if [ -z "$1" ]; then
ORG="roboto-public"
else
ORG=$1
fi

# Step 1: Build and deploy docker image
./scripts/build.sh
./scripts/setup.sh
./scripts/deploy.sh "$ORG"

# Step 2: Create dataset on platform and parse its ID
create_output=$(roboto datasets create --org "$ORG")
dataset_id=$(echo "$create_output" | jq -r '.dataset_id')

if [ -z "$dataset_id" ]; then
echo "Failed to create dataset"
exit 1
fi

echo "Dataset created with ID: $dataset_id"

# Step 3: Upload a test file to the dataset
roboto datasets upload-files -d "$dataset_id" -p "$INPUT_FILE_PATH"

# Step 4: Invoke the Action and parse Invocation ID
invoke_output=$(roboto actions invoke "$ACTION_NAME" --dataset-id "$dataset_id" --input-data "$INPUT_DATA" --org "$ORG")
invocation_id=$(echo "$invoke_output" | grep -oP "Invocation ID: '\K[^']+" )

if [ -z "$invocation_id" ]; then
echo "Failed to invoke action"
exit 1
fi

echo "Action invoked with ID: $invocation_id"

# Step 5: Monitor the invocation status
end_time=$(date -ud "10 minutes" +%s)
while true; do
now=$(date +%s)
if (( now > end_time )); then
echo "Invocation monitoring timed out for $ACTION_NAME"
exit 1
fi

status_output=$(roboto invocations status "$invocation_id")
status=$(echo "$status_output" | jq -r '.[-1].status')

case $status in
"Completed")
echo "Action completed"
break
;;
"Failed")
echo "Action failed for $ACTION_NAME"
roboto invocations logs $invocation_id
exit 1
;;
*)
echo "Current status: $status. Checking again in 20 seconds..."
sleep 20
;;
esac
done

# Step 6: Verify the output files
list_output=$(roboto datasets list-files -d "$dataset_id")
if echo "$list_output" | grep -q "$EXPECTED_OUTPUT_FILE_NAME"; then
echo "Test succeeded for $ACTION_NAME"
else
echo "Test failed for $ACTION_NAME"
fi

21 changes: 10 additions & 11 deletions actions/ulog_ingestion/src/ulog_ingestion/__main__.py
Original file line number Diff line number Diff line change
Expand Up @@ -18,13 +18,13 @@
SigV4AuthDecorator,
)
from roboto.transactions import TransactionManager

from roboto.env import RobotoEnvKey
import ulog_ingestion.utils as utils

log = logging.getLogger("Ingesting ULog files to Roboto")


def load_env_var(env_var: actions.InvocationEnvVar) -> str:
def load_env_var(env_var: RobotoEnvKey) -> str:
"""
Load an environment variable, and exit if it is not found.
Expand All @@ -48,11 +48,11 @@ def setup_env():
Returns:
- A tuple containing the organization ID, input directory, output directory, topic delegate, and dataset.
"""
roboto_service_url = load_env_var(actions.InvocationEnvVar.RobotoServiceUrl)
org_id = load_env_var(actions.InvocationEnvVar.OrgId)
invocation_id = load_env_var(actions.InvocationEnvVar.InvocationId)
input_dir = load_env_var(actions.InvocationEnvVar.InputDir)
output_dir = load_env_var(actions.InvocationEnvVar.OutputDir)
roboto_service_url = load_env_var(RobotoEnvKey.RobotoServiceUrl)
org_id = load_env_var(RobotoEnvKey.OrgId)
invocation_id = load_env_var(RobotoEnvKey.InvocationId)
input_dir = load_env_var(RobotoEnvKey.InputDir)
output_dir = load_env_var(RobotoEnvKey.OutputDir)

http_client = HttpClient(default_auth=SigV4AuthDecorator("execute-api"))

Expand All @@ -64,8 +64,7 @@ def setup_env():
invocation_id,
invocation_delegate=actions.InvocationHttpDelegate(
roboto_service_base_url=roboto_service_url, http_client=http_client
),
org_id=org_id,
)
)
dataset = datasets.Dataset.from_id(
invocation.data_source.data_source_id,
Expand Down Expand Up @@ -263,7 +262,7 @@ def ingest_ulog(ulog_file_path: str, topics: List[str] = None):
type=pathlib.Path,
required=False,
help="Directory containing input files to process",
default=os.environ.get(actions.InvocationEnvVar.InputDir.value),
default=os.environ.get(RobotoEnvKey.InputDir.value),
)

parser.add_argument(
Expand All @@ -273,7 +272,7 @@ def ingest_ulog(ulog_file_path: str, topics: List[str] = None):
type=pathlib.Path,
required=False,
help="Directory to which to write any output files to be uploaded",
default=os.environ.get(actions.InvocationEnvVar.OutputDir.value),
default=os.environ.get(RobotoEnvKey.OutputDir.value),
)

parser.add_argument(
Expand Down
Binary file added actions/ulog_ingestion/test/input/test.ulg
Binary file not shown.

0 comments on commit 0f873b6

Please sign in to comment.