Add new Action tag_ulog

actions/tag_ulog/.gitignore

@@ -0,0 +1,13 @@
+.venv
+.mypy_cache
+**/*.egg-info
+**/.mypy_cache/
+**/.venv/
+**/__pycache__/
+**/.pytest_cache
+**/dist/
+*.swp
+*.pyc
+output
+input
+.idea

actions/tag_ulog/.python-version

@@ -0,0 +1,4 @@
+# This file is a directive to pyenv (https://github.com/pyenv/pyenv) to set matching version of Python in this directory.
+# If you don't use pyenv, you can safely delete this file.
+# The roboto CLI requires Python 3.9 or higher.
+3.10

actions/tag_ulog/Dockerfile

@@ -0,0 +1,11 @@
+ARG PYTHON_MAJOR=3
+ARG PYTHON_MINOR=10
+ARG OS_VARIANT=slim-bookworm
+FROM --platform=linux/amd64 public.ecr.aws/docker/library/python:${PYTHON_MAJOR}.${PYTHON_MINOR}-${OS_VARIANT}
+
+COPY requirements.runtime.txt ./
+RUN python -m pip install --upgrade pip setuptools && python -m pip install -r requirements.runtime.txt
+
+COPY src/tag_ulog/ ./tag_ulog
+
+ENTRYPOINT [ "python", "-m", "tag_ulog" ]

actions/tag_ulog/README.md

@@ -0,0 +1,14 @@
+# tag_ulog
+
+This Action converts ulg files to csv files
+
+## Getting started
+
+1. Setup a virtual environment specific to this project and install development dependencies, including the `roboto` CLI: `./scripts/setup.sh`
+2. Build Docker image: `./scripts/build.sh`
+3. Run Action image locally: `./scripts/run.sh <path-to-input-data-directory>`
+4. Deploy to Roboto Platform: `./scripts/deploy.sh`
+
+## Action configuration file
+
+This Roboto Action is configured in `action.json`. Refer to Roboto's latest documentation for the expected structure.

actions/tag_ulog/action.json

@@ -0,0 +1,19 @@
+{
+    "name": "tag_ulog",
+     "short_description": "Add a tag to the dataset if an error is present in the ULog files.",
+    "description": "This Action extracts logged messages from an ULog file (.ulg) and applies a tag to the dataset if an ERROR is present. Under the hood, it uses the [pyulog](https://github.com/PX4/pyulog) library.",
+
+    "compute_requirements": {
+        "vCPU": 1024,
+        "memory": 2048,
+        "storage": 21
+    },
+    "tags": [
+        "px4",
+        "ulg",
+	"ulog"
+    ],
+    "metadata": {
+        "github_url": "https://github.com/roboto-ai/robologs-px4-actions/tree/main/actions/tag_ulog"
+    }
+}

actions/tag_ulog/requirements.dev.txt

@@ -0,0 +1,7 @@
+# Python packages to install into the this directory's virtual environment
+# for the purpose of development, testing, and deployment.
+
+# Install all required runtime dependencies in local virtual environment.
+-r requirements.runtime.txt
+
+# Add additional Python packages to install here.

actions/tag_ulog/requirements.runtime.txt

@@ -0,0 +1,3 @@
+# Python packages to install within the Docker image associated with this Action.
+roboto
+pyulog

actions/tag_ulog/scripts/build.sh

@@ -0,0 +1,14 @@
+#!/usr/bin/env bash
+
+set -euo pipefail
+
+SCRIPTS_ROOT=$( cd -- "$( dirname -- "${BASH_SOURCE[0]}" )" &> /dev/null && pwd)
+PACKAGE_ROOT=$(dirname "${SCRIPTS_ROOT}")
+
+build_subcommand=(build)
+# if buildx is installed, use it
+if docker buildx version &> /dev/null; then
+    build_subcommand=(buildx build --platform linux/amd64 --output type=image)
+fi
+
+docker "${build_subcommand[@]}" -f $PACKAGE_ROOT/Dockerfile -t tag_ulog:latest $PACKAGE_ROOT

actions/tag_ulog/scripts/deploy.sh

@@ -0,0 +1,50 @@
+#!/usr/bin/env bash
+
+set -euo pipefail
+
+SCRIPTS_ROOT=$( cd -- "$( dirname -- "${BASH_SOURCE[0]}" )" &> /dev/null && pwd)
+PACKAGE_ROOT=$(dirname "${SCRIPTS_ROOT}")
+
+# Early exit if virtual environment does not exist and/or roboto is not yet installed
+if [ ! -f "$PACKAGE_ROOT/.venv/bin/roboto" ]; then
+    echo "Virtual environment with roboto CLI does not exist. Please run ./scripts/setup.sh first."
+    exit 1
+fi
+
+# Set org_id to $ROBOTO_ORG_ID if defined, else the first argument passed to this script
+org_id=${ROBOTO_ORG_ID:-}
+if [ $# -gt 0 ]; then
+    org_id=$1  
+fi
+
+roboto_exe="$PACKAGE_ROOT/.venv/bin/roboto"
+
+echo "Pushing tag_ulog:latest to Roboto's private registry"
+image_push_args=(
+    --suppress-upgrade-check
+    images push
+    --quiet
+)
+if [[ -n $org_id ]]; then
+    image_push_args+=(--org $org_id)
+fi
+image_push_args+=(tag_ulog:latest)
+image_push_ret_code=0
+image_uri=$($roboto_exe "${image_push_args[@]}")
+image_push_ret_code=$?
+
+if [ $image_push_ret_code -ne 0 ]; then
+    echo "Failed to push tag_ulog:latest to Roboto's private registry"
+    exit 1
+fi
+
+echo "Creating tag_ulog action"
+create_args=(
+  --from-file $PACKAGE_ROOT/action.json
+  --image $image_uri
+  --yes
+)
+if [[ -n $org_id ]]; then
+    create_args+=(--org $org_id)
+fi
+$roboto_exe actions create "${create_args[@]}"

actions/tag_ulog/scripts/run.sh

@@ -0,0 +1,40 @@
+#!/usr/bin/env bash
+
+set -euo pipefail
+
+SCRIPTS_ROOT=$( cd -- "$( dirname -- "${BASH_SOURCE[0]}" )" &> /dev/null && pwd)
+PACKAGE_ROOT=$(dirname "${SCRIPTS_ROOT}")
+
+# Set input_dir to $ROBOTO_INPUT_DIR if defined, else the first argument passed to this script
+input_dir=${ROBOTO_INPUT_DIR:-}
+if [ $# -gt 0 ]; then
+    input_dir=$1  
+fi
+
+# Fail if input_dir is not an existing directory
+if [ ! -d "$input_dir" ]; then
+    echo "Specify an existing input directory as the first argument to this script, or set the ROBOTO_INPUT_DIR environment variable"
+    exit 1
+fi
+
+# Set output_dir variable to $ROBOTO_OUTPUT_DIR if defined, else set it to "output/" in the package root (creating if necessary)
+output_dir=${ROBOTO_OUTPUT_DIR:-$PACKAGE_ROOT/output}
+mkdir -p $output_dir
+
+# Assert both directories are absolute paths
+if [[ ! "$input_dir" = /* ]]; then
+    echo "Input directory '$input_dir' must be specified as an absolute path"
+    exit 1
+fi
+
+if [[ ! "$output_dir" = /* ]]; then
+    echo "Output directory '$output_dir' must be specified as an absolute path"
+    exit 1
+fi
+
+docker run --rm -it \
+    -v $input_dir:/input \
+    -v $output_dir:/output \
+    -e ROBOTO_INPUT_DIR=/input \
+    -e ROBOTO_OUTPUT_DIR=/output \
+    tag_ulog:latest

actions/tag_ulog/scripts/setup.sh

@@ -0,0 +1,15 @@
+#!/usr/bin/env bash
+
+set -euo pipefail
+
+SCRIPTS_ROOT=$( cd -- "$( dirname -- "${BASH_SOURCE[0]}" )" &> /dev/null && pwd)
+PACKAGE_ROOT=$(dirname "${SCRIPTS_ROOT}")
+
+venv_dir="$PACKAGE_ROOT/.venv"
+
+# Create a virtual environment
+python -m venv --upgrade-deps $venv_dir
+
+# Install roboto
+pip_exe="$venv_dir/bin/pip"
+$pip_exe install --upgrade -r $PACKAGE_ROOT/requirements.dev.txt

actions/tag_ulog/scripts/test.sh

@@ -0,0 +1,83 @@
+#!/bin/bash
+
+SCRIPTS_ROOT=$( cd -- "$( dirname -- "${BASH_SOURCE[0]}" )" &> /dev/null && pwd)
+PACKAGE_ROOT=$(dirname "${SCRIPTS_ROOT}")
+
+# Define constants for directories and file paths
+
+INPUT_DIR=${PACKAGE_ROOT}/test/input
+ACTUAL_OUTPUT_DIR=${PACKAGE_ROOT}/test/actual_output
+EXPECTED_OUTPUT_DIR=${PACKAGE_ROOT}/test/expected_output
+
+if [ ! -d "$ACTUAL_OUTPUT_DIR" ]; then
+    mkdir -p "$ACTUAL_OUTPUT_DIR"
+fi
+
+# Remove previous outputs
+clean_actual_output() {
+    rm -rf $ACTUAL_OUTPUT_DIR/
+}
+
+# Check if file exists
+file_exists_or_error() {
+    local file_path="$1"
+
+    if [ ! -f "$file_path" ]; then
+        echo "Error: File '$file_path' does not exist."
+        exit 1
+    fi
+    echo "Test passed!"
+
+}
+
+# Run the docker command with the given parameters
+run_docker_test() {
+    local additional_args="$1"
+
+    docker run \
+        -v $INPUT_DIR:/input \
+        -v $ACTUAL_OUTPUT_DIR:/output \
+        -e ROBOTO_INPUT_DIR=/input \
+        -e ROBOTO_OUTPUT_DIR=/output \
+        $additional_args \
+        tag_ulog:latest
+}
+
+function check_file_does_not_exist() {
+    local file_path="$1"
+    if [[ ! -e "$file_path" ]]; then
+        echo "Test passed!"
+    else
+        echo "Test failed: $1 exists!"
+        exit 1
+    fi
+}
+
+# Compare the actual output to the expected output
+compare_outputs() {
+    local actual_file="$1"
+    local expected_file="$2"
+
+    diff $ACTUAL_OUTPUT_DIR/$actual_file $EXPECTED_OUTPUT_DIR/$expected_file
+
+    if [ $? -eq 0 ]; then
+        echo "Test passed!"
+    else
+        echo "Test failed!"
+	exit 1
+    fi
+}
+
+# Main test execution
+main() {
+
+    # Test 1
+    echo "Running Test 1: Test changeset.json creation"
+    clean_actual_output
+    run_docker_test ""
+    file_exists_or_error $ACTUAL_OUTPUT_DIR/changeset.json
+}
+
+# Run the main test execution
+main
+clean_actual_output

actions/tag_ulog/src/tag_ulog/__main__.py

@@ -0,0 +1,78 @@
+from pyulog.core import ULog
+from roboto.domain import actions
+
+import argparse
+import logging
+import os
+import pathlib
+import json
+import re
+logging.basicConfig(level=logging.INFO)
+logger = logging.getLogger(__name__)
+
+
+def extract_text_between_brackets(text):
+    match = re.search(r'\[([^\]]+)\]', text)
+    if match:
+        return match.group(1)
+    else:
+        return None
+
+
+parser = argparse.ArgumentParser()
+parser.add_argument(
+    "-i",
+    "--input-dir",
+    dest="input_dir",
+    type=pathlib.Path,
+    required=False,
+    help="Directory containing input files to process",
+    default=os.environ.get(actions.InvocationEnvVar.InputDir.value),
+)
+
+parser.add_argument(
+    "-o",
+    "--output-dir",
+    dest="output_dir",
+    type=pathlib.Path,
+    required=False,
+    help="Directory to which to write any output files to be uploaded",
+    default=os.environ.get(actions.InvocationEnvVar.OutputDir.value),
+)
+
+
+args = parser.parse_args()
+
+output_path_metadata = os.getenv("ROBOTO_DATASET_METADATA_CHANGESET_FILE", "/output/changeset.json")
+
+put_tags = list()
+
+put_fields = dict()
+
+for root, dirs, files in os.walk(args.input_dir):
+    for file in files:
+        # Check if the file ends with .ulg
+        if file.endswith(".ulg"):
+            _, ulg_file_name = os.path.split(file)
+            msg_filter = []
+            full_path = os.path.join(root, file)
+            ulog = ULog(full_path, msg_filter, True)
+            for m in ulog.logged_messages:
+                if m.log_level_str() == "ERROR":
+                    module_name = extract_text_between_brackets(m.message)
+                    if module_name is not None:
+                        module_name = "ERROR_" + module_name
+                        if module_name not in put_tags:
+                            put_tags.append(module_name)
+                    continue
+                print(f"{m.log_level_str()} {m.message}")
+
+    with open(output_path_metadata, "w") as json_file:
+        metadata_dict = {
+            "put_tags": list(set(put_tags)),
+            "remove_tags": [],
+            "put_fields": put_fields,
+            "remove_fields": [],
+        }
+        logger.info(f"Writing {output_path_metadata}...")
+        json.dump(metadata_dict, json_file, indent=4)