From 54eae121bc8b2992905ac3e5637dde8e332afc6f Mon Sep 17 00:00:00 2001 From: Yves Date: Mon, 11 Mar 2024 10:26:24 +0100 Subject: [PATCH] backup --- .../ulog_ingestion/requirements.runtime.txt | 2 +- .../ulog_ingestion/src/tests/test_utils.py | 12 +++++ .../src/ulog_ingestion/__main__.py | 30 +++++++----- .../src/ulog_ingestion/utils.py | 47 ++++++++++++++++++- 4 files changed, 78 insertions(+), 13 deletions(-) diff --git a/actions/ulog_ingestion/requirements.runtime.txt b/actions/ulog_ingestion/requirements.runtime.txt index c621319..b7b8618 100644 --- a/actions/ulog_ingestion/requirements.runtime.txt +++ b/actions/ulog_ingestion/requirements.runtime.txt @@ -1,5 +1,5 @@ # Python packages to install within the Docker image associated with this Action. -roboto==0.2.11 +roboto==0.2.12 pyulog==1.0.2 mcap==1.1.1 jsonschema>=4.21.1 diff --git a/actions/ulog_ingestion/src/tests/test_utils.py b/actions/ulog_ingestion/src/tests/test_utils.py index 76877f9..f94d1c2 100644 --- a/actions/ulog_ingestion/src/tests/test_utils.py +++ b/actions/ulog_ingestion/src/tests/test_utils.py @@ -45,3 +45,15 @@ def test_is_valid_ulog(): ulog_file_path = "./tests/test.ulg" is_valid = utils.is_valid_ulog(ulog_file_path) assert is_valid is True + + +# def test_add_metadata(): +# ulog_file_path = "./tests/test.ulg" +# #topics = ["vehicle_acceleration"] +# +# utils.add_metadata(ulog_file_path, topics=None) + + # assert os.path.exists(ulog_file_path) + # assert os.path.exists(f"{ulog_file_path}.metadata.json") + # os.remove(f"{ulog_file_path}.metadata.json") + # assert not os.path.exists(f"{ulog_file_path}.metadata.json") diff --git a/actions/ulog_ingestion/src/ulog_ingestion/__main__.py b/actions/ulog_ingestion/src/ulog_ingestion/__main__.py index 02aebc4..1a11980 100644 --- a/actions/ulog_ingestion/src/ulog_ingestion/__main__.py +++ b/actions/ulog_ingestion/src/ulog_ingestion/__main__.py @@ -6,6 +6,7 @@ import sys import pathlib import time +from roboto.env import RobotoEnvKey from concurrent.futures import ProcessPoolExecutor, as_completed from roboto.association import ( @@ -24,7 +25,7 @@ log = logging.getLogger("Ingesting ULog files to Roboto") -def load_env_var(env_var: actions.InvocationEnvVar) -> str: +def load_env_var(env_var: RobotoEnvKey) -> str: """ Load an environment variable, and exit if it is not found. @@ -48,11 +49,11 @@ def setup_env(): Returns: - A tuple containing the organization ID, input directory, output directory, topic delegate, and dataset. """ - roboto_service_url = load_env_var(actions.InvocationEnvVar.RobotoServiceUrl) - org_id = load_env_var(actions.InvocationEnvVar.OrgId) - invocation_id = load_env_var(actions.InvocationEnvVar.InvocationId) - input_dir = load_env_var(actions.InvocationEnvVar.InputDir) - output_dir = load_env_var(actions.InvocationEnvVar.OutputDir) + roboto_service_url = load_env_var(RobotoEnvKey.RobotoServiceUrl) + org_id = load_env_var(RobotoEnvKey.OrgId) + invocation_id = load_env_var(RobotoEnvKey.InvocationId) + input_dir = load_env_var(RobotoEnvKey.InputDir) + output_dir = load_env_var(RobotoEnvKey.OutputDir) http_client = HttpClient(default_auth=SigV4AuthDecorator("execute-api")) @@ -64,9 +65,7 @@ def setup_env(): invocation_id, invocation_delegate=actions.InvocationHttpDelegate( roboto_service_base_url=roboto_service_url, http_client=http_client - ), - org_id=org_id, - ) + )) dataset = datasets.Dataset.from_id( invocation.data_source.data_source_id, datasets.DatasetHttpDelegate( @@ -160,6 +159,7 @@ def process_data( relative_file_name = output_path_per_topic_mcap.split(output_dir_temp)[1][1:] + # Upload MCAP File dataset.upload_file( pathlib.Path(output_path_per_topic_mcap), relative_file_name @@ -167,6 +167,12 @@ def process_data( file_id = dataset.get_file_info(relative_file_name).file_id + + relative_file_name = ulog_file_path.split(input_dir)[1][1:] + print( + f"https://app-beta.roboto.ai/visualize/{utils.generate_config(file_record.file_id, relative_file_name)}" + ) + print( f"Setting default representation for topic: {topic_name_roboto}, file_id: {file_id}" ) @@ -260,7 +266,7 @@ def ingest_ulog(ulog_file_path: str, topics: List[str] = None): type=pathlib.Path, required=False, help="Directory containing input files to process", - default=os.environ.get(actions.InvocationEnvVar.InputDir.value), + default=os.environ.get(RobotoEnvKey.InputDir.value), ) parser.add_argument( @@ -270,7 +276,7 @@ def ingest_ulog(ulog_file_path: str, topics: List[str] = None): type=pathlib.Path, required=False, help="Directory to which to write any output files to be uploaded", - default=os.environ.get(actions.InvocationEnvVar.OutputDir.value), + default=os.environ.get(RobotoEnvKey.OutputDir.value), ) parser.add_argument( @@ -290,6 +296,8 @@ def ingest_ulog(ulog_file_path: str, topics: List[str] = None): full_path = os.path.join(root, file) if utils.is_valid_ulog(full_path): + utils.add_metadata_to_file(full_path, topics=None) + ingest_ulog( ulog_file_path=full_path, topics=args.topic_names, diff --git a/actions/ulog_ingestion/src/ulog_ingestion/utils.py b/actions/ulog_ingestion/src/ulog_ingestion/utils.py index e768411..ce1e5f3 100644 --- a/actions/ulog_ingestion/src/ulog_ingestion/utils.py +++ b/actions/ulog_ingestion/src/ulog_ingestion/utils.py @@ -1,10 +1,15 @@ import hashlib -import json +from roboto.env import RobotoEnvKey +import pathlib import os import tempfile from mcap.writer import Writer from typing import Dict, List, Tuple, Any from roboto.domain import topics +from pyulog.core import ULog +import base64 +import json + # Mapping from message definition types to JSON schema types TYPE_MAPPING = { @@ -321,3 +326,43 @@ def is_valid_ulog(ulog_file_path: str) -> bool: raise TypeError("Invalid ULog file format (Failed to parse header)") return True + + +# Helper function. Will be deleted. +def generate_config(file_id, relative_path): + viz_config = { + "version": "v1", + "files": [{"fileId": file_id, "relativePath": relative_path}], + } + return base64.urlsafe_b64encode(json.dumps(viz_config).encode("utf-8")).decode( + "utf-8" + ) + + +def add_metadata_to_file(ulog_file_path: str, topics: List[str] = None): + + msg_filter = topics.split(",") if topics else None + ulog = ULog(ulog_file_path, msg_filter, True) + + input_dir = os.environ[f"{RobotoEnvKey.InputDir.value}"] + relative_file_name = ulog_file_path.split(input_dir)[1][1:] + + file_metadata_changeset_file_path = os.environ[f"{RobotoEnvKey.FileMetadataChangesetFile.value}"] + file_metadata_changeset_file = pathlib.Path(file_metadata_changeset_file_path) + + json_line = json.dumps( + { + "relative_path": relative_file_name, + "update": { + "metadata_changeset": { + "put_fields": ulog.msg_info_dict, + }, + "description": "", + }, + } + ) + + with file_metadata_changeset_file.open('a') as file: + if file.tell() > 0: + file.write('\n') + file.write(json_line)