backup

actions/ulog_ingestion/requirements.runtime.txt

@@ -1,5 +1,5 @@
 # Python packages to install within the Docker image associated with this Action.
-roboto==0.2.11
+roboto==0.2.12
 pyulog==1.0.2
 mcap==1.1.1
 jsonschema>=4.21.1

actions/ulog_ingestion/src/tests/test_utils.py

@@ -45,3 +45,15 @@ def test_is_valid_ulog():
     ulog_file_path = "./tests/test.ulg"
     is_valid = utils.is_valid_ulog(ulog_file_path)
     assert is_valid is True
+
+
+# def test_add_metadata():
+#     ulog_file_path = "./tests/test.ulg"
+#     #topics = ["vehicle_acceleration"]
+#
+#     utils.add_metadata(ulog_file_path, topics=None)
+
+    # assert os.path.exists(ulog_file_path)
+    # assert os.path.exists(f"{ulog_file_path}.metadata.json")
+    # os.remove(f"{ulog_file_path}.metadata.json")
+    # assert not os.path.exists(f"{ulog_file_path}.metadata.json")

actions/ulog_ingestion/src/ulog_ingestion/__main__.py

@@ -6,6 +6,7 @@
 import sys
 import pathlib
 import time
+from roboto.env import RobotoEnvKey
 from concurrent.futures import ProcessPoolExecutor, as_completed
 
 from roboto.association import (
@@ -24,7 +25,7 @@
 log = logging.getLogger("Ingesting ULog files to Roboto")
 
 
-def load_env_var(env_var: actions.InvocationEnvVar) -> str:
+def load_env_var(env_var: RobotoEnvKey) -> str:
     """
     Load an environment variable, and exit if it is not found.
 
@@ -48,11 +49,11 @@ def setup_env():
     Returns:
     - A tuple containing the organization ID, input directory, output directory, topic delegate, and dataset.
     """
-    roboto_service_url = load_env_var(actions.InvocationEnvVar.RobotoServiceUrl)
-    org_id = load_env_var(actions.InvocationEnvVar.OrgId)
-    invocation_id = load_env_var(actions.InvocationEnvVar.InvocationId)
-    input_dir = load_env_var(actions.InvocationEnvVar.InputDir)
-    output_dir = load_env_var(actions.InvocationEnvVar.OutputDir)
+    roboto_service_url = load_env_var(RobotoEnvKey.RobotoServiceUrl)
+    org_id = load_env_var(RobotoEnvKey.OrgId)
+    invocation_id = load_env_var(RobotoEnvKey.InvocationId)
+    input_dir = load_env_var(RobotoEnvKey.InputDir)
+    output_dir = load_env_var(RobotoEnvKey.OutputDir)
 
     http_client = HttpClient(default_auth=SigV4AuthDecorator("execute-api"))
 
@@ -64,9 +65,7 @@ def setup_env():
         invocation_id,
         invocation_delegate=actions.InvocationHttpDelegate(
             roboto_service_base_url=roboto_service_url, http_client=http_client
-        ),
-        org_id=org_id,
-    )
+        ))
     dataset = datasets.Dataset.from_id(
         invocation.data_source.data_source_id,
         datasets.DatasetHttpDelegate(
@@ -160,13 +159,20 @@ def process_data(
 
     relative_file_name = output_path_per_topic_mcap.split(output_dir_temp)[1][1:]
 
+
     # Upload MCAP File
     dataset.upload_file(
         pathlib.Path(output_path_per_topic_mcap), relative_file_name
     )
 
     file_id = dataset.get_file_info(relative_file_name).file_id
 
+
+    relative_file_name = ulog_file_path.split(input_dir)[1][1:]
+    print(
+        f"https://app-beta.roboto.ai/visualize/{utils.generate_config(file_record.file_id, relative_file_name)}"
+    )
+
     print(
         f"Setting default representation for topic: {topic_name_roboto}, file_id: {file_id}"
     )
@@ -260,7 +266,7 @@ def ingest_ulog(ulog_file_path: str, topics: List[str] = None):
     type=pathlib.Path,
     required=False,
     help="Directory containing input files to process",
-    default=os.environ.get(actions.InvocationEnvVar.InputDir.value),
+    default=os.environ.get(RobotoEnvKey.InputDir.value),
 )
 
 parser.add_argument(
@@ -270,7 +276,7 @@ def ingest_ulog(ulog_file_path: str, topics: List[str] = None):
     type=pathlib.Path,
     required=False,
     help="Directory to which to write any output files to be uploaded",
-    default=os.environ.get(actions.InvocationEnvVar.OutputDir.value),
+    default=os.environ.get(RobotoEnvKey.OutputDir.value),
 )
 
 parser.add_argument(
@@ -290,6 +296,8 @@ def ingest_ulog(ulog_file_path: str, topics: List[str] = None):
         full_path = os.path.join(root, file)
         if utils.is_valid_ulog(full_path):
 
+            utils.add_metadata_to_file(full_path, topics=None)
+
             ingest_ulog(
                 ulog_file_path=full_path,
                 topics=args.topic_names,

actions/ulog_ingestion/src/ulog_ingestion/utils.py

@@ -1,10 +1,15 @@
 import hashlib
-import json
+from roboto.env import RobotoEnvKey
+import pathlib
 import os
 import tempfile
 from mcap.writer import Writer
 from typing import Dict, List, Tuple, Any
 from roboto.domain import topics
+from pyulog.core import ULog
+import base64
+import json
+
 
 # Mapping from message definition types to JSON schema types
 TYPE_MAPPING = {
@@ -321,3 +326,43 @@ def is_valid_ulog(ulog_file_path: str) -> bool:
             raise TypeError("Invalid ULog file format (Failed to parse header)")
 
         return True
+
+
+# Helper function. Will be deleted.
+def generate_config(file_id, relative_path):
+    viz_config = {
+        "version": "v1",
+        "files": [{"fileId": file_id, "relativePath": relative_path}],
+    }
+    return base64.urlsafe_b64encode(json.dumps(viz_config).encode("utf-8")).decode(
+        "utf-8"
+    )
+
+
+def add_metadata_to_file(ulog_file_path: str, topics: List[str] = None):
+
+    msg_filter = topics.split(",") if topics else None
+    ulog = ULog(ulog_file_path, msg_filter, True)
+
+    input_dir = os.environ[f"{RobotoEnvKey.InputDir.value}"]
+    relative_file_name = ulog_file_path.split(input_dir)[1][1:]
+
+    file_metadata_changeset_file_path = os.environ[f"{RobotoEnvKey.FileMetadataChangesetFile.value}"]
+    file_metadata_changeset_file = pathlib.Path(file_metadata_changeset_file_path)
+
+    json_line = json.dumps(
+        {
+            "relative_path": relative_file_name,
+            "update": {
+                "metadata_changeset": {
+                    "put_fields": ulog.msg_info_dict,
+                },
+                "description": "",
+            },
+        }
+    )
+
+    with file_metadata_changeset_file.open('a') as file:
+        if file.tell() > 0:
+            file.write('\n')
+        file.write(json_line)