Add file-based metadata

actions/ulog_ingestion/src/tests/test_utils.py

@@ -45,3 +45,15 @@ def test_is_valid_ulog():
     ulog_file_path = "./tests/test.ulg"
     is_valid = utils.is_valid_ulog(ulog_file_path)
     assert is_valid is True
+
+
+# def test_add_metadata():
+#     ulog_file_path = "./tests/test.ulg"
+#     #topics = ["vehicle_acceleration"]
+#
+#     utils.add_metadata(ulog_file_path, topics=None)
+
+    # assert os.path.exists(ulog_file_path)
+    # assert os.path.exists(f"{ulog_file_path}.metadata.json")
+    # os.remove(f"{ulog_file_path}.metadata.json")
+    # assert not os.path.exists(f"{ulog_file_path}.metadata.json")

actions/ulog_ingestion/src/ulog_ingestion/__main__.py

@@ -6,6 +6,7 @@
 import sys
 import pathlib
 import time
+from roboto.env import RobotoEnvKey
 from concurrent.futures import ProcessPoolExecutor, as_completed
 
 from roboto.association import (
@@ -160,13 +161,20 @@ def process_data(
 
     relative_file_name = output_path_per_topic_mcap.split(output_dir_temp)[1][1:]
 
+
     # Upload MCAP File
     dataset.upload_file(
         pathlib.Path(output_path_per_topic_mcap), relative_file_name
     )
 
     file_id = dataset.get_file_info(relative_file_name).file_id
 
+
+    relative_file_name = ulog_file_path.split(input_dir)[1][1:]
+    print(
+        f"https://app-beta.roboto.ai/visualize/{utils.generate_config(file_record.file_id, relative_file_name)}"
+    )
+
     print(
         f"Setting default representation for topic: {topic_name_roboto}, file_id: {file_id}"
     )
@@ -292,6 +300,8 @@ def ingest_ulog(ulog_file_path: str, topics: List[str] = None):
         full_path = os.path.join(root, file)
         if utils.is_valid_ulog(full_path):
 
+            utils.add_metadata_to_file(full_path, topics=None)
+
             ingest_ulog(
                 ulog_file_path=full_path,
                 topics=args.topic_names,

actions/ulog_ingestion/src/ulog_ingestion/utils.py

@@ -1,12 +1,16 @@
 import hashlib
-import json
+from roboto.env import RobotoEnvKey
+import pathlib
 import os
 import tempfile
 import math
 from mcap.writer import Writer
 from typing import Dict, List, Tuple, Any
 from roboto.domain import topics
 import base64
+from pyulog.core import ULog
+import json
+
 
 # Mapping from message definition types to JSON schema types
 TYPE_MAPPING = {
@@ -337,3 +341,43 @@ def is_valid_ulog(ulog_file_path: str) -> bool:
             raise TypeError("Invalid ULog file format (Failed to parse header)")
 
         return True
+
+
+# Helper function. Will be deleted.
+def generate_config(file_id, relative_path):
+    viz_config = {
+        "version": "v1",
+        "files": [{"fileId": file_id, "relativePath": relative_path}],
+    }
+    return base64.urlsafe_b64encode(json.dumps(viz_config).encode("utf-8")).decode(
+        "utf-8"
+    )
+
+
+def add_metadata_to_file(ulog_file_path: str, topics: List[str] = None):
+
+    msg_filter = topics.split(",") if topics else None
+    ulog = ULog(ulog_file_path, msg_filter, True)
+
+    input_dir = os.environ[f"{RobotoEnvKey.InputDir.value}"]
+    relative_file_name = ulog_file_path.split(input_dir)[1][1:]
+
+    file_metadata_changeset_file_path = os.environ[f"{RobotoEnvKey.FileMetadataChangesetFile.value}"]
+    file_metadata_changeset_file = pathlib.Path(file_metadata_changeset_file_path)
+
+    json_line = json.dumps(
+        {
+            "relative_path": relative_file_name,
+            "update": {
+                "metadata_changeset": {
+                    "put_fields": ulog.msg_info_dict,
+                },
+                "description": "",
+            },
+        }
+    )
+
+    with file_metadata_changeset_file.open('a') as file:
+        if file.tell() > 0:
+            file.write('\n')
+        file.write(json_line)