[measurement] HDF5 v6, HDF5 API v3 (#1375)

2 Hdf5 file format versions supported for writing (v5, v6).
2 API versions available (v2, v3).
V6 Supports storing multiple channels with the same name.
All eCAL tools still write file format version v5, using API version v2.

app/meas_cutter/CMakeLists.txt

@@ -55,7 +55,7 @@ target_include_directories(${PROJECT_NAME}
 target_link_libraries(${PROJECT_NAME}   yaml-cpp::yaml-cpp
                                         tclap::tclap
                                         eCAL::ecal-utils
-                                        eCAL::measurement_hdf5
+                                        eCAL::hdf5
                                         Threads::Threads)
 
 target_compile_features(${PROJECT_NAME} PRIVATE cxx_std_14) 

app/meas_cutter/src/measurement_exporter.cpp

@@ -1,6 +1,6 @@
 /* ========================= eCAL LICENSE =================================
  *
- * Copyright (C) 2016 - 2019 Continental Corporation
+ * Copyright (C) 2016 - 2024 Continental Corporation
  *
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
@@ -18,18 +18,17 @@
 */
 
 #include "measurement_exporter.h"
-#include <ecal/measurement/hdf5/writer.h>
 
 MeasurementExporter::MeasurementExporter():
-  _writer(std::make_unique<eCAL::experimental::measurement::hdf5::Writer>())
+  _writer(std::make_unique<eCAL::eh5::v2::HDF5Meas>())
 {
 }
 
 void MeasurementExporter::setPath(const std::string& path, const std::string& base_name, const size_t& max_size_per_file)
 {
   _root_output_path = EcalUtils::Filesystem::CleanPath(path);
   _output_path = EcalUtils::Filesystem::CleanPath(_root_output_path + EcalUtils::Filesystem::NativeSeparator(EcalUtils::Filesystem::OsStyle::Current) + eCALMeasCutterUtils::kDefaultFolderOutput, EcalUtils::Filesystem::OsStyle::Current);
-  if (!_writer->Open(_output_path))
+  if (!_writer->Open(_output_path, eCAL::eh5::v2::eAccessType::CREATE))
   {
     throw ExporterException("Unable to create HDF5 protobuf output path " + path + ".");
   }
@@ -76,10 +75,10 @@ void MeasurementExporter::setData(eCALMeasCutterUtils::Timestamp timestamp, cons
   const auto sender_timestamp = (iter != meta_data.end()) ? iter->second.sender_timestamp : static_cast<eCALMeasCutterUtils::Timestamp>(0);
 
   iter = meta_data.find(eCALMeasCutterUtils::MetaDatumKey::SENDER_ID);
-  const auto sender_id = (iter != meta_data.end()) ? iter->second.sender_id : static_cast<uint64_t>(0);
+  const auto sender_id = (iter != meta_data.end()) ? iter->second.sender_id : 0;
 
   iter = meta_data.find(eCALMeasCutterUtils::MetaDatumKey::SENDER_CLOCK);
-  const auto sender_clock = (iter != meta_data.end()) ? iter->second.sender_clock : static_cast<uint64_t>(0);
+  const auto sender_clock = (iter != meta_data.end()) ? iter->second.sender_clock : 0;
 
   if (!_writer->AddEntryToFile(payload.data(), payload.size(), sender_timestamp, timestamp, _current_channel_name, sender_id, sender_clock))
   {

app/meas_cutter/src/measurement_exporter.h

@@ -1,6 +1,6 @@
 /* ========================= eCAL LICENSE =================================
  *
- * Copyright (C) 2016 - 2019 Continental Corporation
+ * Copyright (C) 2016 - 2024 Continental Corporation
  *
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
@@ -18,11 +18,14 @@
 */
 
 #pragma once
+
 #include <iostream>
-#include <unordered_map>
 #include <map>
+#include <memory>
+#include <string>
+#include <unordered_map>
 
-#include <ecal/measurement/base/writer.h>
+#include <ecalhdf5/eh5_meas.h>
 #include <ecal_utils/filesystem.h>
 #include "utils.h"
 
@@ -43,7 +46,7 @@ class MeasurementExporter
   std::string getRootOutputPath() const;
 
 private:
-  std::unique_ptr<eCAL::experimental::measurement::base::Writer>      _writer;
+  std::unique_ptr<eCAL::eh5::v2::HDF5Meas>              _writer;
   std::string                                           _current_channel_name;
   std::string                                           _output_path;
   std::string                                           _root_output_path;

app/meas_cutter/src/measurement_importer.cpp

@@ -1,6 +1,6 @@
 /* ========================= eCAL LICENSE =================================
  *
- * Copyright (C) 2016 - 2019 Continental Corporation
+ * Copyright (C) 2016 - 2024 Continental Corporation
  *
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
@@ -18,10 +18,10 @@
 */
 
 #include "measurement_importer.h"
-#include <ecal/measurement/hdf5/reader.h>
+#include <ecalhdf5/eh5_meas.h>
 
 MeasurementImporter::MeasurementImporter() :
-  _reader(std::make_unique<eCAL::experimental::measurement::hdf5::Reader>()),
+  _reader(std::make_unique<eCAL::eh5::v2::HDF5Meas>()),
   _current_opened_channel_data()
 {
 }

app/meas_cutter/src/measurement_importer.h

@@ -1,6 +1,6 @@
 /* ========================= eCAL LICENSE =================================
  *
- * Copyright (C) 2016 - 2019 Continental Corporation
+ * Copyright (C) 2016 - 2024 Continental Corporation
  *
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
@@ -18,14 +18,16 @@
 */
 
 #pragma once
-#include <iostream>
 #include <algorithm>
-#include <regex>
 #include <cctype>
-#include <array>
+#include <iostream>
+#include <memory>
+#include <list>
+#include <regex>
+#include <utility>
 
 #include <ecal_utils/filesystem.h>
-#include <ecal/measurement/base/reader.h>
+#include <ecalhdf5/eh5_meas.h>
 
 #include "utils.h"
 
@@ -53,7 +55,7 @@ class MeasurementImporter
 private:
   bool                                 isEcalMeasFile(const std::string& path);
   bool                                 isProtoChannel(const eCAL::experimental::measurement::base::DataTypeInformation& channel_info);
-  std::unique_ptr<eCAL::experimental::measurement::base::Reader>      _reader;
+  std::unique_ptr<eCAL::eh5::v2::HDF5Meas>              _reader;
   eCALMeasCutterUtils::ChannelData                      _current_opened_channel_data;
   std::string                                           _loaded_path;
   eCALMeasCutterUtils::ChannelNameSet                   _channel_names;

app/meas_cutter/src/utils.h

@@ -312,10 +312,10 @@ namespace eCALMeasCutterUtils
   {
     Timestamp receiver_timestamp;
     Timestamp sender_timestamp;
-    uint64_t sender_id;
-    uint64_t sender_clock;
+    int64_t sender_id;
+    int64_t sender_clock;
 
-      std::array<char,64> __union_size;
+    std::array<char,64> __union_size;
   };
 
   typedef std::unordered_map<MetaDatumKey, MetaDatumValue, MetaDatumHash> MetaData;

app/play/play_core/CMakeLists.txt

@@ -1,6 +1,6 @@
 # ========================= eCAL LICENSE =================================
 #
-# Copyright (C) 2016 - 2019 Continental Corporation
+# Copyright (C) 2016 - 2024 Continental Corporation
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
@@ -58,7 +58,7 @@ target_link_libraries(${PROJECT_NAME} PUBLIC
   eCAL::core_protobuf
   eCAL::app_pb
   eCAL::ecaltime_pb
-  eCAL::measurement_hdf5
+  eCAL::hdf5
 )
 
 target_link_libraries(${PROJECT_NAME} PRIVATE

app/play/play_core/src/ecal_play.cpp

@@ -1,6 +1,6 @@
 /* ========================= eCAL LICENSE =================================
  *
- * Copyright (C) 2016 - 2019 Continental Corporation
+ * Copyright (C) 2016 - 2024 Continental Corporation
  *
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
@@ -19,16 +19,16 @@
 
 #include "ecal_play.h"
 
-#include <iostream>
+#include <chrono>
+#include <clocale>
 #include <fstream>
+#include <iostream>
+#include <memory>
 #include <sstream>
-#include <clocale>
-#include <chrono>
 
 #include "ecal_play_logger.h"
 #include "play_thread.h"
-#include <ecal/measurement/hdf5/reader.h>
-
+#include <ecalhdf5/eh5_meas.h>
 #include <ecal_utils/string.h>
 #include <ecal_utils/filesystem.h>
 #include <ecal_utils/str_convert.h>
@@ -58,7 +58,7 @@ bool EcalPlay::LoadMeasurement(const std::string& path)
 {
   EcalPlayLogger::Instance()->info("Loading measurement...");
 
-  std::shared_ptr<eCAL::experimental::measurement::base::Reader> measurement(std::make_shared<eCAL::experimental::measurement::hdf5::Reader>());
+  std::shared_ptr<eCAL::eh5::v2::HDF5Meas> measurement(std::make_shared<eCAL::eh5::v2::HDF5Meas>());
 
   std::string meas_dir;               // The directory of the measurement
   std::string path_to_load;           // The actual path we load the measurement from. May be a directory or a .hdf5 file
@@ -129,7 +129,7 @@ bool EcalPlay::LoadMeasurement(const std::string& path)
 void EcalPlay::CloseMeasurement()
 {
   description_ = "";
-  play_thread_->SetMeasurement(std::shared_ptr<eCAL::experimental::measurement::base::Reader>(nullptr));
+  play_thread_->SetMeasurement(std::shared_ptr<eCAL::eh5::v2::HDF5Meas>(nullptr));
   measurement_path_ = "";
   clearScenariosPath();
   channel_mapping_path_ = "";

app/play/play_core/src/measurement_container.cpp

@@ -1,6 +1,6 @@
 /* ========================= eCAL LICENSE =================================
  *
- * Copyright (C) 2016 - 2019 Continental Corporation
+ * Copyright (C) 2016 - 2024 Continental Corporation
  *
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
@@ -20,13 +20,13 @@
 #include "measurement_container.h"
 
 #include <ecal/ecal_util.h>
-#include <ecal/measurement/hdf5/reader.h>
+#include <ecalhdf5/eh5_meas.h>
 
 #include <algorithm>
 #include <math.h>
 #include <stdlib.h>
 
-MeasurementContainer::MeasurementContainer(std::shared_ptr<eCAL::experimental::measurement::base::Reader> hdf5_meas, const std::string& meas_dir, bool use_receive_timestamp)
+MeasurementContainer::MeasurementContainer(std::shared_ptr<eCAL::eh5::v2::HDF5Meas> hdf5_meas, const std::string& meas_dir, bool use_receive_timestamp)
   : hdf5_meas_             (hdf5_meas)
   , meas_dir_              (meas_dir)
   , use_receive_timestamp_ (use_receive_timestamp)

app/play/play_core/src/measurement_container.h

@@ -1,6 +1,6 @@
 /* ========================= eCAL LICENSE =================================
  *
- * Copyright (C) 2016 - 2019 Continental Corporation
+ * Copyright (C) 2016 - 2024 Continental Corporation
  *
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
@@ -24,14 +24,14 @@
 #include <memory>
 
 #include <ecal/ecal.h>
-#include <ecal/measurement/base/reader.h>
+#include <ecalhdf5/eh5_meas.h>
 
 #include "continuity_report.h"
 
 class MeasurementContainer
 {
 public:
-  MeasurementContainer(std::shared_ptr<eCAL::experimental::measurement::base::Reader> hdf5_meas, const std::string& meas_dir = "", bool use_receive_timestamp = true);
+  MeasurementContainer(std::shared_ptr<eCAL::eh5::v2::HDF5Meas> hdf5_meas, const std::string& meas_dir = "", bool use_receive_timestamp = true);
   ~MeasurementContainer();
 
   void CreatePublishers();
@@ -108,7 +108,7 @@ class MeasurementContainer
     PublisherInfo*                     publisher_info_;
   };
 
-  std::shared_ptr<eCAL::experimental::measurement::base::Reader>      hdf5_meas_;
+  std::shared_ptr<eCAL::eh5::v2::HDF5Meas>              hdf5_meas_;
   std::string                                           meas_dir_;
   bool                                                  use_receive_timestamp_;
 

app/play/play_core/src/play_thread.cpp

@@ -1,6 +1,6 @@
 /* ========================= eCAL LICENSE =================================
  *
- * Copyright (C) 2016 - 2019 Continental Corporation
+ * Copyright (C) 2016 - 2024 Continental Corporation
  *
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
@@ -576,7 +576,7 @@ void PlayThread::LogChannelMapping(const std::map<std::string, std::string>& cha
 //// Measurement                                                            ////
 ////////////////////////////////////////////////////////////////////////////////
 
-void PlayThread::SetMeasurement(const std::shared_ptr<eCAL::experimental::measurement::base::Reader>& measurement, const std::string& path)
+void PlayThread::SetMeasurement(const std::shared_ptr<eCAL::eh5::v2::HDF5Meas>& measurement, const std::string& path)
 {
   std::unique_ptr<MeasurementContainer> new_measurment_container;
 

app/play/play_core/src/play_thread.h

@@ -1,6 +1,6 @@
 /* ========================= eCAL LICENSE =================================
  *
- * Copyright (C) 2016 - 2019 Continental Corporation
+ * Copyright (C) 2016 - 2024 Continental Corporation
  *
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
@@ -79,7 +79,7 @@ class PlayThread : public InterruptibleThread
    * @param measurement    The new measurement
    * @param path           The (optional) path from where the measurement was loaded
    */
-  void SetMeasurement(const std::shared_ptr<eCAL::experimental::measurement::base::Reader>& measurement, const std::string& path = "");
+  void SetMeasurement(const std::shared_ptr<eCAL::eh5::v2::HDF5Meas>& measurement, const std::string& path = "");
 
   /**
    * @brief Returns whether a measurement has successfully been loaded

app/rec/rec_client_core/CMakeLists.txt

@@ -1,6 +1,6 @@
 # ========================= eCAL LICENSE =================================
 #
-# Copyright (C) 2016 - 2019 Continental Corporation
+# Copyright (C) 2016 - 2024 Continental Corporation
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
@@ -98,7 +98,7 @@ target_link_libraries(${PROJECT_NAME}
     eCAL::core_pb
     eCAL::app_pb
   PRIVATE
-    eCAL::measurement_hdf5
+    eCAL::hdf5
     ThreadingUtils
     Threads::Threads
     eCAL::ecal-utils

app/rec/rec_client_core/src/job/hdf5_writer_thread.cpp

@@ -1,6 +1,6 @@
 /* ========================= eCAL LICENSE =================================
  *
- * Copyright (C) 2016 - 2019 Continental Corporation
+ * Copyright (C) 2016 - 2024 Continental Corporation
  *
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
@@ -18,7 +18,6 @@
 */
 
 #include "hdf5_writer_thread.h"
-#include <ecal/measurement/hdf5/writer.h>
 
 #include "rec_client_core/ecal_rec_logger.h"
 
@@ -42,7 +41,7 @@ namespace eCAL
       , new_topic_info_map_available_(true)
       , flushing_                    (false)
     {
-      hdf5_writer_ = std::make_unique<eCAL::experimental::measurement::hdf5::Writer>();
+      hdf5_writer_ = std::make_unique<eCAL::eh5::v2::HDF5Meas>();
     }
 
     Hdf5WriterThread::~Hdf5WriterThread()
@@ -186,7 +185,7 @@ namespace eCAL
             frame->data_.size(),
             std::chrono::duration_cast<std::chrono::microseconds>(frame->ecal_publish_time_.time_since_epoch()).count(),
             std::chrono::duration_cast<std::chrono::microseconds>(frame->ecal_receive_time_.time_since_epoch()).count(),
-            frame->topic_name_,
+            frame->topic_name_, 
             frame->id_,
             frame->clock_
           ))
@@ -265,7 +264,7 @@ namespace eCAL
 #endif // NDEBUG
       std::unique_lock<decltype(hdf5_writer_mutex_)> hdf5_writer_lock(hdf5_writer_mutex_);
 
-      if (hdf5_writer_->Open(hdf5_dir))
+      if (hdf5_writer_->Open(hdf5_dir, eCAL::eh5::v2::eAccessType::CREATE))
       {
 #ifndef NDEBUG
         EcalRecLogger::Instance()->debug("Hdf5WriterThread::Open(): Successfully opened HDF5-Writer with path \"" + hdf5_dir + "\"");