Merge pull request #12 from hpuhr/devel

Devel

CMakeLists.txt

@@ -21,7 +21,7 @@ add_definitions ( -Wall -std=c++11 -Wno-int-in-bool-context)
 
 set(CMAKE_INCLUDE_CURRENT_DIR ON)
 # Instruct CMake to run moc automatically when needed.
-set(CMAKE_AUTOMOC ON)
+#set(CMAKE_AUTOMOC ON)
 
 add_library(jasterix "")
 
@@ -95,7 +95,7 @@ message("Installing using prefix: ${CMAKE_INSTALL_PREFIX}")
 #install(DIRECTORY "specifications" DESTINATION jasterix)
 
 set_target_properties(jasterix PROPERTIES PUBLIC_HEADER
-    "include/jasterix/jasterix.h;include/jasterix/category.h;include/jasterix/edition.h;include/jasterix/mapping.h;include/jasterix/frameparser.h;include/jasterix/record.h;include/jasterix/itemparserbase.h;build/include/jasterix/global.h")
+    "include/jasterix/jasterix.h;include/jasterix/category.h;include/jasterix/edition.h;include/jasterix/refedition.h;include/jasterix/ref.h;include/jasterix/mapping.h;include/jasterix/frameparser.h;include/jasterix/record.h;include/jasterix/itemparserbase.h;build/include/jasterix/global.h")
 
 #install (TARGETS jasterix DESTINATION lib)
 install (TARGETS jasterix DESTINATION lib

README.md

@@ -30,7 +30,7 @@ Currently supported ASTERIX categories & editions:
 |  001|             1.1|     |
 |  002|             1.0|     |
 |  019|             1.2|     |
-|  020|        1.5, 1.8|     |
+|  020|        1.5, 1.8|  1.3|
 |  021|             2.1|     |
 |  023|             1.2|     |
 |  034|            1.26|     |

definitions/categories/020/cat020_1.5.json

@@ -1008,7 +1008,7 @@
                                     "comment":"Correlation coefficient in two’s complement",
                                     "type":"fixed_bytes",
                                     "length":2,
-                                    "data_type":"uint",
+                                    "data_type":"int",
                                     "lsb":0.25
                                 }
                             ]

definitions/categories/020/cat020_1.8.json

@@ -1008,7 +1008,7 @@
                                     "comment":"Correlation coefficient in two’s complement",
                                     "type":"fixed_bytes",
                                     "length":2,
-                                    "data_type":"uint",
+                                    "data_type":"int",
                                     "lsb":0.25
                                 }
                             ]

definitions/categories/categories.json

@@ -54,6 +54,7 @@
     "020":{
         "comment":"Category 020 Multilateration Target Reports",
         "default_edition":"1.8",
+        "default_ref_edition":"1.3",
         "default_mapping":"",
         "editions":{
             "1.5":{
@@ -68,6 +69,13 @@
                 "comment":"Same as 1.5"
             }
         },
+        "ref_editions":{
+            "1.3":{
+                "document":"EUROCONTROL-SPEC-0149-14A",
+                "date":"24/10/2016",
+                "file":"020/cat020_ref_1.3.json"
+            }
+        },
         "mappings":{
             "1.0":{
                 "comment":"conversion to general format 1.0",

include/jasterix/CMakeLists.txt

@@ -11,6 +11,8 @@ target_sources(jasterix
         "${CMAKE_CURRENT_LIST_DIR}/category.h"
         "${CMAKE_CURRENT_LIST_DIR}/mapping.h"
         "${CMAKE_CURRENT_LIST_DIR}/record.h"
+        "${CMAKE_CURRENT_LIST_DIR}/refedition.h"
+        "${CMAKE_CURRENT_LIST_DIR}/ref.h"
         "${CMAKE_CURRENT_LIST_DIR}/itemparserbase.h"
 )
 

include/jasterix/category.h

@@ -21,6 +21,7 @@
 
 #include "json.hpp"
 #include <jasterix/edition.h>
+#include <jasterix/refedition.h>
 #include <jasterix/mapping.h>
 
 #include <string>
@@ -32,11 +33,12 @@ class Category
 {
 public:
     Category(const std::string& number, const nlohmann::json& definition, const std::string& definition_path);
-    virtual ~Category() {}
+    virtual ~Category();
 
     std::string number() const;
     std::string comment() const;
 
+    // edition stuff
     bool hasEdition (const std::string& edition_str) const;
     std::shared_ptr<Edition> edition (const std::string& edition_str);
     std::string editionPath (const std::string& edition_str) const;
@@ -45,15 +47,27 @@ class Category
     void setCurrentEdition (const std::string& edition_str);
     std::shared_ptr<Edition> getCurrentEdition();
 
+    // ref stuff
+    bool hasREFEdition (const std::string& edition_str) const;
+    std::shared_ptr<REFEdition> refEdition (const std::string& edition_str);
+    std::string refEditionPath (const std::string& edition_str) const;
+
+    std::string defaultREFEdition () const;
+    void setCurrentREFEdition (const std::string& edition_str);
+    bool hasCurrentREFEdition();
+    std::shared_ptr<REFEdition> getCurrentREFEdition ();
+
+    // mapping stuff
     bool hasMapping (const std::string& mapping_str);
     std::shared_ptr<Mapping> mapping (const std::string& mapping_str);
 
-
     std::string defaultMapping() const;
     bool hasCurrentMapping();
+    void setCurrentMapping (const std::string& mapping_str);
     std::shared_ptr<Mapping> getCurrentMapping();
 
     const std::map<std::string, std::shared_ptr<Edition>>& editions() const;
+    const std::map<std::string, std::shared_ptr<REFEdition>>& refEditions() const;
     const std::map<std::string, std::shared_ptr<Mapping>>& mappings() const;
 
     bool decode() const;
@@ -62,12 +76,19 @@ class Category
 protected:
     std::string number_;
     std::string comment_;
+
     std::string default_edition_;
-    std::string current_edition_;
+    std::string default_ref_edition_;
     std::string default_mapping_;
+
+    std::string current_edition_;
+    std::string current_ref_edition_;
+    std::string current_mapping_;
+
     bool decode_{true};
 
     std::map<std::string, std::shared_ptr<Edition>> editions_;
+    std::map<std::string, std::shared_ptr<REFEdition>> ref_editions_;
     std::map<std::string, std::shared_ptr<Mapping>> mappings_;
 };
 

include/jasterix/edition.h

@@ -33,7 +33,7 @@ class Edition
 {
 public:
     Edition(const std::string& number, const nlohmann::json& definition, const std::string& definition_path);
-    virtual ~Edition() {}
+    virtual ~Edition();
 
     std::string number() const;
     std::string document() const;

include/jasterix/frameparser.h

@@ -35,11 +35,12 @@ class FrameParser
     // return number of parsed bytes
     size_t parseHeader (const char* data, size_t index, size_t size, nlohmann::json& target, bool debug);
 
-    std::tuple<size_t, size_t, bool> findFrames (const char* data, size_t index, size_t size, nlohmann::json& target,
-                                                  bool debug);
     // parsed bytes, num frames, done flag
+    std::tuple<size_t, size_t, bool> findFrames (const char* data, size_t index, size_t size, nlohmann::json* target,
+                                                  bool debug);
 
-    size_t decodeFrames (const char* data, nlohmann::json& target, bool debug);
+    // num records, num errors
+    std::pair<size_t, size_t> decodeFrames (const char* data, nlohmann::json* target, bool debug);
 
     bool hasFileHeaderItems() const;
 
@@ -52,8 +53,8 @@ class FrameParser
 
     size_t sum_frames_cnt_ {0};
 
-    // returns number of records
-    size_t decodeFrame (const char* data, nlohmann::json& json_frame, bool debug);
+    // returns number of records, num errors
+    std::pair<size_t, size_t> decodeFrame (const char* data, nlohmann::json& json_frame, bool debug);
 };
 
 }

include/jasterix/itemparserbase.h

@@ -53,7 +53,7 @@ class ItemParserBase
     std::string type_;
 };
 
-bool variableHasValue (const nlohmann::json& data, const std::string& variable_name,
+bool variableHasValue (const nlohmann::json& data, const std::vector <std::string>& variable_name_parts,
                        const nlohmann::json& variable_value);
 
 inline unsigned char reverseBits(unsigned char b)

include/jasterix/jasterix.h

@@ -43,49 +43,45 @@ extern int frame_chunk_size;
 extern int record_chunk_size;
 extern int data_write_size;
 
-//class Category;
-
 class jASTERIX
 {
 public:
-    jASTERIX( const std::string& definition_path, bool print, bool debug, bool debug_exclude_framing);
+    jASTERIX(const std::string& definition_path, bool print, bool debug, bool debug_exclude_framing);
     virtual ~jASTERIX();
 
-    bool hasCategory(const std::string& cat_str);
-    bool decodeCategory(const std::string& cat_str);
-    void setDecodeCategory (const std::string& cat_str, bool decode);
+    bool hasCategory(unsigned int cat);
+    bool decodeCategory(unsigned int cat);
+    void setDecodeCategory (unsigned int cat, bool decode);
     void decodeNoCategories();
 
-    bool hasEdition (const std::string& cat_str, const std::string& edition_str);
-    void setEdition (const std::string& cat_str, const std::string& edition_str);
-
-    bool hasMapping (const std::string& cat_str, const std::string& mapping_str);
-    void setMapping (const std::string& cat_str, const std::string& mapping_str);
+    std::shared_ptr<Category> category (unsigned int cat);
+    const std::map<unsigned int, std::shared_ptr<Category>>& categories() { return category_definitions_; }
 
     void decodeFile (const std::string& filename, const std::string& framing_str,
-                     std::function<void(nlohmann::json&, size_t, size_t)> data_callback=nullptr);
-    // callback gets moved chunk, accumulated number of frames, number of records
+                     std::function<void(std::unique_ptr<nlohmann::json>, size_t, size_t, size_t)> data_callback=nullptr);
+    // callback gets moved chunk, accumulated number of frames, number of records, number of errors
     void decodeFile (const std::string& filename,
-                     std::function<void(nlohmann::json&, size_t, size_t)> data_callback=nullptr);
+                     std::function<void(std::unique_ptr<nlohmann::json>, size_t, size_t, size_t)> data_callback=nullptr);
 
     void decodeASTERIX (const char* data, size_t size,
-                 std::function<void(nlohmann::json&, size_t, size_t)> data_callback=nullptr);
+                 std::function<void(std::unique_ptr<nlohmann::json>, size_t, size_t, size_t)> data_callback=nullptr);
 
     size_t numFrames() const;
     size_t numRecords() const;
+    size_t numErrors() const;
 
-    void addDataBlockChunk (nlohmann::json& data_block_chunk, bool done);
-    void addDataChunk (nlohmann::json& data_chunk, bool done);
+    void addDataBlockChunk (std::unique_ptr<nlohmann::json> data_block_chunk, bool error, bool done);
+    void addDataChunk (std::unique_ptr<nlohmann::json> data_chunk, bool done);
 
     const std::vector<std::string>& framings() { return framings_; }
-    const std::map<std::string, Category>& categories() { return category_definitions_; }
 
     const std::string& dataBlockDefinitionPath() const;
     const std::string& categoriesDefinitionPath() const;
     const std::string& framingsFolderPath() const;
 
     void setDebug(bool debug);
 
+
 private:
     //tbb::task_scheduler_init init_;
 
@@ -103,26 +99,23 @@ class jASTERIX
     std::string categories_definition_path_;
     nlohmann::json categories_definition_;
 
-    std::map<std::string, Category> category_definitions_;
-    std::map<unsigned int, std::shared_ptr<Edition>> current_category_editions_; // cat -> edition
-    std::map<unsigned int, std::shared_ptr<Mapping>> current_category_mappings_; // cat -> edition
+    std::map<unsigned int, std::shared_ptr<Category>> category_definitions_;
 
 #if USE_BOOST
     boost::iostreams::mapped_file_source file_;
 #else
     char* file_buffer_{nullptr};
 #endif
 
-    tbb::concurrent_queue<nlohmann::json> data_block_chunks_;
+    tbb::concurrent_queue<std::unique_ptr<nlohmann::json>> data_block_chunks_;
     bool data_block_processing_done_ {false};
 
-    tbb::concurrent_queue<nlohmann::json> data_chunks_;
+    tbb::concurrent_queue<std::unique_ptr<nlohmann::json>> data_chunks_;
     bool data_processing_done_ {false};
 
     size_t num_frames_{0};
     size_t num_records_{0};
-
-    void updateCurrentEditionsAndMappings ();
+    size_t num_errors_{0};
 };
 }
 

include/jasterix/record.h

@@ -20,6 +20,7 @@
 #define RECORD_H
 
 #include <jasterix/itemparserbase.h>
+#include <jasterix/ref.h>
 
 #include <vector>
 #include <memory>
@@ -31,10 +32,16 @@ class Record : public ItemParserBase
 {
 public:
     Record (const nlohmann::json& item_definition);
-    virtual ~Record() {}
+    virtual ~Record() override {}
 
     virtual size_t parseItem (const char* data, size_t index, size_t size, size_t current_parsed_bytes,
                               nlohmann::json& target, bool debug) override;
+    bool decodeREF() const;
+    void decodeREF(bool decodeREF);
+
+    std::shared_ptr<ReservedExpansionField> ref() const;
+    void setRef(const std::shared_ptr<ReservedExpansionField> &ref);
+
 protected:
     std::unique_ptr<ItemParserBase> field_specification_;
     std::vector<std::string> uap_names_;
@@ -45,7 +52,24 @@ class Record : public ItemParserBase
     std::map<std::string, std::vector<std::string>> conditional_uap_names_;
     std::map<std::string, std::unique_ptr<ItemParserBase>> items_;
 
-    bool compareKey (const nlohmann::json& container, const std::string& value);
+    bool decode_ref_ {false};
+    std::shared_ptr<ReservedExpansionField> ref_;
+
+    inline bool compareKey (const nlohmann::json& container, const std::string& value)
+    {
+        const nlohmann::json* val_ptr = &container;
+
+        for (const std::string& sub_key : conditional_uaps_sub_keys_)
+        {
+            //loginf << "UGA '" << sub_key << "' json '" << val_ptr->dump(4) << "'"<< logendl;
+            val_ptr = &(*val_ptr)[sub_key];
+        }
+
+        if (val_ptr->type() == nlohmann::json::value_t::string) // from string conv
+            return val_ptr->get<std::string>() == value;
+        else
+            return val_ptr->dump() == value;
+    }
 };
 
 }

include/jasterix/ref.h

@@ -0,0 +1,47 @@
+/*
+ * This file is part of ATSDB.
+ *
+ * ATSDB is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * ATSDB is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+
+ * You should have received a copy of the GNU General Public License
+ * along with ATSDB.  If not, see <http://www.gnu.org/licenses/>.
+ */
+
+
+#ifndef RESERVEDEXPSIONFIELD_H
+#define RESERVEDEXPSIONFIELD_H
+
+#include <jasterix/itemparserbase.h>
+
+#include <vector>
+#include <memory>
+
+namespace jASTERIX
+{
+// decodes a field specification/availablity field (ending with extend bit), and list of items
+class ReservedExpansionField : public ItemParserBase
+{
+public:
+    ReservedExpansionField (const nlohmann::json& item_definition);
+    virtual ~ReservedExpansionField() override;
+
+    virtual size_t parseItem (const char* data, size_t index, size_t size, size_t current_parsed_bytes,
+                              nlohmann::json& target, bool debug) override;
+protected:
+    std::unique_ptr<ItemParserBase> field_specification_;
+    std::vector<std::string> items_names_;
+    std::map<std::string, std::unique_ptr<ItemParserBase>> items_;
+
+    //bool compareKey (const nlohmann::json& container, const std::string& value);
+};
+
+}
+#endif // RESERVEDEXPSIONFIELD_H