Merge pull request #13 from hpuhr/devel

Devel
hpuhr · Nov 4, 2019 · 95f5f54 · 95f5f54
2 parents e7ab9e0 + 123dc0b
commit 95f5f54
Show file tree

Hide file tree

Showing 30 changed files with 1,111 additions and 136 deletions.
diff --git a/CMakeLists.txt b/CMakeLists.txt
@@ -9,7 +9,7 @@ set ( CMAKE_BUILD_TYPE Release )
 project( jASTERIX )
 SET(CPACK_PACKAGE_VERSION_MAJOR "0")
 SET(CPACK_PACKAGE_VERSION_MINOR "0")
-SET(CPACK_PACKAGE_VERSION_PATCH "1")
+SET(CPACK_PACKAGE_VERSION_PATCH "3")
 
 message("  System: ${CMAKE_SYSTEM}")
 set(CMAKE_ARCHIVE_OUTPUT_DIRECTORY ${CMAKE_BINARY_DIR}/lib)
@@ -31,6 +31,9 @@ message("USE_LOG4CPP: ${USE_LOG4CPP}")
 set (USE_BOOST true)
 message("USE_BOOST: ${USE_BOOST}")
 
+set (USE_OPENSSL true)
+message("USE_OPENSSL: ${USE_OPENSSL}")
+
 configure_file(include/jasterix/global.h.in include/jasterix/global.h)
 
 IF (USE_BOOST)
@@ -67,6 +70,16 @@ target_link_libraries ( jasterix
     )
 ENDIF()
 
+IF (USE_OPENSSL)
+find_package ( OpenSSL REQUIRED )
+message("  OPENSSL_INCLUDE_DIR: ${OPENSSL_INCLUDE_DIR}")
+message("  OPENSSL_CRYPTO_LIBRARY: ${OPENSSL_CRYPTO_LIBRARY}")
+
+target_link_libraries ( jasterix
+    ${OPENSSL_CRYPTO_LIBRARY}
+    )
+ENDIF()
+
 find_package(TBB REQUIRED)
 
 find_package ( LibArchive REQUIRED )
@@ -95,7 +108,7 @@ message("Installing using prefix: ${CMAKE_INSTALL_PREFIX}")
 #install(DIRECTORY "specifications" DESTINATION jasterix)
 
 set_target_properties(jasterix PROPERTIES PUBLIC_HEADER
-    "include/jasterix/jasterix.h;include/jasterix/category.h;include/jasterix/edition.h;include/jasterix/refedition.h;include/jasterix/ref.h;include/jasterix/mapping.h;include/jasterix/frameparser.h;include/jasterix/record.h;include/jasterix/itemparserbase.h;build/include/jasterix/global.h")
+    "include/jasterix/jasterix.h;include/jasterix/category.h;include/jasterix/edition.h;include/jasterix/refedition.h;include/jasterix/ref.h;include/jasterix/spfedition.h;include/jasterix/spf.h;include/jasterix/mapping.h;include/jasterix/frameparser.h;include/jasterix/record.h;include/jasterix/itemparserbase.h;build/include/jasterix/global.h")
 
 #install (TARGETS jasterix DESTINATION lib)
 install (TARGETS jasterix DESTINATION lib

diff --git a/README.md b/README.md
@@ -8,7 +8,7 @@ The library allows decoding of binary ASTERIX data into JSON. The ASTERIX defini
 
 Many ASTERIX decoder libraries exist today, but none of them was suitable/available for the ATSDB project. For this reason, the author decided to implement one, with a strong focus on flexibility first and performance second.
 
-- High flexibility using configuration only
+- High flexibility using configuration only, e.g. a user can add:
   - New framings
   - New categories 
   - New editions
@@ -23,22 +23,22 @@ Currently support framings:
 - IOSS: IOSS Final Format
 - RFF: Comsoft RFF format
 
-Currently supported ASTERIX categories & editions:
+Currently supported ASTERIX categories & editions, reserved expansion fields and special purpose fields:
 
-| CAT |       Editions | REFs|
-|-----|----------------|-----|
-|  001|             1.1|     |
-|  002|             1.0|     |
-|  019|             1.2|     |
-|  020|        1.5, 1.8|  1.3|
-|  021|             2.1|     |
-|  023|             1.2|     |
-|  034|            1.26|     |
-|  048|            1.15|     |
-|  062|      1.12, 1.16|     |
-|  063|             1.0|     |
-|  063|             1.1|     |
-|  065|        1.2, 1.3|     |
+| CAT |       Editions | REFs|       SPFs|
+|-----|----------------|-----|-----------|
+|  001|             1.1|     |           |
+|  002|             1.0|     |           |
+|  019|             1.2|     |           |
+|  020|        1.5, 1.8|  1.3|           |
+|  021|             2.1|     |           |
+|  023|             1.2|     |           |
+|  034|            1.26|     |           |
+|  048|            1.15|     |           |
+|  062|      1.12, 1.16|     | ARTAS TRIs|
+|  063|             1.0|     |           |
+|  063|             1.1|     |           |
+|  065|        1.2, 1.3|     |           |
 
 
 ## Contents
@@ -58,10 +58,23 @@ The following libraries are mandatory:
 The following libaries are optional (can be deactivated in CMakeList.txt):
 - Boost
 - Log4Cpp
+- OpenSSL
 
 Also, the Nlohmann::JSON library is used.
 
-## Installation / Building
+## Client Installation without Building
+
+Since v0.0.3 an client AppImage is supplied, which can be executed (without setup effort) under all recent Linux distributions (since Ubuntu 14.04).
+
+Download the AppImage and the jASTERIX definitions from the [Releases](https://github.com/hpuhr/jASTERIX/releases) page, and extract the definitions into a local folder, e.g. 'definitions'. Execute the following command to add the executable flag to the AppImage:
+
+```
+chmod +x jASTERIX_client_v0.0.3-x86_64.AppImage
+```
+
+After this, the jASTERIX client can be run from the console (see Usage section).
+
+## Installation with Building
 
 Download the source code from this page, then execute the following commands in the check-out folder:
 ```
@@ -79,31 +92,37 @@ sudo make install
 
 After building, the following jASTERIX client can be used (from the check-out folder):
 ```
-/build/bin/jasterix_client --help
+./jASTERIX_client_v0.0.3-x86_64.AppImage --help
 INFO    : Allowed options:
-  --help                 produce help message
-  --filename arg         input file name
-  --definition_path arg  path to jASTERIX definition files
-  --framing arg          input framine format, as specified in the framing 
-                         definitions. netto is default
-  --frame_limit arg      number of frames to process, default -1, use -1 to 
-                         disable.
-  --frame_chunk_size arg number of frames to process in one chunk, default 
-                         1000, use -1 to disable.
-  --data_write_size arg  number of frame chunks to write in one file write, 
-                         default 100, use -1 to disable.
-  --debug                print debug output
-  --print                print JSON output
-  --print_indent arg     intendation of json print, use -1 to disable.
-  --write_type arg       optional write type, e.g. text,zip. needs 
-                         write_filename.
-  --write_filename arg   optional write filename, e.g. test.zip.
+  --help                   produce help message
+  --filename arg           input file name
+  --definition_path arg    path to jASTERIX definition files
+  --framing arg            input framine format, as specified in the framing 
+                           definitions. raw/netto is default
+  --frame_limit arg        number of frames to process, default -1, use -1 to 
+                           disable.
+  --frame_chunk_size arg   number of frames to process in one chunk, default 
+                           1000, use -1 to disable.
+  --data_write_size arg    number of frame chunks to write in one file write, 
+                           default 100, use -1 to disable.
+  --debug                  print debug output
+  --debug_include_framing  print debug output including framing, debug still 
+                           has to be set, disable per default
+  --single_thread          process data in single thread
+  --add_artas_md5          add ARTAS MD5 hashes
+  --check_artas_md5        add and check ARTAS MD5 hashes
+  --add_record_data        add original record data in hex
+  --print                  print JSON output
+  --print_indent arg       intendation of json print, use -1 to disable.
+  --write_type arg         optional write type, e.g. text,zip. needs 
+                           write_filename.
+  --write_filename arg     optional write filename, e.g. test.zip.
 
 ```
 
 To decode and print an ASTERIX file as JSON text, use:
 ```
-./build/bin/jasterix_client --definition_path definitions/ --filename src/test/cat020ed1.5.bin --print
+./jASTERIX_client_v0.0.3-x86_64.AppImage --definition_path definitions/ --filename src/test/cat020ed1.5.bin --print
 INFO    : {
     "data_blocks": [
         {

diff --git a/definitions/categories/062/cat062_spf_artas.json b/definitions/categories/062/cat062_spf_artas.json
@@ -0,0 +1,25 @@
+{
+    "name":"cat062_spf_artas",
+    "type":"SimpleSpecialPurposeField",
+    "items":[
+        {
+            "name":"Target Report Identifiers",
+            "type":"repetitive",
+            "repetition_item":{
+                "name":"REP",
+                "type":"fixed_bytes",
+                "length":1,
+                "data_type":"uint"
+            },
+            "items":[
+                {
+                    "name":"TRI",
+                    "comment":"",
+                    "type":"fixed_bytes",
+                    "length":4,
+                    "data_type":"bin"
+                }
+            ]
+        }
+    ]
+}
diff --git a/definitions/categories/categories.json b/definitions/categories/categories.json
@@ -11,10 +11,6 @@
             }
         },
         "mappings":{
-            "atsdb":{
-                "comment":"conversion to atsdb cat048",
-                "file":"001/cat001_1.1_to_cat048_.1.15_mapping.json"
-            }
         }
     },
     "002":{
@@ -151,6 +147,7 @@
     "062":{
         "comment":"Category 062 SURVEILLANCE DATA EXCHANGE SDPS Track Messages",
         "default_edition":"1.16",
+        "default_spf_edition":"ARTAS",
         "default_mapping":"",
         "editions":{
             "1.12":{
@@ -164,6 +161,13 @@
                 "file":"062/cat062_1.16.json"
             }
         },
+        "spf_editions":{
+            "ARTAS":{
+                "document":"SDIS/SUR/ARTAS/ASTX.015",
+                "date":"06/01/2009",
+                "file":"062/cat062_spf_artas.json"
+            }
+        },
         "mappings":{
             "1.0":{
                     "comment":"conversion to general format 1.0",

diff --git a/include/jasterix/CMakeLists.txt b/include/jasterix/CMakeLists.txt
@@ -13,6 +13,8 @@ target_sources(jasterix
         "${CMAKE_CURRENT_LIST_DIR}/record.h"
         "${CMAKE_CURRENT_LIST_DIR}/refedition.h"
         "${CMAKE_CURRENT_LIST_DIR}/ref.h"
+        "${CMAKE_CURRENT_LIST_DIR}/spfedition.h"
+        "${CMAKE_CURRENT_LIST_DIR}/spf.h"
         "${CMAKE_CURRENT_LIST_DIR}/itemparserbase.h"
 )
 

diff --git a/include/jasterix/category.h b/include/jasterix/category.h
@@ -22,6 +22,7 @@
 #include "json.hpp"
 #include <jasterix/edition.h>
 #include <jasterix/refedition.h>
+#include <jasterix/spfedition.h>
 #include <jasterix/mapping.h>
 
 #include <string>
@@ -57,6 +58,16 @@ class Category
     bool hasCurrentREFEdition();
     std::shared_ptr<REFEdition> getCurrentREFEdition ();
 
+    // spf stuff
+    bool hasSPFEdition (const std::string& edition_str) const;
+    std::shared_ptr<SPFEdition> spfEdition (const std::string& edition_str);
+    std::string spfEditionPath (const std::string& edition_str) const;
+
+    std::string defaultSPFEdition () const;
+    void setCurrentSPFEdition (const std::string& edition_str);
+    bool hasCurrentSPFEdition();
+    std::shared_ptr<SPFEdition> getCurrentSPFEdition ();
+
     // mapping stuff
     bool hasMapping (const std::string& mapping_str);
     std::shared_ptr<Mapping> mapping (const std::string& mapping_str);
@@ -68,6 +79,7 @@ class Category
 
     const std::map<std::string, std::shared_ptr<Edition>>& editions() const;
     const std::map<std::string, std::shared_ptr<REFEdition>>& refEditions() const;
+    const std::map<std::string, std::shared_ptr<SPFEdition>>& spfEditions() const;
     const std::map<std::string, std::shared_ptr<Mapping>>& mappings() const;
 
     bool decode() const;
@@ -79,16 +91,19 @@ class Category
 
     std::string default_edition_;
     std::string default_ref_edition_;
+    std::string default_spf_edition_;
     std::string default_mapping_;
 
     std::string current_edition_;
     std::string current_ref_edition_;
+    std::string current_spf_edition_;
     std::string current_mapping_;
 
     bool decode_{true};
 
     std::map<std::string, std::shared_ptr<Edition>> editions_;
     std::map<std::string, std::shared_ptr<REFEdition>> ref_editions_;
+    std::map<std::string, std::shared_ptr<SPFEdition>> spf_editions_;
     std::map<std::string, std::shared_ptr<Mapping>> mappings_;
 };
 

diff --git a/include/jasterix/global.h.in b/include/jasterix/global.h.in
@@ -22,6 +22,7 @@
 
 #define USE_LOG4CPP @USE_LOG4CPP@
 #define USE_BOOST @USE_BOOST@
+#define USE_OPENSSL @USE_OPENSSL@
 static const std::string PACKAGE_VERSION = "v@CPACK_PACKAGE_VERSION_MAJOR@.@CPACK_PACKAGE_VERSION_MINOR@.@CPACK_PACKAGE_VERSION_PATCH@";
 
 #endif
diff --git a/include/jasterix/itemparserbase.h b/include/jasterix/itemparserbase.h
@@ -74,7 +74,7 @@ inline std::vector<std::string> split(const std::string &s, char delim)
     {
         elems.push_back(item);
     }
-    return std::move(elems);
+    return elems;
 }
 
 }

diff --git a/include/jasterix/jasterix.h b/include/jasterix/jasterix.h
@@ -43,6 +43,14 @@ extern int frame_chunk_size;
 extern int record_chunk_size;
 extern int data_write_size;
 
+extern bool single_thread;
+
+#if USE_OPENSSL
+extern bool add_artas_md5_hash;
+#endif
+
+extern bool add_record_data;
+
 class jASTERIX
 {
 public:

diff --git a/include/jasterix/record.h b/include/jasterix/record.h
@@ -21,6 +21,7 @@
 
 #include <jasterix/itemparserbase.h>
 #include <jasterix/ref.h>
+#include <jasterix/spf.h>
 
 #include <vector>
 #include <memory>
@@ -42,6 +43,9 @@ class Record : public ItemParserBase
     std::shared_ptr<ReservedExpansionField> ref() const;
     void setRef(const std::shared_ptr<ReservedExpansionField> &ref);
 
+    std::shared_ptr<SpecialPurposeField> spf() const;
+    void setSpf(const std::shared_ptr<SpecialPurposeField> &spf);
+
 protected:
     std::unique_ptr<ItemParserBase> field_specification_;
     std::vector<std::string> uap_names_;
@@ -54,6 +58,7 @@ class Record : public ItemParserBase
 
     bool decode_ref_ {false};
     std::shared_ptr<ReservedExpansionField> ref_;
+    std::shared_ptr<SpecialPurposeField> spf_;
 
     inline bool compareKey (const nlohmann::json& container, const std::string& value)
     {

diff --git a/include/jasterix/ref.h b/include/jasterix/ref.h
@@ -40,7 +40,6 @@ class ReservedExpansionField : public ItemParserBase
     std::vector<std::string> items_names_;
     std::map<std::string, std::unique_ptr<ItemParserBase>> items_;
 
-    //bool compareKey (const nlohmann::json& container, const std::string& value);
 };
 
 }

diff --git a/include/jasterix/spf.h b/include/jasterix/spf.h
@@ -0,0 +1,37 @@
+#ifndef SPECIALPURPOSEFIELD_H
+#define SPECIALPURPOSEFIELD_H
+
+#include <jasterix/itemparserbase.h>
+
+#include <vector>
+#include <memory>
+
+namespace jASTERIX
+{
+// decodes a field specification/availablity field (ending with extend bit), and list of items
+class SpecialPurposeField : public ItemParserBase
+{
+public:
+    SpecialPurposeField (const nlohmann::json& item_definition);
+    virtual ~SpecialPurposeField() override;
+
+    virtual size_t parseItem (const char* data, size_t index, size_t size, size_t current_parsed_bytes,
+                              nlohmann::json& target, bool debug) override;
+protected:
+    std::unique_ptr<ItemParserBase> complex_field_specification_;
+    std::vector<std::string> complex_items_names_;
+    std::map<std::string, std::unique_ptr<ItemParserBase>> complex_items_;
+
+    std::vector<std::unique_ptr<ItemParserBase>> simple_items_;
+
+    size_t parseSimpleItem (const char* data, size_t index, size_t size, size_t current_parsed_bytes,
+                            nlohmann::json& target, bool debug);
+
+    size_t parseComplexItem (const char* data, size_t index, size_t size, size_t current_parsed_bytes,
+                            nlohmann::json& target, bool debug);
+
+};
+
+}
+
+#endif // SPECIALPURPOSEFIELD_H