Merge pull request #150 from george0st/change

Remove null values in parquet

01-model/model.json

@@ -3,7 +3,7 @@
   "description": "The machine learning meta-model with synthetic data (useful for MLOps/feature store), part of the quality gate concept.",
   "kind": "model",
   "spec": {
-    "version": "0.2.2",
+    "version": "0.2.3",
     "CSV_SEPARATOR": ",",
     "CSV_DECIMAL": "."
   }

dev-requirements.txt

@@ -3,8 +3,7 @@ click~=8.1
 
 numpy~=1.26
 pandas==2.2.2
-#pyarrow==15.0.2
-#fastparquet==2024.2.0
+pyarrow==15.0.2
 
 # generation of synthetic data
 faker~=24.2

generator/basic_contact.py

@@ -49,16 +49,20 @@ def generate(self, count):
                 # generate different amount of contact information
                 # Customer = email + phone
                 # !Customer = random email or phone
-                contact_detail=ContactEnum.Full if party['party-type']=="Customer" else ContactEnum.Email\
+                contact_detail=ContactEnum.Full if party['party-type']=="Customer" else ContactEnum.Email \
                     if self.rnd_bool() else ContactEnum.Phone
 
+                # "name": "contact-email"
                 if contact_detail & ContactEnum.Email:
-                    # "name": "contact-email"
-                    model['contact-email']=self.fake.email()
+                    model['contact-email'] = self.fake.email()
+                else:
+                    model['contact-email'] = ""
 
+                # "name": "contact-phone"
                 if contact_detail & ContactEnum.Phone:
-                    # "name": "contact-phone"
-                    model['contact-phone']=self.fake.phone_number()
+                    model['contact-phone'] = self.fake.phone_number()
+                else:
+                    model['contact-phone'] = ""
 
                 # "name": "contact-state"
                 if count==0:
@@ -72,4 +76,3 @@ def generate(self, count):
                 model['record-date']=self.gmodel["NOW"]
 
                 self.model.append(model)
-

generator/basic_transaction.py

@@ -100,7 +100,8 @@ def generate(self, count):
 
                 # "name": "counterparty-other",
                 # "description": "Transaction counterparty other information",
-                # TODO: Add
+                # TODO: Add relevant value
+                model["counterparty-other"]=""
 
                 fraud=False
                 fraud_anomaly=0
@@ -118,7 +119,7 @@ def generate(self, count):
                 # "description": "Identification of fraud (True - fraud, False - without fraud)",
                 model["transaction-fraud"] = int(fraud)
 
-            # "name": "record-date",
+                # "name": "record-date",
                 # "description": "The date when the record was created",
                 model['record-date']=self.gmodel["NOW"]
 

generator/version.py

@@ -1,3 +1,3 @@
 # Store the version here so:
 
-__version__ = '0.2.0'
+__version__ = '0.2.1'

requirements.txt

@@ -3,8 +3,7 @@ click~=8.1
 
 numpy~=1.26
 pandas==2.2.2
-#pyarrow==15.0.2
-#fastparquet==2024.2.0
+pyarrow==15.0.2
 
 # generation of synthetic data
 faker~=24.2