Merge pull request #149 from george0st/change

Use parquets with compression
george0st · Apr 16, 2024 · 1ae4029 · 1ae4029
2 parents 62b1521 + 5a024b8
commit 1ae4029
Show file tree

Hide file tree

Showing 32 changed files with 572 additions and 581 deletions.
diff --git a/02-data/01-size-100/01-basic-party.csv.gz b/02-data/01-size-100/01-basic-party.csv.gz
diff --git a/02-data/01-size-100/01-basic-party.parquet b/02-data/01-size-100/01-basic-party.parquet
diff --git a/02-data/01-size-100/02-basic-contact.csv.gz b/02-data/01-size-100/02-basic-contact.csv.gz
diff --git a/02-data/01-size-100/02-basic-contact.parquet b/02-data/01-size-100/02-basic-contact.parquet
diff --git a/02-data/01-size-100/03-basic-relation.csv.gz b/02-data/01-size-100/03-basic-relation.csv.gz
diff --git a/02-data/01-size-100/03-basic-relation.parquet b/02-data/01-size-100/03-basic-relation.parquet
diff --git a/02-data/01-size-100/04-basic-account.csv.gz b/02-data/01-size-100/04-basic-account.csv.gz
diff --git a/02-data/01-size-100/04-basic-account.parquet b/02-data/01-size-100/04-basic-account.parquet
diff --git a/02-data/01-size-100/05-basic-transaction.csv.gz b/02-data/01-size-100/05-basic-transaction.csv.gz
diff --git a/02-data/01-size-100/05-basic-transaction.parquet b/02-data/01-size-100/05-basic-transaction.parquet
diff --git a/02-data/01-size-100/06-basic-event.csv.gz b/02-data/01-size-100/06-basic-event.csv.gz
diff --git a/02-data/01-size-100/06-basic-event.parquet b/02-data/01-size-100/06-basic-event.parquet
diff --git a/02-data/01-size-100/07-basic-communication.csv.gz b/02-data/01-size-100/07-basic-communication.csv.gz
diff --git a/02-data/01-size-100/07-basic-communication.parquet b/02-data/01-size-100/07-basic-communication.parquet
diff --git a/02-data/02-size-1K/01-basic-party.csv.gz b/02-data/02-size-1K/01-basic-party.csv.gz
diff --git a/02-data/02-size-1K/01-basic-party.parquet b/02-data/02-size-1K/01-basic-party.parquet
diff --git a/02-data/02-size-1K/02-basic-contact.csv.gz b/02-data/02-size-1K/02-basic-contact.csv.gz
diff --git a/02-data/02-size-1K/02-basic-contact.parquet b/02-data/02-size-1K/02-basic-contact.parquet
diff --git a/02-data/02-size-1K/03-basic-relation.csv.gz b/02-data/02-size-1K/03-basic-relation.csv.gz
diff --git a/02-data/02-size-1K/03-basic-relation.parquet b/02-data/02-size-1K/03-basic-relation.parquet
diff --git a/02-data/02-size-1K/04-basic-account.csv.gz b/02-data/02-size-1K/04-basic-account.csv.gz
diff --git a/02-data/02-size-1K/04-basic-account.parquet b/02-data/02-size-1K/04-basic-account.parquet
diff --git a/02-data/02-size-1K/05-basic-transaction.csv.gz b/02-data/02-size-1K/05-basic-transaction.csv.gz
diff --git a/02-data/02-size-1K/05-basic-transaction.parquet b/02-data/02-size-1K/05-basic-transaction.parquet
diff --git a/02-data/02-size-1K/06-basic-event.csv.gz b/02-data/02-size-1K/06-basic-event.csv.gz
diff --git a/02-data/02-size-1K/06-basic-event.parquet b/02-data/02-size-1K/06-basic-event.parquet
diff --git a/02-data/02-size-1K/07-basic-communication.csv.gz b/02-data/02-size-1K/07-basic-communication.csv.gz
diff --git a/02-data/02-size-1K/07-basic-communication.parquet b/02-data/02-size-1K/07-basic-communication.parquet
diff --git a/03-test/01-size-100.json b/03-test/01-size-100.json
diff --git a/03-test/02-size-1k.json b/03-test/02-size-1k.json
diff --git a/docs/todo_list.md b/docs/todo_list.md
@@ -2,5 +2,4 @@
 
 The list of expected/future improvements:
 
-1. Change 'target' in project to 'adapter' or 'replace'
-.  - Motivation: Ability to use for Target or Source also 
+1. Add ability to select type of output format (CSV or parquet) in commandline)
diff --git a/generator/base_data.py b/generator/base_data.py
@@ -71,7 +71,7 @@ def save(self, path, append: bool, dir: str, compress: bool):
         # write parquet
         table = pa.Table.from_pandas(df)
         if self._parquet_writer is None:
-            self._parquet_writer = pq.ParquetWriter(os.path.join(path, f"{self.name}.parquet"), table.schema)
+            self._parquet_writer = pq.ParquetWriter(os.path.join(path, f"{self.name}.parquet"), table.schema, compression=compression_opts)
         self._parquet_writer.write_table(table=table)
 
         # free memory