From b3de849c92319689cdf3f4ea44c7e76a537cbd48 Mon Sep 17 00:00:00 2001 From: lrcouto Date: Wed, 28 Feb 2024 10:21:39 -0300 Subject: [PATCH 01/11] Update kedro-datasets optional requirements naming Signed-off-by: lrcouto --- .../{{ cookiecutter.repo_name }}/requirements.txt | 2 +- databricks-iris/{{ cookiecutter.repo_name }}/requirements.txt | 2 +- .../{{ cookiecutter.repo_name }}/requirements.txt | 2 +- .../{{ cookiecutter.repo_name }}/requirements.txt | 2 +- .../{{ cookiecutter.repo_name }}/requirements.txt | 2 +- .../{{ cookiecutter.repo_name }}/requirements.txt | 2 +- 6 files changed, 6 insertions(+), 6 deletions(-) diff --git a/astro-airflow-iris/{{ cookiecutter.repo_name }}/requirements.txt b/astro-airflow-iris/{{ cookiecutter.repo_name }}/requirements.txt index f1aa0979..a5913bee 100644 --- a/astro-airflow-iris/{{ cookiecutter.repo_name }}/requirements.txt +++ b/astro-airflow-iris/{{ cookiecutter.repo_name }}/requirements.txt @@ -2,7 +2,7 @@ ipython>=8.10 jupyterlab>=3.0 notebook kedro~={{ cookiecutter.kedro_version }} -kedro-datasets[pandas.CSVDataset]>=1.0 +kedro-datasets[pandas-csvdataset]>=1.0 kedro-airflow~=0.5 kedro-telemetry>=0.3.1 pytest-cov~=3.0 diff --git a/databricks-iris/{{ cookiecutter.repo_name }}/requirements.txt b/databricks-iris/{{ cookiecutter.repo_name }}/requirements.txt index a3d3d113..bd90bae1 100644 --- a/databricks-iris/{{ cookiecutter.repo_name }}/requirements.txt +++ b/databricks-iris/{{ cookiecutter.repo_name }}/requirements.txt @@ -2,7 +2,7 @@ ipython>=8.10 jupyterlab>=3.0 notebook kedro~={{ cookiecutter.kedro_version }} -kedro-datasets[spark.SparkDataset, pandas.ParquetDataset]>=1.0 +kedro-datasets[spark-sparkdataset, pandas-parquetdataset]>=1.0 kedro-telemetry>=0.3.1 numpy~=1.21 pytest-cov~=3.0 diff --git a/spaceflights-pandas-viz/{{ cookiecutter.repo_name }}/requirements.txt b/spaceflights-pandas-viz/{{ cookiecutter.repo_name }}/requirements.txt index 0aaa6250..e032f713 100644 --- a/spaceflights-pandas-viz/{{ cookiecutter.repo_name }}/requirements.txt +++ b/spaceflights-pandas-viz/{{ cookiecutter.repo_name }}/requirements.txt @@ -2,7 +2,7 @@ ipython>=8.10 jupyterlab>=3.0 notebook kedro~={{ cookiecutter.kedro_version }} -kedro-datasets[pandas.CSVDataset, pandas.ExcelDataset, pandas.ParquetDataset, plotly.PlotlyDataset, plotly.JSONDataset, matplotlib.MatplotlibWriter]>=1.0 +kedro-datasets[pandas-csvdataset, pandas-exceldataset, pandas-parquetdataset, plotly-plotlydataset, plotly-jsondataset, matplotlib-matplotlibwriter]>=1.0 kedro-telemetry>=0.3.1 kedro-viz>=6.7.0 pytest-cov~=3.0 diff --git a/spaceflights-pandas/{{ cookiecutter.repo_name }}/requirements.txt b/spaceflights-pandas/{{ cookiecutter.repo_name }}/requirements.txt index c41f20b4..fee12453 100644 --- a/spaceflights-pandas/{{ cookiecutter.repo_name }}/requirements.txt +++ b/spaceflights-pandas/{{ cookiecutter.repo_name }}/requirements.txt @@ -2,7 +2,7 @@ ipython>=8.10 jupyterlab>=3.0 notebook kedro~={{ cookiecutter.kedro_version }} -kedro-datasets[pandas.CSVDataset, pandas.ExcelDataset, pandas.ParquetDataset]>=1.0 +kedro-datasets[pandas-csvdataset, pandas-exceldataset, pandas-parquetdataset]>=1.0 kedro-telemetry>=0.3.1 kedro-viz>=6.7.0 pytest-cov~=3.0 diff --git a/spaceflights-pyspark-viz/{{ cookiecutter.repo_name }}/requirements.txt b/spaceflights-pyspark-viz/{{ cookiecutter.repo_name }}/requirements.txt index 48b5f058..8fa4cf08 100644 --- a/spaceflights-pyspark-viz/{{ cookiecutter.repo_name }}/requirements.txt +++ b/spaceflights-pyspark-viz/{{ cookiecutter.repo_name }}/requirements.txt @@ -2,7 +2,7 @@ ipython>=8.10 jupyterlab>=3.0 notebook kedro~={{ cookiecutter.kedro_version }} -kedro-datasets[pandas.CSVDataset, pandas.ExcelDataset, pandas.ParquetDataset, spark.SparkDataset, plotly.PlotlyDataset, plotly.JSONDataset, matplotlib.MatplotlibWriter]>=1.0 +kedro-datasets[pandas-csvdataset, pandas-exceldataset, pandas-parquetdataset, spark-sparkdataset, plotly-plotlydataset, plotly-jsondataset, matplotlib-matplotlibwriter]>=1.0 kedro-telemetry>=0.3.1 kedro-viz>=6.7.0 pytest-cov~=3.0 diff --git a/spaceflights-pyspark/{{ cookiecutter.repo_name }}/requirements.txt b/spaceflights-pyspark/{{ cookiecutter.repo_name }}/requirements.txt index dafb7874..f1dde932 100644 --- a/spaceflights-pyspark/{{ cookiecutter.repo_name }}/requirements.txt +++ b/spaceflights-pyspark/{{ cookiecutter.repo_name }}/requirements.txt @@ -2,7 +2,7 @@ ipython>=8.10 jupyterlab>=3.0 notebook kedro~={{ cookiecutter.kedro_version }} -kedro-datasets[pandas.CSVDataset, pandas.ExcelDataset, pandas.ParquetDataset, spark.SparkDataset]>=1.0 +kedro-datasets[pandas-csvdataset, pandas-exceldataset, pandas-parquetdataset, spark-sparkdataset]>=1.0 kedro-telemetry>=0.3.1 kedro-viz>=6.7.0 pytest-cov~=3.0 From 6aad22091b4871912feef3f41c99b99f27dfcc92 Mon Sep 17 00:00:00 2001 From: lrcouto Date: Thu, 11 Apr 2024 14:15:38 -0300 Subject: [PATCH 02/11] Rename nodes in pyspark starters Signed-off-by: lrcouto --- .../{{ cookiecutter.repo_name }}/conf/base/catalog.yml | 2 +- .../pipelines/data_processing/pipeline.py | 4 ++-- .../{{ cookiecutter.repo_name }}/conf/base/catalog.yml | 2 +- .../pipelines/data_processing/pipeline.py | 4 ++-- 4 files changed, 6 insertions(+), 6 deletions(-) diff --git a/spaceflights-pyspark-viz/{{ cookiecutter.repo_name }}/conf/base/catalog.yml b/spaceflights-pyspark-viz/{{ cookiecutter.repo_name }}/conf/base/catalog.yml index e168de3a..fd14e65c 100644 --- a/spaceflights-pyspark-viz/{{ cookiecutter.repo_name }}/conf/base/catalog.yml +++ b/spaceflights-pyspark-viz/{{ cookiecutter.repo_name }}/conf/base/catalog.yml @@ -63,7 +63,7 @@ reviews: header: True mode: overwrite -shuttles@excel: +shuttles_excel_data@excel: type: pandas.ExcelDataset filepath: data/01_raw/shuttles.xlsx diff --git a/spaceflights-pyspark-viz/{{ cookiecutter.repo_name }}/src/{{ cookiecutter.python_package }}/pipelines/data_processing/pipeline.py b/spaceflights-pyspark-viz/{{ cookiecutter.repo_name }}/src/{{ cookiecutter.python_package }}/pipelines/data_processing/pipeline.py index 7f11642b..8bec4a89 100755 --- a/spaceflights-pyspark-viz/{{ cookiecutter.repo_name }}/src/{{ cookiecutter.python_package }}/pipelines/data_processing/pipeline.py +++ b/spaceflights-pyspark-viz/{{ cookiecutter.repo_name }}/src/{{ cookiecutter.python_package }}/pipelines/data_processing/pipeline.py @@ -14,8 +14,8 @@ def create_pipeline(**kwargs) -> Pipeline: [ node( func=load_shuttles_to_csv, - inputs="shuttles@excel", - outputs="shuttles@csv", + inputs="shuttles_excel_data@excel", + outputs="shuttles_csv_data@csv", name="load_shuttles_to_csv_node", ), node( diff --git a/spaceflights-pyspark/{{ cookiecutter.repo_name }}/conf/base/catalog.yml b/spaceflights-pyspark/{{ cookiecutter.repo_name }}/conf/base/catalog.yml index 186c0b54..cc526041 100644 --- a/spaceflights-pyspark/{{ cookiecutter.repo_name }}/conf/base/catalog.yml +++ b/spaceflights-pyspark/{{ cookiecutter.repo_name }}/conf/base/catalog.yml @@ -63,7 +63,7 @@ reviews: header: True mode: overwrite -shuttles@excel: +shuttles_excel_data@excel: type: pandas.ExcelDataset filepath: data/01_raw/shuttles.xlsx diff --git a/spaceflights-pyspark/{{ cookiecutter.repo_name }}/src/{{ cookiecutter.python_package }}/pipelines/data_processing/pipeline.py b/spaceflights-pyspark/{{ cookiecutter.repo_name }}/src/{{ cookiecutter.python_package }}/pipelines/data_processing/pipeline.py index edafae85..da7d64d5 100755 --- a/spaceflights-pyspark/{{ cookiecutter.repo_name }}/src/{{ cookiecutter.python_package }}/pipelines/data_processing/pipeline.py +++ b/spaceflights-pyspark/{{ cookiecutter.repo_name }}/src/{{ cookiecutter.python_package }}/pipelines/data_processing/pipeline.py @@ -14,8 +14,8 @@ def create_pipeline(**kwargs) -> Pipeline: [ node( func=load_shuttles_to_csv, - inputs="shuttles@excel", - outputs="shuttles@csv", + inputs="shuttles_excel_data@excel", + outputs="shuttles_csv_data@csv", name="load_shuttles_to_csv_node", ), node( From 41220c0732a4032e8ca7ca5c5ef612f9365e1003 Mon Sep 17 00:00:00 2001 From: lrcouto Date: Thu, 11 Apr 2024 14:47:17 -0300 Subject: [PATCH 03/11] Restore old datasets naming convention for python < 3.9 Signed-off-by: lrcouto --- .../{{ cookiecutter.repo_name }}/requirements.txt | 3 ++- databricks-iris/{{ cookiecutter.repo_name }}/requirements.txt | 3 ++- .../{{ cookiecutter.repo_name }}/requirements.txt | 3 ++- .../{{ cookiecutter.repo_name }}/requirements.txt | 3 ++- .../{{ cookiecutter.repo_name }}/requirements.txt | 3 ++- .../{{ cookiecutter.repo_name }}/requirements.txt | 3 ++- 6 files changed, 12 insertions(+), 6 deletions(-) diff --git a/astro-airflow-iris/{{ cookiecutter.repo_name }}/requirements.txt b/astro-airflow-iris/{{ cookiecutter.repo_name }}/requirements.txt index a5913bee..dbe981bf 100644 --- a/astro-airflow-iris/{{ cookiecutter.repo_name }}/requirements.txt +++ b/astro-airflow-iris/{{ cookiecutter.repo_name }}/requirements.txt @@ -2,7 +2,8 @@ ipython>=8.10 jupyterlab>=3.0 notebook kedro~={{ cookiecutter.kedro_version }} -kedro-datasets[pandas-csvdataset]>=1.0 +kedro-datasets[pandas-csvdataset]>=1.0; python_version >= "3.9" +kedro-datasets[pandas.CSVDataset]>=1.0; python_version < '3.9' kedro-airflow~=0.5 kedro-telemetry>=0.3.1 pytest-cov~=3.0 diff --git a/databricks-iris/{{ cookiecutter.repo_name }}/requirements.txt b/databricks-iris/{{ cookiecutter.repo_name }}/requirements.txt index bd90bae1..96817d8f 100644 --- a/databricks-iris/{{ cookiecutter.repo_name }}/requirements.txt +++ b/databricks-iris/{{ cookiecutter.repo_name }}/requirements.txt @@ -2,7 +2,8 @@ ipython>=8.10 jupyterlab>=3.0 notebook kedro~={{ cookiecutter.kedro_version }} -kedro-datasets[spark-sparkdataset, pandas-parquetdataset]>=1.0 +kedro-datasets[spark-sparkdataset, pandas-parquetdataset]>=1.0; python_version >= "3.9" +kedro-datasets[spark.SparkDataset, pandas.ParquetDataset]>=1.0; python_version < "3.9" kedro-telemetry>=0.3.1 numpy~=1.21 pytest-cov~=3.0 diff --git a/spaceflights-pandas-viz/{{ cookiecutter.repo_name }}/requirements.txt b/spaceflights-pandas-viz/{{ cookiecutter.repo_name }}/requirements.txt index e032f713..060ad4b2 100644 --- a/spaceflights-pandas-viz/{{ cookiecutter.repo_name }}/requirements.txt +++ b/spaceflights-pandas-viz/{{ cookiecutter.repo_name }}/requirements.txt @@ -2,7 +2,8 @@ ipython>=8.10 jupyterlab>=3.0 notebook kedro~={{ cookiecutter.kedro_version }} -kedro-datasets[pandas-csvdataset, pandas-exceldataset, pandas-parquetdataset, plotly-plotlydataset, plotly-jsondataset, matplotlib-matplotlibwriter]>=1.0 +kedro-datasets[pandas-csvdataset, pandas-exceldataset, pandas-parquetdataset, plotly-plotlydataset, plotly-jsondataset, matplotlib-matplotlibwriter]>=1.0; python_version >= "3.9" +kedro-datasets[pandas.CSVDataset, pandas.ExcelDataset, pandas.ParquetDataset, plotly.PlotlyDataset, plotly.JSONDataset, matplotlib.MatplotlibWriter]>=1.0; python_version < "3.9" kedro-telemetry>=0.3.1 kedro-viz>=6.7.0 pytest-cov~=3.0 diff --git a/spaceflights-pandas/{{ cookiecutter.repo_name }}/requirements.txt b/spaceflights-pandas/{{ cookiecutter.repo_name }}/requirements.txt index fee12453..31f9c5a5 100644 --- a/spaceflights-pandas/{{ cookiecutter.repo_name }}/requirements.txt +++ b/spaceflights-pandas/{{ cookiecutter.repo_name }}/requirements.txt @@ -2,7 +2,8 @@ ipython>=8.10 jupyterlab>=3.0 notebook kedro~={{ cookiecutter.kedro_version }} -kedro-datasets[pandas-csvdataset, pandas-exceldataset, pandas-parquetdataset]>=1.0 +kedro-datasets[pandas-csvdataset, pandas-exceldataset, pandas-parquetdataset]>=1.0; python_version >= "3.9" +kedro-datasets[pandas.CSVDataset, pandas.ExcelDataset, pandas.ParquetDataset]>=1.0; python_version < "3.9" kedro-telemetry>=0.3.1 kedro-viz>=6.7.0 pytest-cov~=3.0 diff --git a/spaceflights-pyspark-viz/{{ cookiecutter.repo_name }}/requirements.txt b/spaceflights-pyspark-viz/{{ cookiecutter.repo_name }}/requirements.txt index 8fa4cf08..9d5108ce 100644 --- a/spaceflights-pyspark-viz/{{ cookiecutter.repo_name }}/requirements.txt +++ b/spaceflights-pyspark-viz/{{ cookiecutter.repo_name }}/requirements.txt @@ -2,7 +2,8 @@ ipython>=8.10 jupyterlab>=3.0 notebook kedro~={{ cookiecutter.kedro_version }} -kedro-datasets[pandas-csvdataset, pandas-exceldataset, pandas-parquetdataset, spark-sparkdataset, plotly-plotlydataset, plotly-jsondataset, matplotlib-matplotlibwriter]>=1.0 +kedro-datasets[pandas-csvdataset, pandas-exceldataset, pandas-parquetdataset, spark-sparkdataset, plotly-plotlydataset, plotly-jsondataset, matplotlib-matplotlibwriter]>=1.0; python_version >= "3.9" +kedro-datasets[pandas.CSVDataset, pandas.ExcelDataset, pandas.ParquetDataset, spark.SparkDataset, plotly.PlotlyDataset, plotly.JSONDataset, matplotlib.MatplotlibWriter]>=1.0; python_version < "3.9" kedro-telemetry>=0.3.1 kedro-viz>=6.7.0 pytest-cov~=3.0 diff --git a/spaceflights-pyspark/{{ cookiecutter.repo_name }}/requirements.txt b/spaceflights-pyspark/{{ cookiecutter.repo_name }}/requirements.txt index f1dde932..05595135 100644 --- a/spaceflights-pyspark/{{ cookiecutter.repo_name }}/requirements.txt +++ b/spaceflights-pyspark/{{ cookiecutter.repo_name }}/requirements.txt @@ -2,7 +2,8 @@ ipython>=8.10 jupyterlab>=3.0 notebook kedro~={{ cookiecutter.kedro_version }} -kedro-datasets[pandas-csvdataset, pandas-exceldataset, pandas-parquetdataset, spark-sparkdataset]>=1.0 +kedro-datasets[pandas-csvdataset, pandas-exceldataset, pandas-parquetdataset, spark-sparkdataset]>=1.0; python_version >= "3.9" +kedro-datasets[pandas.CSVDataset, pandas.ExcelDataset, pandas.ParquetDataset, spark.SparkDataset]>=1.0; python_version < "3.9" kedro-telemetry>=0.3.1 kedro-viz>=6.7.0 pytest-cov~=3.0 From 556960cedf02491175ea2b5cd8f98735bfa41ba3 Mon Sep 17 00:00:00 2001 From: lrcouto Date: Thu, 11 Apr 2024 15:30:42 -0300 Subject: [PATCH 04/11] Rename shuttles node Signed-off-by: lrcouto --- .../{{ cookiecutter.repo_name }}/conf/base/catalog.yml | 2 +- .../{{ cookiecutter.repo_name }}/conf/base/catalog.yml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/spaceflights-pyspark-viz/{{ cookiecutter.repo_name }}/conf/base/catalog.yml b/spaceflights-pyspark-viz/{{ cookiecutter.repo_name }}/conf/base/catalog.yml index fd14e65c..5599d7e7 100644 --- a/spaceflights-pyspark-viz/{{ cookiecutter.repo_name }}/conf/base/catalog.yml +++ b/spaceflights-pyspark-viz/{{ cookiecutter.repo_name }}/conf/base/catalog.yml @@ -67,7 +67,7 @@ shuttles_excel_data@excel: type: pandas.ExcelDataset filepath: data/01_raw/shuttles.xlsx -shuttles@csv: +shuttles_csv_data@csv: type: pandas.CSVDataset filepath: data/02_intermediate/shuttles.csv diff --git a/spaceflights-pyspark/{{ cookiecutter.repo_name }}/conf/base/catalog.yml b/spaceflights-pyspark/{{ cookiecutter.repo_name }}/conf/base/catalog.yml index cc526041..516074a0 100644 --- a/spaceflights-pyspark/{{ cookiecutter.repo_name }}/conf/base/catalog.yml +++ b/spaceflights-pyspark/{{ cookiecutter.repo_name }}/conf/base/catalog.yml @@ -67,7 +67,7 @@ shuttles_excel_data@excel: type: pandas.ExcelDataset filepath: data/01_raw/shuttles.xlsx -shuttles@csv: +shuttles_csv_data@csv: type: pandas.CSVDataset filepath: data/02_intermediate/shuttles.csv From c2975e2c46436d973fa1f262ddb17232048b47b9 Mon Sep 17 00:00:00 2001 From: Merel Theisen Date: Fri, 12 Apr 2024 15:34:52 +0100 Subject: [PATCH 05/11] Try pin kedro-datasets to 3.0 for python 3.9+ Signed-off-by: Merel Theisen --- .../{{ cookiecutter.repo_name }}/requirements.txt | 2 +- .../{{ cookiecutter.repo_name }}/requirements.txt | 2 +- .../{{ cookiecutter.repo_name }}/requirements.txt | 2 +- .../{{ cookiecutter.repo_name }}/requirements.txt | 2 +- .../{{ cookiecutter.repo_name }}/requirements.txt | 2 +- 5 files changed, 5 insertions(+), 5 deletions(-) diff --git a/astro-airflow-iris/{{ cookiecutter.repo_name }}/requirements.txt b/astro-airflow-iris/{{ cookiecutter.repo_name }}/requirements.txt index dbe981bf..fd18ac7b 100644 --- a/astro-airflow-iris/{{ cookiecutter.repo_name }}/requirements.txt +++ b/astro-airflow-iris/{{ cookiecutter.repo_name }}/requirements.txt @@ -2,7 +2,7 @@ ipython>=8.10 jupyterlab>=3.0 notebook kedro~={{ cookiecutter.kedro_version }} -kedro-datasets[pandas-csvdataset]>=1.0; python_version >= "3.9" +kedro-datasets[pandas-csvdataset]>=3.0; python_version >= "3.9" kedro-datasets[pandas.CSVDataset]>=1.0; python_version < '3.9' kedro-airflow~=0.5 kedro-telemetry>=0.3.1 diff --git a/spaceflights-pandas-viz/{{ cookiecutter.repo_name }}/requirements.txt b/spaceflights-pandas-viz/{{ cookiecutter.repo_name }}/requirements.txt index 060ad4b2..bb2e2a25 100644 --- a/spaceflights-pandas-viz/{{ cookiecutter.repo_name }}/requirements.txt +++ b/spaceflights-pandas-viz/{{ cookiecutter.repo_name }}/requirements.txt @@ -2,7 +2,7 @@ ipython>=8.10 jupyterlab>=3.0 notebook kedro~={{ cookiecutter.kedro_version }} -kedro-datasets[pandas-csvdataset, pandas-exceldataset, pandas-parquetdataset, plotly-plotlydataset, plotly-jsondataset, matplotlib-matplotlibwriter]>=1.0; python_version >= "3.9" +kedro-datasets[pandas-csvdataset, pandas-exceldataset, pandas-parquetdataset, plotly-plotlydataset, plotly-jsondataset, matplotlib-matplotlibwriter]>=3.0; python_version >= "3.9" kedro-datasets[pandas.CSVDataset, pandas.ExcelDataset, pandas.ParquetDataset, plotly.PlotlyDataset, plotly.JSONDataset, matplotlib.MatplotlibWriter]>=1.0; python_version < "3.9" kedro-telemetry>=0.3.1 kedro-viz>=6.7.0 diff --git a/spaceflights-pandas/{{ cookiecutter.repo_name }}/requirements.txt b/spaceflights-pandas/{{ cookiecutter.repo_name }}/requirements.txt index 31f9c5a5..b58e63f2 100644 --- a/spaceflights-pandas/{{ cookiecutter.repo_name }}/requirements.txt +++ b/spaceflights-pandas/{{ cookiecutter.repo_name }}/requirements.txt @@ -2,7 +2,7 @@ ipython>=8.10 jupyterlab>=3.0 notebook kedro~={{ cookiecutter.kedro_version }} -kedro-datasets[pandas-csvdataset, pandas-exceldataset, pandas-parquetdataset]>=1.0; python_version >= "3.9" +kedro-datasets[pandas-csvdataset, pandas-exceldataset, pandas-parquetdataset]>=3.0; python_version >= "3.9" kedro-datasets[pandas.CSVDataset, pandas.ExcelDataset, pandas.ParquetDataset]>=1.0; python_version < "3.9" kedro-telemetry>=0.3.1 kedro-viz>=6.7.0 diff --git a/spaceflights-pyspark-viz/{{ cookiecutter.repo_name }}/requirements.txt b/spaceflights-pyspark-viz/{{ cookiecutter.repo_name }}/requirements.txt index 9d5108ce..0b77cff7 100644 --- a/spaceflights-pyspark-viz/{{ cookiecutter.repo_name }}/requirements.txt +++ b/spaceflights-pyspark-viz/{{ cookiecutter.repo_name }}/requirements.txt @@ -2,7 +2,7 @@ ipython>=8.10 jupyterlab>=3.0 notebook kedro~={{ cookiecutter.kedro_version }} -kedro-datasets[pandas-csvdataset, pandas-exceldataset, pandas-parquetdataset, spark-sparkdataset, plotly-plotlydataset, plotly-jsondataset, matplotlib-matplotlibwriter]>=1.0; python_version >= "3.9" +kedro-datasets[pandas-csvdataset, pandas-exceldataset, pandas-parquetdataset, spark-sparkdataset, plotly-plotlydataset, plotly-jsondataset, matplotlib-matplotlibwriter]>=3.0; python_version >= "3.9" kedro-datasets[pandas.CSVDataset, pandas.ExcelDataset, pandas.ParquetDataset, spark.SparkDataset, plotly.PlotlyDataset, plotly.JSONDataset, matplotlib.MatplotlibWriter]>=1.0; python_version < "3.9" kedro-telemetry>=0.3.1 kedro-viz>=6.7.0 diff --git a/spaceflights-pyspark/{{ cookiecutter.repo_name }}/requirements.txt b/spaceflights-pyspark/{{ cookiecutter.repo_name }}/requirements.txt index 05595135..6516cc7a 100644 --- a/spaceflights-pyspark/{{ cookiecutter.repo_name }}/requirements.txt +++ b/spaceflights-pyspark/{{ cookiecutter.repo_name }}/requirements.txt @@ -2,7 +2,7 @@ ipython>=8.10 jupyterlab>=3.0 notebook kedro~={{ cookiecutter.kedro_version }} -kedro-datasets[pandas-csvdataset, pandas-exceldataset, pandas-parquetdataset, spark-sparkdataset]>=1.0; python_version >= "3.9" +kedro-datasets[pandas-csvdataset, pandas-exceldataset, pandas-parquetdataset, spark-sparkdataset]>=3.0; python_version >= "3.9" kedro-datasets[pandas.CSVDataset, pandas.ExcelDataset, pandas.ParquetDataset, spark.SparkDataset]>=1.0; python_version < "3.9" kedro-telemetry>=0.3.1 kedro-viz>=6.7.0 From 256c95480137a7490a1cbe4138b7293e245b56f8 Mon Sep 17 00:00:00 2001 From: Elena Khaustova Date: Fri, 12 Apr 2024 16:57:03 +0100 Subject: [PATCH 06/11] Rolled back to old datasets naming Signed-off-by: Elena Khaustova --- .../{{ cookiecutter.repo_name }}/requirements.txt | 3 +-- databricks-iris/{{ cookiecutter.repo_name }}/requirements.txt | 3 +-- .../{{ cookiecutter.repo_name }}/requirements.txt | 3 +-- .../{{ cookiecutter.repo_name }}/requirements.txt | 3 +-- .../{{ cookiecutter.repo_name }}/requirements.txt | 3 +-- .../{{ cookiecutter.repo_name }}/requirements.txt | 3 +-- 6 files changed, 6 insertions(+), 12 deletions(-) diff --git a/astro-airflow-iris/{{ cookiecutter.repo_name }}/requirements.txt b/astro-airflow-iris/{{ cookiecutter.repo_name }}/requirements.txt index fd18ac7b..f1aa0979 100644 --- a/astro-airflow-iris/{{ cookiecutter.repo_name }}/requirements.txt +++ b/astro-airflow-iris/{{ cookiecutter.repo_name }}/requirements.txt @@ -2,8 +2,7 @@ ipython>=8.10 jupyterlab>=3.0 notebook kedro~={{ cookiecutter.kedro_version }} -kedro-datasets[pandas-csvdataset]>=3.0; python_version >= "3.9" -kedro-datasets[pandas.CSVDataset]>=1.0; python_version < '3.9' +kedro-datasets[pandas.CSVDataset]>=1.0 kedro-airflow~=0.5 kedro-telemetry>=0.3.1 pytest-cov~=3.0 diff --git a/databricks-iris/{{ cookiecutter.repo_name }}/requirements.txt b/databricks-iris/{{ cookiecutter.repo_name }}/requirements.txt index 96817d8f..a3d3d113 100644 --- a/databricks-iris/{{ cookiecutter.repo_name }}/requirements.txt +++ b/databricks-iris/{{ cookiecutter.repo_name }}/requirements.txt @@ -2,8 +2,7 @@ ipython>=8.10 jupyterlab>=3.0 notebook kedro~={{ cookiecutter.kedro_version }} -kedro-datasets[spark-sparkdataset, pandas-parquetdataset]>=1.0; python_version >= "3.9" -kedro-datasets[spark.SparkDataset, pandas.ParquetDataset]>=1.0; python_version < "3.9" +kedro-datasets[spark.SparkDataset, pandas.ParquetDataset]>=1.0 kedro-telemetry>=0.3.1 numpy~=1.21 pytest-cov~=3.0 diff --git a/spaceflights-pandas-viz/{{ cookiecutter.repo_name }}/requirements.txt b/spaceflights-pandas-viz/{{ cookiecutter.repo_name }}/requirements.txt index bb2e2a25..0aaa6250 100644 --- a/spaceflights-pandas-viz/{{ cookiecutter.repo_name }}/requirements.txt +++ b/spaceflights-pandas-viz/{{ cookiecutter.repo_name }}/requirements.txt @@ -2,8 +2,7 @@ ipython>=8.10 jupyterlab>=3.0 notebook kedro~={{ cookiecutter.kedro_version }} -kedro-datasets[pandas-csvdataset, pandas-exceldataset, pandas-parquetdataset, plotly-plotlydataset, plotly-jsondataset, matplotlib-matplotlibwriter]>=3.0; python_version >= "3.9" -kedro-datasets[pandas.CSVDataset, pandas.ExcelDataset, pandas.ParquetDataset, plotly.PlotlyDataset, plotly.JSONDataset, matplotlib.MatplotlibWriter]>=1.0; python_version < "3.9" +kedro-datasets[pandas.CSVDataset, pandas.ExcelDataset, pandas.ParquetDataset, plotly.PlotlyDataset, plotly.JSONDataset, matplotlib.MatplotlibWriter]>=1.0 kedro-telemetry>=0.3.1 kedro-viz>=6.7.0 pytest-cov~=3.0 diff --git a/spaceflights-pandas/{{ cookiecutter.repo_name }}/requirements.txt b/spaceflights-pandas/{{ cookiecutter.repo_name }}/requirements.txt index b58e63f2..c41f20b4 100644 --- a/spaceflights-pandas/{{ cookiecutter.repo_name }}/requirements.txt +++ b/spaceflights-pandas/{{ cookiecutter.repo_name }}/requirements.txt @@ -2,8 +2,7 @@ ipython>=8.10 jupyterlab>=3.0 notebook kedro~={{ cookiecutter.kedro_version }} -kedro-datasets[pandas-csvdataset, pandas-exceldataset, pandas-parquetdataset]>=3.0; python_version >= "3.9" -kedro-datasets[pandas.CSVDataset, pandas.ExcelDataset, pandas.ParquetDataset]>=1.0; python_version < "3.9" +kedro-datasets[pandas.CSVDataset, pandas.ExcelDataset, pandas.ParquetDataset]>=1.0 kedro-telemetry>=0.3.1 kedro-viz>=6.7.0 pytest-cov~=3.0 diff --git a/spaceflights-pyspark-viz/{{ cookiecutter.repo_name }}/requirements.txt b/spaceflights-pyspark-viz/{{ cookiecutter.repo_name }}/requirements.txt index 0b77cff7..48b5f058 100644 --- a/spaceflights-pyspark-viz/{{ cookiecutter.repo_name }}/requirements.txt +++ b/spaceflights-pyspark-viz/{{ cookiecutter.repo_name }}/requirements.txt @@ -2,8 +2,7 @@ ipython>=8.10 jupyterlab>=3.0 notebook kedro~={{ cookiecutter.kedro_version }} -kedro-datasets[pandas-csvdataset, pandas-exceldataset, pandas-parquetdataset, spark-sparkdataset, plotly-plotlydataset, plotly-jsondataset, matplotlib-matplotlibwriter]>=3.0; python_version >= "3.9" -kedro-datasets[pandas.CSVDataset, pandas.ExcelDataset, pandas.ParquetDataset, spark.SparkDataset, plotly.PlotlyDataset, plotly.JSONDataset, matplotlib.MatplotlibWriter]>=1.0; python_version < "3.9" +kedro-datasets[pandas.CSVDataset, pandas.ExcelDataset, pandas.ParquetDataset, spark.SparkDataset, plotly.PlotlyDataset, plotly.JSONDataset, matplotlib.MatplotlibWriter]>=1.0 kedro-telemetry>=0.3.1 kedro-viz>=6.7.0 pytest-cov~=3.0 diff --git a/spaceflights-pyspark/{{ cookiecutter.repo_name }}/requirements.txt b/spaceflights-pyspark/{{ cookiecutter.repo_name }}/requirements.txt index 6516cc7a..dafb7874 100644 --- a/spaceflights-pyspark/{{ cookiecutter.repo_name }}/requirements.txt +++ b/spaceflights-pyspark/{{ cookiecutter.repo_name }}/requirements.txt @@ -2,8 +2,7 @@ ipython>=8.10 jupyterlab>=3.0 notebook kedro~={{ cookiecutter.kedro_version }} -kedro-datasets[pandas-csvdataset, pandas-exceldataset, pandas-parquetdataset, spark-sparkdataset]>=3.0; python_version >= "3.9" -kedro-datasets[pandas.CSVDataset, pandas.ExcelDataset, pandas.ParquetDataset, spark.SparkDataset]>=1.0; python_version < "3.9" +kedro-datasets[pandas.CSVDataset, pandas.ExcelDataset, pandas.ParquetDataset, spark.SparkDataset]>=1.0 kedro-telemetry>=0.3.1 kedro-viz>=6.7.0 pytest-cov~=3.0 From 5cc4943743150166d9b85079a8775b1e078a188e Mon Sep 17 00:00:00 2001 From: lrcouto Date: Fri, 12 Apr 2024 13:56:23 -0300 Subject: [PATCH 07/11] test pulling starters from this branch Signed-off-by: lrcouto --- .../{{ cookiecutter.repo_name }}/requirements.txt | 3 ++- features/environment.py | 2 +- .../{{ cookiecutter.repo_name }}/requirements.txt | 3 ++- .../{{ cookiecutter.repo_name }}/requirements.txt | 3 ++- .../{{ cookiecutter.repo_name }}/requirements.txt | 3 ++- .../{{ cookiecutter.repo_name }}/requirements.txt | 3 ++- test_requirements.txt | 2 +- 7 files changed, 12 insertions(+), 7 deletions(-) diff --git a/astro-airflow-iris/{{ cookiecutter.repo_name }}/requirements.txt b/astro-airflow-iris/{{ cookiecutter.repo_name }}/requirements.txt index f1aa0979..b49c5e61 100644 --- a/astro-airflow-iris/{{ cookiecutter.repo_name }}/requirements.txt +++ b/astro-airflow-iris/{{ cookiecutter.repo_name }}/requirements.txt @@ -2,7 +2,8 @@ ipython>=8.10 jupyterlab>=3.0 notebook kedro~={{ cookiecutter.kedro_version }} -kedro-datasets[pandas.CSVDataset]>=1.0 +kedro-datasets[pandas-csvdataset]>=3.0; python_version >= "3.9" +kedro-datasets[pandas.CSVDataset]>=1.0; python_version < "3.9" kedro-airflow~=0.5 kedro-telemetry>=0.3.1 pytest-cov~=3.0 diff --git a/features/environment.py b/features/environment.py index 0f6052f4..c3aa5b10 100644 --- a/features/environment.py +++ b/features/environment.py @@ -101,7 +101,7 @@ def before_scenario(context, scenario): "install", "-U", # pip==23.3 breaks dependency resolution - "pip==23.2", + "pip==23.3.1", ], env=context.env, ) diff --git a/spaceflights-pandas-viz/{{ cookiecutter.repo_name }}/requirements.txt b/spaceflights-pandas-viz/{{ cookiecutter.repo_name }}/requirements.txt index 0aaa6250..bb2e2a25 100644 --- a/spaceflights-pandas-viz/{{ cookiecutter.repo_name }}/requirements.txt +++ b/spaceflights-pandas-viz/{{ cookiecutter.repo_name }}/requirements.txt @@ -2,7 +2,8 @@ ipython>=8.10 jupyterlab>=3.0 notebook kedro~={{ cookiecutter.kedro_version }} -kedro-datasets[pandas.CSVDataset, pandas.ExcelDataset, pandas.ParquetDataset, plotly.PlotlyDataset, plotly.JSONDataset, matplotlib.MatplotlibWriter]>=1.0 +kedro-datasets[pandas-csvdataset, pandas-exceldataset, pandas-parquetdataset, plotly-plotlydataset, plotly-jsondataset, matplotlib-matplotlibwriter]>=3.0; python_version >= "3.9" +kedro-datasets[pandas.CSVDataset, pandas.ExcelDataset, pandas.ParquetDataset, plotly.PlotlyDataset, plotly.JSONDataset, matplotlib.MatplotlibWriter]>=1.0; python_version < "3.9" kedro-telemetry>=0.3.1 kedro-viz>=6.7.0 pytest-cov~=3.0 diff --git a/spaceflights-pandas/{{ cookiecutter.repo_name }}/requirements.txt b/spaceflights-pandas/{{ cookiecutter.repo_name }}/requirements.txt index c41f20b4..b58e63f2 100644 --- a/spaceflights-pandas/{{ cookiecutter.repo_name }}/requirements.txt +++ b/spaceflights-pandas/{{ cookiecutter.repo_name }}/requirements.txt @@ -2,7 +2,8 @@ ipython>=8.10 jupyterlab>=3.0 notebook kedro~={{ cookiecutter.kedro_version }} -kedro-datasets[pandas.CSVDataset, pandas.ExcelDataset, pandas.ParquetDataset]>=1.0 +kedro-datasets[pandas-csvdataset, pandas-exceldataset, pandas-parquetdataset]>=3.0; python_version >= "3.9" +kedro-datasets[pandas.CSVDataset, pandas.ExcelDataset, pandas.ParquetDataset]>=1.0; python_version < "3.9" kedro-telemetry>=0.3.1 kedro-viz>=6.7.0 pytest-cov~=3.0 diff --git a/spaceflights-pyspark-viz/{{ cookiecutter.repo_name }}/requirements.txt b/spaceflights-pyspark-viz/{{ cookiecutter.repo_name }}/requirements.txt index 48b5f058..0b77cff7 100644 --- a/spaceflights-pyspark-viz/{{ cookiecutter.repo_name }}/requirements.txt +++ b/spaceflights-pyspark-viz/{{ cookiecutter.repo_name }}/requirements.txt @@ -2,7 +2,8 @@ ipython>=8.10 jupyterlab>=3.0 notebook kedro~={{ cookiecutter.kedro_version }} -kedro-datasets[pandas.CSVDataset, pandas.ExcelDataset, pandas.ParquetDataset, spark.SparkDataset, plotly.PlotlyDataset, plotly.JSONDataset, matplotlib.MatplotlibWriter]>=1.0 +kedro-datasets[pandas-csvdataset, pandas-exceldataset, pandas-parquetdataset, spark-sparkdataset, plotly-plotlydataset, plotly-jsondataset, matplotlib-matplotlibwriter]>=3.0; python_version >= "3.9" +kedro-datasets[pandas.CSVDataset, pandas.ExcelDataset, pandas.ParquetDataset, spark.SparkDataset, plotly.PlotlyDataset, plotly.JSONDataset, matplotlib.MatplotlibWriter]>=1.0; python_version < "3.9" kedro-telemetry>=0.3.1 kedro-viz>=6.7.0 pytest-cov~=3.0 diff --git a/spaceflights-pyspark/{{ cookiecutter.repo_name }}/requirements.txt b/spaceflights-pyspark/{{ cookiecutter.repo_name }}/requirements.txt index dafb7874..6516cc7a 100644 --- a/spaceflights-pyspark/{{ cookiecutter.repo_name }}/requirements.txt +++ b/spaceflights-pyspark/{{ cookiecutter.repo_name }}/requirements.txt @@ -2,7 +2,8 @@ ipython>=8.10 jupyterlab>=3.0 notebook kedro~={{ cookiecutter.kedro_version }} -kedro-datasets[pandas.CSVDataset, pandas.ExcelDataset, pandas.ParquetDataset, spark.SparkDataset]>=1.0 +kedro-datasets[pandas-csvdataset, pandas-exceldataset, pandas-parquetdataset, spark-sparkdataset]>=3.0; python_version >= "3.9" +kedro-datasets[pandas.CSVDataset, pandas.ExcelDataset, pandas.ParquetDataset, spark.SparkDataset]>=1.0; python_version < "3.9" kedro-telemetry>=0.3.1 kedro-viz>=6.7.0 pytest-cov~=3.0 diff --git a/test_requirements.txt b/test_requirements.txt index a79bcb7b..8d542443 100644 --- a/test_requirements.txt +++ b/test_requirements.txt @@ -2,4 +2,4 @@ bandit>=1.6.2, <2.0 behave>=1.2.6, <2.0 PyYAML>=4.2, <7.0 ruff~=0.1.8 -git+https://github.com/kedro-org/kedro.git@main#egg=kedro +git+https://github.com/kedro-org/kedro.git@debug/point-to-test-starters-branch#egg=kedro From 14b775c586898b54961efd096493362a40b44ee0 Mon Sep 17 00:00:00 2001 From: lrcouto Date: Fri, 12 Apr 2024 16:42:51 -0300 Subject: [PATCH 08/11] Point back to main kedro branch on test requirements Signed-off-by: lrcouto --- test_requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test_requirements.txt b/test_requirements.txt index 8d542443..a79bcb7b 100644 --- a/test_requirements.txt +++ b/test_requirements.txt @@ -2,4 +2,4 @@ bandit>=1.6.2, <2.0 behave>=1.2.6, <2.0 PyYAML>=4.2, <7.0 ruff~=0.1.8 -git+https://github.com/kedro-org/kedro.git@debug/point-to-test-starters-branch#egg=kedro +git+https://github.com/kedro-org/kedro.git@main#egg=kedro From 654d8c8c1246733c61a8d247700f0c3e5a6b107d Mon Sep 17 00:00:00 2001 From: Elena Khaustova Date: Mon, 15 Apr 2024 12:42:13 +0100 Subject: [PATCH 09/11] Updated datasets's names to keep using transcoding Signed-off-by: Elena Khaustova --- .../{{ cookiecutter.repo_name }}/conf/base/catalog.yml | 4 ++-- .../pipelines/data_processing/pipeline.py | 4 ++-- .../{{ cookiecutter.repo_name }}/conf/base/catalog.yml | 4 ++-- .../pipelines/data_processing/pipeline.py | 4 ++-- 4 files changed, 8 insertions(+), 8 deletions(-) diff --git a/spaceflights-pyspark-viz/{{ cookiecutter.repo_name }}/conf/base/catalog.yml b/spaceflights-pyspark-viz/{{ cookiecutter.repo_name }}/conf/base/catalog.yml index 5599d7e7..a0755689 100644 --- a/spaceflights-pyspark-viz/{{ cookiecutter.repo_name }}/conf/base/catalog.yml +++ b/spaceflights-pyspark-viz/{{ cookiecutter.repo_name }}/conf/base/catalog.yml @@ -63,11 +63,11 @@ reviews: header: True mode: overwrite -shuttles_excel_data@excel: +shuttles_excel: type: pandas.ExcelDataset filepath: data/01_raw/shuttles.xlsx -shuttles_csv_data@csv: +shuttles@csv: type: pandas.CSVDataset filepath: data/02_intermediate/shuttles.csv diff --git a/spaceflights-pyspark-viz/{{ cookiecutter.repo_name }}/src/{{ cookiecutter.python_package }}/pipelines/data_processing/pipeline.py b/spaceflights-pyspark-viz/{{ cookiecutter.repo_name }}/src/{{ cookiecutter.python_package }}/pipelines/data_processing/pipeline.py index 8bec4a89..fbe37cd2 100755 --- a/spaceflights-pyspark-viz/{{ cookiecutter.repo_name }}/src/{{ cookiecutter.python_package }}/pipelines/data_processing/pipeline.py +++ b/spaceflights-pyspark-viz/{{ cookiecutter.repo_name }}/src/{{ cookiecutter.python_package }}/pipelines/data_processing/pipeline.py @@ -14,8 +14,8 @@ def create_pipeline(**kwargs) -> Pipeline: [ node( func=load_shuttles_to_csv, - inputs="shuttles_excel_data@excel", - outputs="shuttles_csv_data@csv", + inputs="shuttles_excel", + outputs="shuttles@csv", name="load_shuttles_to_csv_node", ), node( diff --git a/spaceflights-pyspark/{{ cookiecutter.repo_name }}/conf/base/catalog.yml b/spaceflights-pyspark/{{ cookiecutter.repo_name }}/conf/base/catalog.yml index 516074a0..55c1d844 100644 --- a/spaceflights-pyspark/{{ cookiecutter.repo_name }}/conf/base/catalog.yml +++ b/spaceflights-pyspark/{{ cookiecutter.repo_name }}/conf/base/catalog.yml @@ -63,11 +63,11 @@ reviews: header: True mode: overwrite -shuttles_excel_data@excel: +shuttles_excel: type: pandas.ExcelDataset filepath: data/01_raw/shuttles.xlsx -shuttles_csv_data@csv: +shuttles@csv: type: pandas.CSVDataset filepath: data/02_intermediate/shuttles.csv diff --git a/spaceflights-pyspark/{{ cookiecutter.repo_name }}/src/{{ cookiecutter.python_package }}/pipelines/data_processing/pipeline.py b/spaceflights-pyspark/{{ cookiecutter.repo_name }}/src/{{ cookiecutter.python_package }}/pipelines/data_processing/pipeline.py index da7d64d5..44e1b21d 100755 --- a/spaceflights-pyspark/{{ cookiecutter.repo_name }}/src/{{ cookiecutter.python_package }}/pipelines/data_processing/pipeline.py +++ b/spaceflights-pyspark/{{ cookiecutter.repo_name }}/src/{{ cookiecutter.python_package }}/pipelines/data_processing/pipeline.py @@ -14,8 +14,8 @@ def create_pipeline(**kwargs) -> Pipeline: [ node( func=load_shuttles_to_csv, - inputs="shuttles_excel_data@excel", - outputs="shuttles_csv_data@csv", + inputs="shuttles_excel", + outputs="shuttles@csv", name="load_shuttles_to_csv_node", ), node( From 502034745f96815749feb77aa550f1df339f8a02 Mon Sep 17 00:00:00 2001 From: Elena Khaustova Date: Mon, 15 Apr 2024 12:43:33 +0100 Subject: [PATCH 10/11] Point to test branch to ensure tests passing Signed-off-by: Elena Khaustova --- test_requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test_requirements.txt b/test_requirements.txt index a79bcb7b..8d542443 100644 --- a/test_requirements.txt +++ b/test_requirements.txt @@ -2,4 +2,4 @@ bandit>=1.6.2, <2.0 behave>=1.2.6, <2.0 PyYAML>=4.2, <7.0 ruff~=0.1.8 -git+https://github.com/kedro-org/kedro.git@main#egg=kedro +git+https://github.com/kedro-org/kedro.git@debug/point-to-test-starters-branch#egg=kedro From 5a4e821e0e8110e69b7dd175ab0a9eb386f45f96 Mon Sep 17 00:00:00 2001 From: Elena Khaustova Date: Mon, 15 Apr 2024 14:38:26 +0100 Subject: [PATCH 11/11] Point back to main kedro branch Signed-off-by: Elena Khaustova --- test_requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test_requirements.txt b/test_requirements.txt index 8d542443..a79bcb7b 100644 --- a/test_requirements.txt +++ b/test_requirements.txt @@ -2,4 +2,4 @@ bandit>=1.6.2, <2.0 behave>=1.2.6, <2.0 PyYAML>=4.2, <7.0 ruff~=0.1.8 -git+https://github.com/kedro-org/kedro.git@debug/point-to-test-starters-branch#egg=kedro +git+https://github.com/kedro-org/kedro.git@main#egg=kedro