cleanup

InseeFrLab · Jan 3, 2024 · a617e16 · a617e16
1 parent 6e42a5b
commit a617e16
Show file tree

Hide file tree

Showing 2 changed files with 61 additions and 78 deletions.
diff --git a/cartiflette/mapshaper/mapshaperize.py b/cartiflette/mapshaper/mapshaperize.py
@@ -1,4 +1,6 @@
+import os
 import subprocess
+
 from cartiflette.utils import DICT_CORRESP_IGN
 from .mapshaper_wrangling import mapshaper_enrich, mapshaper_split
 
@@ -67,6 +69,8 @@ def mapshaperize_split(
         f"{local_dir}/{territory}/{niveau_agreg}/{format_output}/{simplification=}"
     )
 
+    os.makedirs(output_path, exist_ok=True)
+
     if simplification_percent != 0:
         option_simplify = f"-simplify {simplification_percent}% "
     else:

diff --git a/misc/prototype_mapshaper.py b/misc/prototype_mapshaper.py
@@ -1,32 +1,24 @@
-from cartiflette.s3 import upload_s3_raw
-from cartiflette.pipeline import crossproduct_parameters_production
-from cartiflette.pipeline import mapshaperize_split_from_s3, mapshaperize_merge_split_from_s3
-from cartiflette.download.download import _download_sources
-
-
-import subprocess
-
-from cartiflette.utils import import_yaml_config
-from cartiflette.pipeline.prepare_mapshaper import prepare_local_directory_mapshaper
-from cartiflette import FS, BUCKET, PATH_WITHIN_BUCKET
-
-
-# DATA RETRIEVING STEP =========================
-
-path_within_bucket = "test-download28"
-
-from cartiflette import BUCKET, PATH_WITHIN_BUCKET, FS, DICT_CORRESP_IGN
+from cartiflette import BUCKET, PATH_WITHIN_BUCKET, FS
 from cartiflette.utils import create_path_bucket
 from cartiflette.mapshaper.mapshaper_split import mapshaper_enrich
-from cartiflette.pipeline.combine_adminexpress_france import combine_adminexpress_territory
+from cartiflette.pipeline.combine_adminexpress_france import (
+    combine_adminexpress_territory,
+)
 from cartiflette.pipeline.prepare_cog_metadata import prepare_cog_metadata
 
+from cartiflette.pipeline import crossproduct_parameters_production
+from cartiflette.pipeline import (
+    mapshaperize_split_from_s3,
+    mapshaperize_merge_split_from_s3,
+)
+
+# DATA RETRIEVING STEP =========================
 
-bucket=BUCKET
-path_within_bucket="test-download28"
-year=2022
-fs=FS
-#path_within_bucket = PATH_WITHIN_BUCKET
+bucket = BUCKET
+path_within_bucket = "test-download28"
+year = 2022
+fs = FS
+# path_within_bucket = PATH_WITHIN_BUCKET
 
 # PART 1/ COMBINE RAW FILES TOGETHER AND WRITE TO S3
 
@@ -35,113 +27,100 @@
 )
 
 path_raw_s3 = create_path_bucket(
-            {
-                "bucket": bucket,
-                "path_within_bucket": path_within_bucket,
-                "year": year,
-                "borders": "france",
-                "crs": 4326,
-                "filter_by": "preprocessed",
-                "value": "before_cog",
-                "vectorfile_format": "geojson",
-                "provider": "IGN",
-                "dataset_family": 'ADMINEXPRESS',
-                "source": 'EXPRESS-COG-CARTO-TERRITOIRE',
-                "territory": "france",
-                "filename": "raw.geojson",
-                "simplification": 0,
-            }
-        )
+    {
+        "bucket": bucket,
+        "path_within_bucket": path_within_bucket,
+        "year": year,
+        "borders": "france",
+        "crs": 4326,
+        "filter_by": "preprocessed",
+        "value": "before_cog",
+        "vectorfile_format": "geojson",
+        "provider": "IGN",
+        "dataset_family": "ADMINEXPRESS",
+        "source": "EXPRESS-COG-CARTO-TERRITOIRE",
+        "territory": "france",
+        "filename": "raw.geojson",
+        "simplification": 0,
+    }
+)
 
 fs.put_file(path_combined_files, path_raw_s3)
 
 
 # STEP 2: ENRICH AND SPLIT ----------------------
 
 # Retrieve COG metadata
-tagc_metadata = prepare_cog_metadata(
-    path_within_bucket
-)
+tagc_metadata = prepare_cog_metadata(path_within_bucket)
 tagc_metadata.drop(columns=["LIBGEO"]).to_csv("temp/tagc.csv")
 
 
 mapshaperize_split_from_s3(
     {
-        'path_within_bucket': path_within_bucket,
+        "path_within_bucket": path_within_bucket,
         "level_polygons": "COMMUNE",
         "filter_by": "REGION",
-        "simplification": 50
+        "simplification": 50,
     }
 )
 
 mapshaperize_merge_split_from_s3(
     {
-        'path_within_bucket': path_within_bucket,
+        "path_within_bucket": path_within_bucket,
         "level_polygons": "COMMUNE",
         "filter_by": "REGION",
-        "simplification": 50
+        "simplification": 50,
     }
 )
 
 
-
 # GENERALISATION
 
 croisement_decoupage_level = {
     ## structure -> niveau geo: [niveau decoupage macro],
     "REGION": ["FRANCE_ENTIERE"],
-    #"ARRONDISSEMENT_MUNICIPAL" : ['DEPARTEMENT'], 
-    #"COMMUNE_ARRONDISSEMENT": ["DEPARTEMENT", "REGION", "FRANCE_ENTIERE"],
+    # "ARRONDISSEMENT_MUNICIPAL" : ['DEPARTEMENT'],
+    # "COMMUNE_ARRONDISSEMENT": ["DEPARTEMENT", "REGION", "FRANCE_ENTIERE"],
     "COMMUNE": ["DEPARTEMENT", "REGION", "FRANCE_ENTIERE"],
-    "DEPARTEMENT": ["REGION", "FRANCE_ENTIERE"]
+    "DEPARTEMENT": ["REGION", "FRANCE_ENTIERE"],
 }
 
 
 formats = ["topojson", "geojson"]
-formats = ["geojson"]
 
-years = [y for y in range(2021, 2023)]
-#years = [2022]
+# years = [y for y in range(2021, 2023)]
+years = [2022]
 
-#crs_list = [4326, 2154, "official"]
-#crs_list = [4326, 2154]
-crs_list = [4326, 2154]
+# crs_list = [4326, 2154, "official"]
+# crs_list = [4326, 2154]
+crs_list = [4326]
 
 sources = ["EXPRESS-COG-CARTO-TERRITOIRE"]
 
 
 tempdf = crossproduct_parameters_production(
-        croisement_filter_by_borders=croisement_decoupage_level,
-        list_format=formats,
-        years=years,
-        crs_list=crs_list,
-        sources=sources,
-        simplifications=[0, 50]
-    )
-
+    croisement_filter_by_borders=croisement_decoupage_level,
+    list_format=formats,
+    years=years,
+    crs_list=crs_list,
+    sources=sources,
+    simplifications=[0, 50],
+)
 
 for index, row in tempdf.iterrows():
     print(row)
     mapshaperize_split_from_s3(
-        path_bucket_adminexpress,
-        {
-            **{'path_within_bucket': path_within_bucket},
-            **row.to_dict()
-        }
+        {**{"path_within_bucket": path_within_bucket}, **row.to_dict()},
     )
 
 
 # niveau commune_arrondissement
 
-tempdf_arr = tempdf.loc[tempdf['level_polygons'] == "COMMUNE"].copy()
-tempdf_arr = tempdf_arr.drop(columns = ['level_polygons'])
+tempdf_arr = tempdf.loc[tempdf["level_polygons"] == "COMMUNE"].copy()
+tempdf_arr = tempdf_arr.drop(columns=["level_polygons"])
 
 for index, row in tempdf_arr.iterrows():
     print(row)
     mapshaperize_merge_split_from_s3(
-        path_bucket_adminexpress,
-        {
-            **{'path_within_bucket': path_within_bucket},
-            **row.to_dict()
-        }
+        {**{"path_within_bucket": path_within_bucket}, **row.to_dict()},
     )