From e17f8b2336f011c583493678eb10e61b06088d41 Mon Sep 17 00:00:00 2001 From: Daniel Mannarino Date: Tue, 10 Dec 2024 16:16:43 -0500 Subject: [PATCH 1/7] Replace out-of-tree dockerfiles (except pixetl) with one dockerfile to rule them all --- batch/universal_batch.dockerfile | 66 ++++++++++++++++++++++++++++++++ scripts/develop | 4 +- scripts/test | 4 +- scripts/test_v2 | 4 +- terraform/main.tf | 6 +-- tests/conftest.py | 6 +-- 6 files changed, 75 insertions(+), 15 deletions(-) create mode 100644 batch/universal_batch.dockerfile diff --git a/batch/universal_batch.dockerfile b/batch/universal_batch.dockerfile new file mode 100644 index 00000000..a19156bb --- /dev/null +++ b/batch/universal_batch.dockerfile @@ -0,0 +1,66 @@ +FROM ghcr.io/osgeo/gdal:ubuntu-full-3.9.3 +LABEL desc="Docker image with ALL THE THINGS for use in Batch by the GFW data API" +LABEL version="v1.1" + +ENV TIPPECANOE_VERSION=2.72.0 + +ENV VENV_DIR="/.venv" + +RUN apt-get update -y \ + && apt-get install --no-install-recommends -y python3 python-dev-is-python3 python3-venv \ + postgresql-client jq curl libsqlite3-dev zlib1g-dev zip libpq-dev build-essential gcc g++ \ + && apt-get clean \ + && rm -rf /var/lib/apt/lists/* + +RUN ln -s /usr/include /usr/include/gdal + +# --system-site-packages is needed to copy the GDAL Python libs into the venv +RUN python -m venv ${VENV_DIR} --system-site-packages \ + && . ${VENV_DIR}/bin/activate \ + && python -m ensurepip --upgrade \ + && python -m pip install \ + agate~=1.12.0 \ + asyncpg~=0.30.0 \ + awscli~=1.36.18 \ + awscli-plugin-endpoint~=0.4 \ + boto3~=1.35.77 \ + click~=8.1.7 \ + csvkit~=2.0.1 \ + earthengine-api~=0.1.408 \ + fiona~=1.9.6 \ + gsutil~=5.31 \ + numpy~=1.26.4 \ + pandas~=2.1.4 \ + psycopg2~=2.9.10 \ + rasterio~=1.3.11 \ + setuptools~=75.6 \ + shapely~=2.0.4 \ + SQLAlchemy~=1.3.24 \ + tileputty~=0.2.10 + +RUN ln -s /usr/include /usr/include/gdal + +# Install TippeCanoe +RUN mkdir -p /opt/src +WORKDIR /opt/src +RUN curl https://codeload.github.com/felt/tippecanoe/tar.gz/${TIPPECANOE_VERSION} | tar -xz \ + && cd /opt/src/tippecanoe-${TIPPECANOE_VERSION} \ + && make \ + && make install \ + && rm -R /opt/src/tippecanoe-${TIPPECANOE_VERSION} + +# Copy scripts +COPY ./batch/scripts/ /opt/scripts/ +COPY ./batch/python/ /opt/python/ + +# Make sure scripts are executable +RUN chmod +x -R /opt/scripts/ +RUN chmod +x -R /opt/python/ + +ENV PATH="/opt/scripts:${PATH}" +ENV PATH="/opt/python:${PATH}" + +ENV WORKDIR="/" +WORKDIR / + +ENTRYPOINT ["/opt/scripts/report_status.sh"] \ No newline at end of file diff --git a/scripts/develop b/scripts/develop index 45e62d93..87f97dde 100755 --- a/scripts/develop +++ b/scripts/develop @@ -24,9 +24,7 @@ done set -- "${POSITIONAL[@]}" # restore positional parameters if [ "${BUILD}" = true ]; then - docker build -t batch_gdal-python_test . -f batch/gdal-python.dockerfile - docker build -t batch_postgresql-client_test . -f batch/postgresql-client.dockerfile - docker build -t batch_tile_cache_test . -f batch/tile_cache.dockerfile + docker build -t batch_jobs_test . -f batch/universal_batch.dockerfile docker build -t pixetl_test . -f batch/pixetl.dockerfile docker compose -f docker-compose.dev.yml --project-name gfw-data-api_dev up --abort-on-container-exit --remove-orphans --build else diff --git a/scripts/test b/scripts/test index 95dbf3e8..ce4d3d67 100755 --- a/scripts/test +++ b/scripts/test @@ -60,9 +60,7 @@ if [ $# -eq 0 ]; then fi if [ "${BUILD}" = true ]; then - docker build -t batch_gdal-python_test . -f batch/gdal-python.dockerfile - docker build -t batch_postgresql-client_test . -f batch/postgresql-client.dockerfile - docker build -t batch_tile_cache_test . -f batch/tile_cache.dockerfile + docker build -t batch_jobs_test . -f batch/universal_batch.dockerfile docker build -t pixetl_test . -f batch/pixetl.dockerfile docker compose -f docker-compose.test.yml --project-name gfw-data-api_test build --no-cache app_test fi diff --git a/scripts/test_v2 b/scripts/test_v2 index 1a78adce..df8ec17e 100755 --- a/scripts/test_v2 +++ b/scripts/test_v2 @@ -52,9 +52,7 @@ if [ $# -eq 0 ]; then fi if [ "${BUILD}" = true ]; then - docker build -t batch_gdal-python_test . -f batch/gdal-python.dockerfile - docker build -t batch_postgresql-client_test . -f batch/postgresql-client.dockerfile - docker build -t batch_tile_cache_test . -f batch/tile_cache.dockerfile + docker build -t batch_jobs_test . -f batch/universal_batch.dockerfile docker build -t pixetl_test . -f batch/pixetl.dockerfile docker compose -f docker-compose.test.yml --project-name gfw-data-api_test build --no-cache app_test fi diff --git a/terraform/main.tf b/terraform/main.tf index 08933a1e..2b860673 100644 --- a/terraform/main.tf +++ b/terraform/main.tf @@ -50,7 +50,7 @@ module "batch_gdal_python_image" { image_name = substr(lower("${local.project}-gdal_python${local.name_suffix}"), 0, 64) root_dir = "${path.root}/../" docker_path = "batch" - docker_filename = "gdal-python.dockerfile" + docker_filename = "universal_batch.dockerfile" } # Docker image for PixETL Batch jobs @@ -68,7 +68,7 @@ module "batch_postgresql_client_image" { image_name = substr(lower("${local.project}-postgresql_client${local.name_suffix}"), 0, 64) root_dir = "${path.root}/../" docker_path = "batch" - docker_filename = "postgresql-client.dockerfile" + docker_filename = "universal_batch.dockerfile" } # Docker image for Tile Cache Batch jobs @@ -77,7 +77,7 @@ module "batch_tile_cache_image" { image_name = substr(lower("${local.project}-tile_cache${local.name_suffix}"), 0, 64) root_dir = "${path.root}/../" docker_path = "batch" - docker_filename = "tile_cache.dockerfile" + docker_filename = "universal_batch.dockerfile" } diff --git a/tests/conftest.py b/tests/conftest.py index 74e415d2..ad28b320 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -181,11 +181,11 @@ def patch_run(self, *k, **kwargs): ON_DEMAND_COMPUTE_JOB_QUEUE, cogify_env["computeEnvironmentArn"] ) - aws_mock.add_job_definition(GDAL_PYTHON_JOB_DEFINITION, "batch_gdal-python_test") + aws_mock.add_job_definition(GDAL_PYTHON_JOB_DEFINITION, "batch_jobs_test") aws_mock.add_job_definition( - POSTGRESQL_CLIENT_JOB_DEFINITION, "batch_postgresql-client_test" + POSTGRESQL_CLIENT_JOB_DEFINITION, "batch_jobs_test" ) - aws_mock.add_job_definition(TILE_CACHE_JOB_DEFINITION, "batch_tile_cache_test") + aws_mock.add_job_definition(TILE_CACHE_JOB_DEFINITION, "batch_jobs_test") aws_mock.add_job_definition(PIXETL_JOB_DEFINITION, "pixetl_test", mount_tmp=True) yield aws_mock.mocked_services["batch"]["client"], aws_mock.mocked_services["logs"][ From de1d5faf245dafef71feffd2615b0620eccef117 Mon Sep 17 00:00:00 2001 From: Daniel Mannarino Date: Tue, 10 Dec 2024 16:17:27 -0500 Subject: [PATCH 2/7] Remove old dockerfiles --- batch/gdal-python.dockerfile | 16 ---------------- batch/postgresql-client.dockerfile | 16 ---------------- batch/tile_cache.dockerfile | 16 ---------------- 3 files changed, 48 deletions(-) delete mode 100644 batch/gdal-python.dockerfile delete mode 100644 batch/postgresql-client.dockerfile delete mode 100644 batch/tile_cache.dockerfile diff --git a/batch/gdal-python.dockerfile b/batch/gdal-python.dockerfile deleted file mode 100644 index c2d4a598..00000000 --- a/batch/gdal-python.dockerfile +++ /dev/null @@ -1,16 +0,0 @@ -FROM globalforestwatch/data-api-gdal:v1.2.2 - -# Copy scripts -COPY ./batch/scripts/ /opt/scripts/ -COPY ./batch/python/ /opt/python/ - -# Make sure scripts are executable -RUN chmod +x -R /opt/scripts/ -RUN chmod +x -R /opt/python/ - -ENV PATH="/opt/scripts:${PATH}" -ENV PATH="/opt/python:${PATH}" - -ENV WORKDIR="/tmp" - -ENTRYPOINT ["/opt/scripts/report_status.sh"] \ No newline at end of file diff --git a/batch/postgresql-client.dockerfile b/batch/postgresql-client.dockerfile deleted file mode 100644 index 3dbefb1a..00000000 --- a/batch/postgresql-client.dockerfile +++ /dev/null @@ -1,16 +0,0 @@ -FROM globalforestwatch/data-api-postgresql:v1.1.1 - -# Copy scripts -COPY ./batch/scripts/ /opt/scripts/ -COPY ./batch/python/ /opt/python/ - -# make sure scripts are executable -RUN chmod +x -R /opt/scripts/ -RUN chmod +x -R /opt/python/ - -ENV PATH="/opt/scripts:${PATH}" -ENV PATH="/opt/python:${PATH}" - -WORKDIR /tmp - -ENTRYPOINT ["/opt/scripts/report_status.sh"] \ No newline at end of file diff --git a/batch/tile_cache.dockerfile b/batch/tile_cache.dockerfile deleted file mode 100644 index 3ab80ba3..00000000 --- a/batch/tile_cache.dockerfile +++ /dev/null @@ -1,16 +0,0 @@ -FROM globalforestwatch/data-api-tippecanoe:v1.3.1 - -# Copy scripts -COPY ./batch/scripts/ /opt/scripts/ -COPY ./batch/python/ /opt/python/ - -# make sure scripts are executable -RUN chmod +x -R /opt/scripts/ -RUN chmod +x -R /opt/python/ - -ENV PATH="/opt/scripts:${PATH}" -ENV PATH="/opt/python:${PATH}" - -WORKDIR /tmp - -ENTRYPOINT ["/opt/scripts/report_status.sh"] \ No newline at end of file From b864ec3277b695e66f373a3803545fb4aacdbb93 Mon Sep 17 00:00:00 2001 From: Daniel Mannarino Date: Sun, 15 Dec 2024 19:47:31 -0500 Subject: [PATCH 3/7] Remove obsolete symlinking --- batch/universal_batch.dockerfile | 4 ---- 1 file changed, 4 deletions(-) diff --git a/batch/universal_batch.dockerfile b/batch/universal_batch.dockerfile index a19156bb..f7103a64 100644 --- a/batch/universal_batch.dockerfile +++ b/batch/universal_batch.dockerfile @@ -12,8 +12,6 @@ RUN apt-get update -y \ && apt-get clean \ && rm -rf /var/lib/apt/lists/* -RUN ln -s /usr/include /usr/include/gdal - # --system-site-packages is needed to copy the GDAL Python libs into the venv RUN python -m venv ${VENV_DIR} --system-site-packages \ && . ${VENV_DIR}/bin/activate \ @@ -38,8 +36,6 @@ RUN python -m venv ${VENV_DIR} --system-site-packages \ SQLAlchemy~=1.3.24 \ tileputty~=0.2.10 -RUN ln -s /usr/include /usr/include/gdal - # Install TippeCanoe RUN mkdir -p /opt/src WORKDIR /opt/src From f57c8275ef450995e58c290d048a61d5cdd0e137 Mon Sep 17 00:00:00 2001 From: Daniel Mannarino Date: Sun, 15 Dec 2024 19:56:29 -0500 Subject: [PATCH 4/7] Only build one image in TF, instead of 3 --- terraform/main.tf | 31 ++++++------------------------- 1 file changed, 6 insertions(+), 25 deletions(-) diff --git a/terraform/main.tf b/terraform/main.tf index 2b860673..b55791eb 100644 --- a/terraform/main.tf +++ b/terraform/main.tf @@ -44,15 +44,6 @@ module "app_docker_image" { tag = local.container_tag } -# Docker image for GDAL Python Batch jobs -module "batch_gdal_python_image" { - source = "git::https://github.com/wri/gfw-terraform-modules.git//terraform/modules/container_registry?ref=v0.4.2.3" - image_name = substr(lower("${local.project}-gdal_python${local.name_suffix}"), 0, 64) - root_dir = "${path.root}/../" - docker_path = "batch" - docker_filename = "universal_batch.dockerfile" -} - # Docker image for PixETL Batch jobs module "batch_pixetl_image" { source = "git::https://github.com/wri/gfw-terraform-modules.git//terraform/modules/container_registry?ref=v0.4.2.3" @@ -62,25 +53,15 @@ module "batch_pixetl_image" { docker_filename = "pixetl.dockerfile" } -# Docker image for PostgreSQL Client Batch jobs -module "batch_postgresql_client_image" { +# Docker image for all Batch jobs except those requiring PixETL +module "batch_universal_image" { source = "git::https://github.com/wri/gfw-terraform-modules.git//terraform/modules/container_registry?ref=v0.4.2.3" - image_name = substr(lower("${local.project}-postgresql_client${local.name_suffix}"), 0, 64) + image_name = substr(lower("${local.project}-universal${local.name_suffix}"), 0, 64) root_dir = "${path.root}/../" docker_path = "batch" docker_filename = "universal_batch.dockerfile" } -# Docker image for Tile Cache Batch jobs -module "batch_tile_cache_image" { - source = "git::https://github.com/wri/gfw-terraform-modules.git//terraform/modules/container_registry?ref=v0.4.2.3" - image_name = substr(lower("${local.project}-tile_cache${local.name_suffix}"), 0, 64) - root_dir = "${path.root}/../" - docker_path = "batch" - docker_filename = "universal_batch.dockerfile" -} - - module "fargate_autoscaling" { source = "git::https://github.com/wri/gfw-terraform-modules.git//terraform/modules/fargate_autoscaling?ref=v0.4.2.5" project = local.project @@ -221,10 +202,10 @@ module "batch_job_queues" { environment = var.environment name_suffix = local.name_suffix project = local.project - gdal_repository_url = "${module.batch_gdal_python_image.repository_url}:latest" + gdal_repository_url = "${module.batch_universal_image.repository_url}:latest" pixetl_repository_url = "${module.batch_pixetl_image.repository_url}:latest" - postgres_repository_url = "${module.batch_postgresql_client_image.repository_url}:latest" - tile_cache_repository_url = "${module.batch_tile_cache_image.repository_url}:latest" + postgres_repository_url = "${module.batch_universal_image.repository_url}:latest" + tile_cache_repository_url = "${module.batch_universal_image.repository_url}:latest" iam_policy_arn = [ "arn:aws:iam::aws:policy/AmazonS3ReadOnlyAccess", aws_iam_policy.query_batch_jobs.arn, From 7489662eb816abc002a57afbd9d6f66f9476ed68 Mon Sep 17 00:00:00 2001 From: Daniel Mannarino Date: Sun, 15 Dec 2024 22:12:05 -0500 Subject: [PATCH 5/7] Fix (intermittant?) bug in httpd fixture --- tests/conftest.py | 2 +- tests/routes/datasets/test_versions.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/conftest.py b/tests/conftest.py index ad28b320..00459869 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -223,7 +223,7 @@ def httpd(): t.join() -@pytest.fixture(autouse=True) +@pytest.fixture(autouse=True, scope="function") def flush_request_list(httpd): """Delete request cache before every test.""" _ = httpx.delete(f"http://localhost:{httpd.server_port}") diff --git a/tests/routes/datasets/test_versions.py b/tests/routes/datasets/test_versions.py index 0c3b084f..2eeed859 100755 --- a/tests/routes/datasets/test_versions.py +++ b/tests/routes/datasets/test_versions.py @@ -519,7 +519,7 @@ async def test_version_post_append(async_client: AsyncClient): @pytest.mark.hanging @pytest.mark.asyncio -async def test_version_put_raster_bug_fixes(async_client: AsyncClient): +async def test_version_put_raster_bug_fixes(async_client: AsyncClient, httpd): """Test bug fixes for raster source version operations.""" dataset = "test_version_put_raster_minimal_args" From be3fb398000c9283a361671db70344dfcd313f66 Mon Sep 17 00:00:00 2001 From: Daniel Mannarino Date: Mon, 16 Dec 2024 17:30:53 -0500 Subject: [PATCH 6/7] Replace test.gpkg.zip with one that works (old one contained resource fork); hopefully fix missing geom columns error --- tests/fixtures/test.gpkg.zip | Bin 7016 -> 6738 bytes tests/routes/datasets/test_versions.py | 6 ++++-- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/tests/fixtures/test.gpkg.zip b/tests/fixtures/test.gpkg.zip index 10fbec15631bf606e942e08fe02bf86b8f54e788..3562da31e66eedd399908c37b2f2a1283fa59bfe 100644 GIT binary patch literal 6738 zcmcJUcTm&Yw#S1WK}A3ilrA8>3xW_p4!w5*hAKknO$famQ9w#SdM|?X5=ux29hJ~~ zFQFqy?*W43o!ibi_s-mTGjHbYe}2EU_g=GqYtNeBXYcRQ)wuZ!H2?s(4cLq1ez|qt z2`{7j-2~hK+yO8Gpb$@}pq;yu9mwzw;3mN{3!+l=%FfG|6hQRLDG>nhch{Y4b7uyX z;P%vU5{^Eq-6jcXB{gZQJkvu>b#@3i#ks5KvskAh8zi`0FUz-%Gj-f(O|Y&B!97Lh z*ysJ*b_?SkGdUWd6yHZ=e6m`obvztFy{oWz=^n8X<@B~FwWxFbw{4mM8AN-bXw%iS zP%ollGHRdj4zbpMwIM4?xES)VOUW8AGodtOrr|(;DYef4%A?K3Z@uS^0!9&foD#uT zi%z$jxTCJ0`aQ4=;L(=!Ov;cZPz13T;5OK=5V(-$wxjD&{BZ~D6D~qr#*LhLf9@!2 zTW0rq{IrD}7`%#ZfqoNBAtjraEbEmo{nl5wWg7_!6B%03P7L!uhG=k?FjTPcHdmNH z9*$}9_vJYQDdR*P6nzTbDUG3)YsXXC-n>M{7qFyYxdhbw?3X(=+ z`!ssAMln47gTUv}8J2vnY*niGUUR&Q-z-JSSpPvM-9zl?QiHC^tu4Td4X(5mms ztF?c!?3NjfL_A+hq3=Dqo+AOon~&^Qpf?n<5+W8e?C8m|s*@xewbN^`>aTm!_!QqL z{GC~5v}if+eD!Olo84h`*Q2}Wy#WV^DJe2MA`=-UP$Ar%5ykn8qoNz+n*^~K%(uBq z*f25*4X=IB!>_P2uPzEp&@N?Y)}7Ve?J+rIs&#b87BLC~?hOIUZ$I}%B=yD6@5{WL$tSB=YHUZYqYrqRq)4U4ELc_`L($>+uL8^y|y^QuXV9P-`-p+gG zN$K;-Bfe+zI|d6s$}gi0yRi|04@&NN1+xB}kr(|LT-1Z?<0q5lFhv&9- z=59=GIA~-lZ!@pk>RFN#jozCW#%f$W$R_m=t|3u~NGh2FbyZ{st`hfhCTxG}-yKA% zq?7RR#EcvLkhUQc6FY^FZ45HBu%~L8^**&nwRRn?T>BdYKMv%1I=vs@;%#hMd&bY; z74ssA*4G_G*|phWs&nQSXkbufYCIdulf0G!a;E(09r29Q$7hA-D9a?#FRPprDa)Rc z7F{zRCCL2Y6o>aPXhM#>bS((Na1RZRxyHp6nfst-T}124={9RnrRn4A>C~2yZ)SCC zW7t0ZmV8_3ZR+VS*noGpEPDFnK{N*w5f2xgfm$SeS^2eofvkF8a^E%>&3tpjQ_oUm zwHrTC0+k#*I63#XZ>FLbvgu0Q=rZ2uiiQu%nIs`Me{F|yQn0Pk%|ll(YT*x%kexY_ zcm3k*XCiDSH(ytw7E$W^vtKJ0U>Tu7o-$ZFSX!tJu~gnbnZMbr-M)6-E@MgtuoJyo zwIZ?7BwLVirJA6T#XY|?n={hus`+Ige(i=}{~@*cQgkS{J#P%qy`YT>6Ty|+74c+f z=^buU_{G`1HWk>?*KSNf=IV{V{q_Fd(u{eAB)6v957o8Di@XDgl6_0rJtxRm<3xUF ziX_`O4j?iTKZ;vYD|)ou)-mu)e{?MU2TxzFxn|asL>ou>sx!!QnSoWJ8Zbx1(>GNlr zPJ}w!#46fMrldo8YsMc+ipc~3MPwLa2D5Ww2K$u@#zc#Y-qo0SOD1dEOu^41Z9|4- z8FI++mH`8->*LT9YBg2^cjDBCseTsoR7VWgucqa$PRu0azj^j3o<+5_r&dd(WZhG- z)fv=O*~k}-N23$Rw7t@z6$?z5EVP%DtCqmcCVWf%J%+%h`KXj{`cHBqA+_3KFG{Bx z=998`pBvIuz**!JZ@myQY!DfBDz(R<#eK)CTlBF@5H;?P7j1KRw)G&2$N$>Y-=7T> z+k2lz?35#h{Dk*JPy>A<7?c^c3^kdGPWYF2-E*ioeRf=be}o<9laU$Y7tNOgL%~O| z2gXADS97*o*=>Q3A`*4hv}t~~@j44v%iEuxE5eTBac)AyV_5%)X*$rc}PE<(D-3fmQm|4dg zO(39>aMpo;nY{GtanhWqzjMEgaNf{CkM)ZweZK8O!=r949dwKXr9nn8f$*dkukmj_ z(UB;hleF>{?j`1q<&HCXK;XYE#0Lx}wNbwRK(}GmE#b2gDG5f_fK@T z%obf=81bH>A(%Va7j1~;uPM|0K65vsU~3>%;8UJ!b-m+T1HbbA*BOxzx+E>Gl{cKQ zO;_l!{abXJx+pgasVGt`$W}nD&avxn|K5{@6WI_mTH5zgPC8zZ&Nd9(pAY4ax8i(F zS_&2aVBd0&JyLbOD%nqQ&TC||ktMk%DfB3zq(p@4!&Wnz>c3zy_dl?xp#rUA-tDaj zEjg&Ybhn-GLiA^-+NMFzSO-eB1K-;Qj%?F4y&pI?;}Tu?`1!8sclhyo%Vm+)y9v=k zHpf0`k+x_58z}d$+}ng6c<6<`T-xqsfXJh(f8^X?WOS(wXLJxty3&M}5s1%TAp0e(aB4)O zzj1_EPJx0XW+qC)wD1dkh0^>}N*xoiUvx9jA7oh2-8N89Yl5Di9w6$Xs_cOA>F;@Ld=Trj6Md4>FpQw49fwLd}B}YxfzAkf|oA>1D zv=lZX!7jd&YRjoF=VDT;5 zAnvCJOJ!`aBkxc-JICimMP{ff1d{=8k3jh}+vP5~gXZ4eH-k0Fdd_1BV=d?0)ztq_ zPCmB!f8}J0Zw@d|%P8Rg0!n^hMerk9=5whnVNjm5&n1^Xp=QGe%T@Z=;MRU_%+-vl zO9RxNiD7FZ;-$qfzd%UmDg%=j76_*^S@hABWfr3wgR5MQ{BratEVOMxcr`+qKUIq( zL1K~D`l?wj*u(a?0P~LQCTc3^GV4OK_#*O9{|$UhhJ=Y+gR&@kR7GrE6qK)TfRNNs z!)#auI@W5MegM^%M)Qo3R-!Fe)UZfv^BM2BI$07X638_w7ilItnVdU`>(JRYzi{}; za_$f$gshyqI*F{hLbOr@o5(fWqtvKNu4s)0+31kN26;&4Cf3^zwi zGMEr^hVpVfbSIt3;D5;(OEb?OaSB5+C&SAPPVXW;%3GP(f3%}(f)@)PGz~7)h+|_0 zE&`-hB27(o`~tbO8@w@d7-ufiP)j5OBHP|{zD6L8*z}9GsQ*{RIgtre=6pkx{ugAw zlmH}SQtn;1nJ-JV_^b|nYZb(vR(pA|tMX}}$NPK>|L4+#4k=^VtF5>f*B$W1*30F^zzzIjBW~nNm_k=bRPf>9 zms`fNgR2-q6|+Y>A;?h5*r1H=#Raz9x?CbF@Y7Y}(tTtvO8qHs8xFpBMi}vd2m3C+ z%iIqjK$hi2=PNSm%Lb}Jt#q8nAt;<(5>E8rP5`8Xc};duU3_>Dba*0=f%TZM5D#4c z8MNOM-`3^!2h;(t9{bg!LQY<8!ZuFs@LcR7&n5!F$$m$DIX5g9btks2#>XcoC)u^D zJTA1VzUioYIH;E%U8p{jJ`KcgADXlZfORiEvZtUdcc0qF?V!9nup#03s2ov(pxXE7 zX3bVJx2WJekBoGi9KQF6qnF0AE_{x%q%3#>&Yz|iKeQl=3ub8h^UMKwCHreoL6>~h zAnD^4Vxq44=p6WI_9uJe=Y14=t3l6{D7-CR47Uj zbXRM7JgrYs#rzbK*pil-sJTS7ejmN(KIfx*uUZ4rbJdo6Y2bK}nbWp~hc6UF5NJur z1##0GGS>6yZ{6NA`XsvugjR>UP{W$8W;uKooxPujk(*xA`@|t!6z7bPI9P}jH*_J( z2u?QYI5wYaG#BOR^zM;flf$KLa%jGc!ly$rs*w8^^=v;?6XGpXkXPSfp_McB<`#1o z^)4W|Lngd;O^uy9#Y~<(cSJE#)5)j5J)OXH^BHbkK;4EE${g?3^Kqk&n*J6P0jsKj z#PTriVc7k}vG>?@#;8AICE~0IdTa~NLa(JOppG;U!AkZ6cgxPNOijfW8Ayt4$?@vy zBAq_>!Lo$U+-^xD>MI;(D*jo@&)sJIiBy)8ezh*1&7R6Vj=5t;yf=cQvr3uV8{+mf z+7ceMk7f@7U^43o0s-<;u~o17YC zuoR1(=B}$PsSER73j0C311i($13kZNgLxA3m? z$1CrVhrGD@zD%IK{Hn0=X7p#n?wtV4U>y0*H~MmqOC6dM5Ae~=a7==yVO(tm`a&uS zueZYnzU2VdO{6K0%B`rs@bZ%dv#I5e_l|yUV@gRtVCNp8Rj(Hyp69qSI;PtRsA7|KQDi5^YzzBJWDgK`gIAk`ch+ljDLOaklN zOBcwozLp^aFxzER2~*@XyB#NxxjA5;re!Lhs4gVz$5d`*xjG?i$iHZkzNOhqYV-*i!_#{1PZ;MsR-+J(4$)+=+u|!K4U8t0 zH&qQ)Z?n46tmqETQ_6Ewk+`a|+~*|`yi#Rc`k2)q+_VbwX_3Mw^2cVj#_gQ$*?6BF zZyfH#*ewWE){4#q0!2IoDY-U=WSDa&T~?r(_PeolSdK;)TNyc2msV4OOJ^3OO<{|3 z(}E$A&iUhs>>00^&4Heg!k(wOLg2-Z)|Q4n{PJ~GC6$xV3o1tc7c)55Q3uvgSK^uC zFwXB(0>u`%ncbvLckYMn-CQqU-?P%c7tTdSYL z+wy^Xac~uk&)RWuKl?X@#kjGdWm{a+zDlJLwp0r%~xSkj{)}=^LaRrDS)A zmO49JLyt{*3#}2Goyl8ummb!=rw@fDV3GW*wIF4M*TbP_rF zgeHnKx^$(smYy3^%T4!ryXKX>ze&Ni2FsMAweuK3uew}(*s zwMy(W3IL!vFNP4-fkpl)B@wttC<0X?2*vMPOY;x}bWq6Y^@Hhq%_GM?I-sl5;YTFL zH;=!RUiSnTW%q)muBab`W|gwVvE{+?X7;@wlvUp%oHh2J4U9-pYP2K@p0aW&%x`5EN%(WUuJjq@lHIg< z=hQOxxA_iJQj|XaTq~gcXFovuiWO~0LG864*{Uk1+Mi2(61KZq!Z+Q0x5c@R5Xr&K zUN}}D0(HHmC`?TreWYf7pAISWx?(y5HgelZL%`949c0ogw6^INeJp9t@2gd4+HIFA zdXdF$hXIW^TF3I4KtNrU_Q~tWk>S$kVY7}ipTSrw=Tr>9O3yXtjV@5f*8r$<;OcH0 zRJThuh+7hmQViUlt(i>D{KP=c)qumlzX;x+WtzvgsQ8?FwxQrO*b=uQ^O{U7eT>>* zzX!w3*N*qhpp?4|H*0q>?X8)`;>28N7}3+`#8m-J^O$yPH=knI+a97bnA{VIuP}ge zfAV0xl$jy0&KkJ&$e^>~i2XTp_;Mvjv)>mb>3eQD2QtjcT`J-*^zZ$|PdmT5FFW9U zX+J+X6b9}o6%l0FhyPGa+*4VmzFAo9dK9tz68xC@eRS7CTg%0A7~C?AADdeMN3}Mm zhx~{;+1c#ehhwirW%%}rH%^9&Y44|T+CL7uHbc9wT4$K{6*9rYWm~T4WG2;}WL<~a zyl8@!Gt>;=X4l{O9HYD|($5rj6Wu}(60SpVrYF~b2u#(*a#yndNZFn07y__=kS=Tgy<0t_IN!=9~Ym+(Y|E{(pY^x7WWEe{TJw`12oy XA6<=Ge;1sH07L+t-yL@UUOD<7Xr~I$ literal 7016 zcmcJUcT^P3mcWr9DmiBZisa!VGei{z2@(Ve5*3D=a|Qt==O}5&AQ=P(W{8q=&S^+P z#sP*X$Sl6M@6+A4`_G=Uw@#hv>gu|E>-IVQyH!_J5eJtN>&{KtcVN@T`dhHE?qWT} zayD^t2AJ7fnQ6Qs!otTmrJEez)L*TOI|0@$+|yfFSU1zLu=X=-tQmSeJ|s`zv-Fei zuX{%*Z=x9~J{i4IU^bae0%t18TZOriv!fqN&YGdhdRb}(Ve9+lTzxJXeP5Xbh*WXq z{T~94aeDFOMuq(5j#Y6xPv)@qE2!mSp4L;!rS`M*;K`{iAYbA<$yh|jUCO`av)c7W z;aK#hcx+~*xS5){nYslPp~Fjivb0ro_`2U?A_k@s$icqXm6gh+o#LctryP0AE?~gFQFN>GvpzSiQ!@8h z-usH0?n`0B)Pip^E3NZFTHLtMQ)}n55@^@}Ag+ftOcu;S`7)U{;c2Eyt~)lyjG6`~ zwZ`~RfPEHrzd%fJ?}Zi}w2~t=5V>xX)A!z$jMJKG+ANWLhJi3u1)PUXhb>%+vq^sV zhFr&&0eR(`4+E#FH9Xf;)}Rs(|5rX24?0Vys@b_B}=W|Z zR?%eVWv>hP$zIZ1R1TQ7-z$YCf>*21bmW31UR=Yqb%G}IBS}V0i;#ZSR49w_pxS#y zlus2TqvYOO&of)eCFD^=Bv6P)klqy5I$dUTv?U}gXJc>0qyowA6oh=MnyT*t89~Hp zdbv1OujSn8Sox=FBIcZ}kmH^(D8v~qE}6OZ-80u3?pPE>?36N*MeU{L%_`ep7`MV@ zhLe9ot{$@GP(O?Mk*7PRHZNK=G$t7DVL6-!lyPpnF@@ z1w$9p$xbMYimT_V*R6EgPv(er)AiMv%3cy}r;pu%zPzvDBC>9UN@^JuAq3VQIi;qB zA?XmF(ZyCTO`4xM?GeF8`F2KYpVy0+NCs0>@8vkXf*`fue$vDT90zwDt{jrAwfnc9 zFH6ouM>GAbAjLWVtMbP|SN-2VL97kDHkVY938%b-mGx|z2)sbjxN?nlPVI}egy}(6_61Kkt?9tSu!bCMYBFGs?mswW(NX_ zI?P1k+ z2M6yk7BpRN^Mq+7^}PB)s(LGj2UW}NY$CpVOvEa|PYb(QUrMh~&=@uxrMMx^k4HJp z%>2B$508YKKQ112^L3+d`bS-5cFnZn2UB6~UqtiNnfSgCD+RO4Mk@tX?Ias!+{dWx z?d_3#_c`Uv#1u}Kd#?x^`k-|mNQhl-43CmT?};5uNxq=Rv?$g4efrbH#_8E<-Yr#A z=JY0}z^1-|TM?KW-2JC4$F@0MRWxES>TN#(lK;AU(;nO}oGv2%(cVPte*K5G=%-2$ z42E;Xm*T-656_H?uQ(Ikr*}L(W_#?tZ6=va5zaX(bJeP=B2x4Ik^bkSD-8a}%jEMv z=CpcIxNR;!dTFfI3jDCx6d8);j`>69eRgXEhLu=+uk^RESO=KU9Wxva;EEC1eC%-4_C@)c z^4^6RDP3lZgx@pX-ET<)eU8}e;b&%=pqGoKRqkQ03Q*$_VMD>@U1p2r+31-FmHCuI zS(si<*nmqx{XEoS?45NAQrxmfyvDVtHy{z%eX=njm-%8%&=^9m8ZkQ=!0R^4tVDmR z;MM4d&e%qs<^>Q3ARS+6xEJ+=5rQg0O=gu+xXAc^5LyrJyLkjW?`~oTqPy&K8w>?q z!LG3~U8(}ioY^Aqjx2!-Z~9J(bE)cj+7YlMwMx7iXtLe8Fhzfhqn%n44eK_?!2O%-(8deBMDH`iBR#-yC`FXYeT#{j8VujR z1M^tpe+A~S;6H&$SNQ)1OxXugDv)q6HI;*3Uc-Dmot2KP4ZQZqqDfCzMO6QO=CFfX z(`zAp*qfKZuveC<;S%i46uC(CUVG{)VCl!fYQhR!wld)u~# zvD7jg;dW2%os$PSB}Q=z!bjlCcbXY`!>vQ<4v8vogB8E6=y|%o_jI@L6!9CSIdDL_ zWFjhddvWCE19wGj8S}O)i8h-rz{^Fc9kxRuKko8Q&hEcG5-eNs+z)aYDldwf$_;WJ zQWjU46k%> zHEc1)n*lX7-;cbvcDufY__3Z+0NY#&z4FjrOf4vJS?}N;7{+ti?)KCVf=kWHY=ipf zZM@mBt6)9Ix98m(bh9IU&4GP~Swo)*ChV}yq{8czZ&c<5taiIRn{xhv+KdImPC z4R3&I9|#>I3=Y|hH|QD;Lr*mw>e?j;n5fms;AvyQg6rcZIWktkk6%PMk8-lrD(vuetP=tq#F)8R!OVROzEWrN z^1=>obwVLi)?R)TjiM}G`T5ZP1l*N#N^yVp&ngeUpD?)+fqW!(HXkUP=Pp;Va+4$~ z02O+YIN=6bL^U-n`mg4xOJ!$#(33?~yms9DCrh;hykm?QdR9arKhT+C<(naIa@*}> zTezJ91rz8g=c2#xl|7tm%9}!-U4Gn6B#G~ZvZ8b!oWL&^WIbGBl|!%QwuVysv>dGl z?5505sQhXKt9*KKcUGf7Mv>3Cb|6uaL5^*+(?w@0G`?lXlumzx+FVkLV53hj;Tw(> z{A-`-<}Pqx-!;-z^e^2of?X)@xIv8Hgm9)v3CRS*($szBW&MuE(VpA?TNBQA<@qoe?|dO)G~p z8$T!j!=t%TWh+6v4Bu{V;t$@o!A*VY_KKpwA6_{bV!%HlM0hgzbb2U0!ubvw-B!oqjEY%dNVls~x*$dt|NO^_7;S7Ye`H?2PH+(T|I?YBQ!q zkthm`m}nl(HHw+O>J9s(&tAAzP*SDX^{YeBRGUpgMPZJHFY09``Wi`7(dc?QnP%vQ zIoo?XS!O!;S!2>BL2L0>6m`wM;!E=f>#^LU$9MN=_Yx76Bxj-%6^|^g5I|Ly1-~6h zp#z&MyuW#lHP=Isu^1Q}z#~ycD3@P5e^heO*uy{82Vqp#r|RrGo|*aa;|CjY`N*u5 zvp*d-`V((<8(4bfu!=<3AUb)i3|-oT+pp)a_K7(1RJEbDdgwb{TitbyuRi(kI-P+h z%wV%A*LgxtTzWIW(K8`;-b|r52(jw)a&}g`?`p34 z<99%}(Zi~8y0%SolONwa^7>z%d+ryKte;`<<35W08Iq0grPZYajJCE`%J^%`@T^q6 z5CGw963Lpv_r$3(qPYNNR3O)8y_~nUMMBIzrVm~e))4msA4pGsjvQaQm~~31Y2m1) zmCt84sn9_2|0C8scelS!bOskc z9S_c-6OAD0$Q<@GjjsV$4EJh<1h}jfYpne+2n#Z78X8zvE$Ct=NClaI!jH{>Zgp6JnRfKcG5ynrv9$CVem<-LSyJBf))W~*N0;M6U`;J&39LTpEeo7c) zO``mb2Q_Lo0s%OKeH=-BJ23O|AwRnVb`!*v4B7|H4WFkz8>~aAYmkKx)hOw+?d6a( zY@vK8g4DSwp{Etf5jj;KVGK&m#Dq{@$%jw zA$f-d{v(pp#C)7SevGR96 zXj#*<=Ylod{(g#qVy3PQkAHSH^NW~^**0|POPk&Hf-={>1)3BE!~s~i-+fU622=60 zr>9M8E`&30=Dm&SBCHiK);LmdvM~THaVo`jyf$+1ok%ltTAE7_Inqxhjy1Ei95rJ1 z5ao)MDH5Qf=urbLjC{QOSz842aE$HP?nU!%EO=N|^H~$KYN}{MUd@(9)d#7~kndc9 z_j83zy5&DXc`xY{p*5Oo~Qcbi z`A~kc%;!p)rZYg}B>HldoL?kNSVm`mXFP-j0fvvg`Dz9ytbf8(yzbL7Oum(v`qIDm zDYzqY(_n{^C9-@Rw2+!d+TmEA@^qwkZSB$eVk@p<{yRXjfqn)cu>kMAK-!?evBG!? z3tFX4*!d+5MRkuGJ0+oEv%vlIF~)#%T1)Q?)#if3s;8on(eVWpxwi#r5NWb7sC^!P_}aIXh)*Tq0zEiz2*H-%oF*~HIq5HNK0=kj)b`1%H42!$)$xNMgmjCS|cPVZ?nb*RK!ZK9M`&yzy8;$;(x3eBT4nQR;wl zxrLhaY-$pHinv_dpV?R%-jM9pY+O9^6`B4DoD6mtL}5+dYy#vRJ$U(8YZIK?j!X~M zVmk>_it|}mm|BaIwZD6mEqzE>BG;ev;t1f>3=kcjQ|TFwga;{;oGzqP3D%C`>ekjU zFRlxz38C(Ems&-HEUjrS>=D4l3suB~emVvU>w-K~x`+Ja&e{?_y{6(t-O(E2)q`D2 zj{S3&AZLezM@c5%iV6MD&o79aVXWzmcmm$P>~N)v{g`Shy_0m6#`F?d*DXXczmApp z(|V}O%f3OpM{SAhqMeUK;qcskX8UOpEFZq{;+I$(ITyq5|P|6#w`yj{G zJ81-2#VEne;f(aPOB_E$ z=8OtCflv1tRo!^C%NCua|Rx!VoC#BS{~2PGY~ZBjq?zZsgI60^}{ zb%TStg+O-Xqqa6CVqr@^sF^HBFN^`h%UQJgbod zqg`l()oFm1);55ZS}P~C-j84O5syxjgjs{RhJ|PuKu04ltu#BH=N{5%sK)d}^2`<1 zBvrcSh+Yul%!uB%&PVe$5TtwfIE@qN%?sDXCWTCo6AP%7+br6%fxVhYQYy~4)7nox zTcn+8Ha!uI8hf25gXfxi$R;@5xao1JjTxzs%WdujW+4<|Tf=w4RK!Ma#y7h8p@-Jr z33Zz1YxfCV-RrzsGW>h72E_vB%g3Q&A3hC6mYX=$jBE`Ad1Sfr3Pwz-LoeTNg;CF? zu;|Q?7@H-TU;5ILc!Qn1Y`a{gA}TGUBB;3*y`>@!aArx?{B>Wy#YjC{miqni-7_oS zx;m5VYUTFJbnD{ZS?$ct^j%vV*7M-@6Gv+^`uSck@UWKubU)qO3P&`abg7sQi^3fk zS+CukN&8Z>09P3OjlRNbTE!Bz;s%j(vrKoq1`Qk8y4TH+Filu$etcpH=7>auvsia< zuCCAT$PNLuXFzHQ?_uzNTM;c#`S)p#K)WNRMe4`gF3P)Dd^ZIqvI{(G3Qm#0!upVb zb@|8XO>6GWS&phA{+*rk&Wb>~U)r#-RR71!&ce+YlRVIdu?S{znVhFgam4{sl${vyR=lztJA^(rQc z{n?&>!AG)3Lz(HBCCS>VG4aL8X`j<^D8VM5&wL>i7!%Z)SF_acb-ZMLn26Wtts8Uk z?~tTi+3VjJ_{&ID#aq~vIKNXK-`DWJB&z Date: Mon, 16 Dec 2024 17:31:07 -0500 Subject: [PATCH 7/7] Silence download progress --- batch/scripts/create_vector_schema.sh | 2 +- batch/scripts/load_vector_data.sh | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/batch/scripts/create_vector_schema.sh b/batch/scripts/create_vector_schema.sh index 3a5152f3..31f33d63 100755 --- a/batch/scripts/create_vector_schema.sh +++ b/batch/scripts/create_vector_schema.sh @@ -23,7 +23,7 @@ set -u # TODO: Downloading the whole file for this step might be unnecessary # See https://gfw.atlassian.net/browse/GTC-2233 echo "AWSCLI: COPY DATA FROM $SRC TO $LOCAL_FILE" -aws s3 cp "$SRC" "$LOCAL_FILE" +aws s3 cp "$SRC" "$LOCAL_FILE" --no-progress # use virtual GDAL vsizip wrapper for ZIP files # TODO: [GTC-661] Allow for a more flexible file structure inside the ZIP file diff --git a/batch/scripts/load_vector_data.sh b/batch/scripts/load_vector_data.sh index 4e89fd17..3b81be23 100755 --- a/batch/scripts/load_vector_data.sh +++ b/batch/scripts/load_vector_data.sh @@ -20,7 +20,7 @@ ME=$(basename "$0") set -u echo "AWSCLI: COPY DATA FROM $SRC TO $LOCAL_FILE" -aws s3 cp "$SRC" "$LOCAL_FILE" +aws s3 cp "$SRC" "$LOCAL_FILE" --no-progress # use virtual GDAL vsizip wrapper for ZIP files # TODO: [GTC-661] Allow for a more flexible file structure inside the ZIP file