From 93ad1cd2b91147d0be7e55cd9ce9872c90fbd16e Mon Sep 17 00:00:00 2001 From: Quincy Morgan <2046746+quincylvania@users.noreply.github.com> Date: Tue, 21 May 2024 18:46:05 -0400 Subject: [PATCH 01/13] Add linting action --- renderer/.github/workflows/lint.yml | 28 ++++++++++++++++++++++++++++ 1 file changed, 28 insertions(+) create mode 100644 renderer/.github/workflows/lint.yml diff --git a/renderer/.github/workflows/lint.yml b/renderer/.github/workflows/lint.yml new file mode 100644 index 0000000..3eb7dce --- /dev/null +++ b/renderer/.github/workflows/lint.yml @@ -0,0 +1,28 @@ +--- +name: Lint +on: [ push, pull_request ] +permissions: { } +jobs: + build: + name: Lint + runs-on: ubuntu-latest + + permissions: + contents: read + packages: read + # To report GitHub Actions status checks + statuses: write + + steps: + - name: Checkout code + uses: actions/checkout@v4 + with: + # super-linter needs the full git history to get the + # list of files that changed across commits + fetch-depth: 0 + + - name: Super-linter + uses: super-linter/super-linter@v6.5.0 # x-release-please-version + env: + # To report GitHub Actions status checks + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} \ No newline at end of file From cf7d8ff327bbf7b0ea9daeeab39278f01562a485 Mon Sep 17 00:00:00 2001 From: Quincy Morgan <2046746+quincylvania@users.noreply.github.com> Date: Tue, 21 May 2024 18:47:46 -0400 Subject: [PATCH 02/13] Fix file location --- {renderer/.github => .github}/workflows/lint.yml | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename {renderer/.github => .github}/workflows/lint.yml (100%) diff --git a/renderer/.github/workflows/lint.yml b/.github/workflows/lint.yml similarity index 100% rename from renderer/.github/workflows/lint.yml rename to .github/workflows/lint.yml From 93afc15495822156ccad0ed827939ab11bd6e7be Mon Sep 17 00:00:00 2001 From: Quincy Morgan <2046746+quincylvania@users.noreply.github.com> Date: Tue, 21 May 2024 19:06:31 -0400 Subject: [PATCH 03/13] Fix some yml warnings caught by the linter --- .github/workflows/lint.yml | 4 +++- renderer/layers/trails.yml | 2 +- renderer/layers/water_trails.yml | 2 +- renderer/layers/water_trails_poi.yml | 2 +- 4 files changed, 6 insertions(+), 4 deletions(-) diff --git a/.github/workflows/lint.yml b/.github/workflows/lint.yml index 3eb7dce..0d8d31f 100644 --- a/.github/workflows/lint.yml +++ b/.github/workflows/lint.yml @@ -1,6 +1,8 @@ --- name: Lint -on: [ push, pull_request ] +on: + - push + - pull_request permissions: { } jobs: build: diff --git a/renderer/layers/trails.yml b/renderer/layers/trails.yml index aa4b7e1..9426159 100644 --- a/renderer/layers/trails.yml +++ b/renderer/layers/trails.yml @@ -1,3 +1,4 @@ +--- schema_name: OSM US Trails schema_description: A map of trails attribution: © OpenStreetMap contributors @@ -296,4 +297,3 @@ args: area: description: Geofabrik area to download default: rhode-island - diff --git a/renderer/layers/water_trails.yml b/renderer/layers/water_trails.yml index e539762..369476e 100644 --- a/renderer/layers/water_trails.yml +++ b/renderer/layers/water_trails.yml @@ -1,3 +1,4 @@ +--- schema_name: OSM US Water Trails schema_description: Canoe and kayak waterways and portages. attribution: © OpenStreetMap contributors @@ -229,4 +230,3 @@ args: area: description: Geofabrik area to download default: rhode-island - diff --git a/renderer/layers/water_trails_poi.yml b/renderer/layers/water_trails_poi.yml index 84801fb..f9f7bf9 100644 --- a/renderer/layers/water_trails_poi.yml +++ b/renderer/layers/water_trails_poi.yml @@ -1,3 +1,4 @@ +--- schema_name: OSM US Water Trail POIs schema_description: Slipways, access points, dams, weirs, and waterfalls. attribution: © OpenStreetMap contributors @@ -100,4 +101,3 @@ args: area: description: Geofabrik area to download default: rhode-island - From d03fa36d716dfe40018d577244b3528702399f04 Mon Sep 17 00:00:00 2001 From: Quincy Morgan <2046746+quincylvania@users.noreply.github.com> Date: Tue, 21 May 2024 19:06:50 -0400 Subject: [PATCH 04/13] Fix some warnings caught by the linter --- README.md | 2 +- renderer/_example.sh | 2 +- renderer/render.sh | 2 +- renderer/render_seashells.sh | 4 ++-- 4 files changed, 5 insertions(+), 5 deletions(-) diff --git a/README.md b/README.md index 3822a26..7b0916c 100644 --- a/README.md +++ b/README.md @@ -1,2 +1,2 @@ # osmus/tileservice -Central repo for the OSM US vector tile service +Central repository for the OSM US vector tile service diff --git a/renderer/_example.sh b/renderer/_example.sh index e0c4751..d54e224 100644 --- a/renderer/_example.sh +++ b/renderer/_example.sh @@ -19,7 +19,7 @@ else touch "${LOCKFILE}" # Ensure the lock file is removed when we exit and when we receive signals - trap "rm -f ${LOCKFILE}; trap - SIGTERM && kill -- -$$" SIGINT SIGTERM EXIT + trap "rm -f '${LOCKFILE}'; trap - SIGTERM && kill -- -'$$'" SIGINT SIGTERM EXIT fi # The local path to the tileservice repo diff --git a/renderer/render.sh b/renderer/render.sh index 0ddd662..55e5c39 100755 --- a/renderer/render.sh +++ b/renderer/render.sh @@ -3,4 +3,4 @@ # Get the directory of the current script DIR="$(dirname "$0")" -"$DIR/render_seashells.sh" $1 \ No newline at end of file +"$DIR/render_seashells.sh" "$1" \ No newline at end of file diff --git a/renderer/render_seashells.sh b/renderer/render_seashells.sh index 0e8b3be..054b5d9 100755 --- a/renderer/render_seashells.sh +++ b/renderer/render_seashells.sh @@ -23,7 +23,7 @@ tail -f "$LOG_FILE" | nc seashells.io 1337 > /tmp/seashells_render & sleep 10 OSM_PLANET_SIZE=$(stat -c%s "$WORKING_DIR/data/sources/planet.osm.pbf") # Print the size with comma separators -OSM_PLANET_SIZE=$(printf "%'d" $OSM_PLANET_SIZE) +OSM_PLANET_SIZE=$(printf "%'d" "$OSM_PLANET_SIZE") RSS_FILE="$WORKING_DIR/rss.xml" PLANET="$WORKING_DIR/data/planet.pmtiles" @@ -61,7 +61,7 @@ run() { PMTILES_PLANET_SIZE=$(stat -c%s "$PLANET") # Print the size with comma separators - PMTILES_PLANET_SIZE=$(printf "%'d" $PMTILES_PLANET_SIZE) + PMTILES_PLANET_SIZE=$(printf "%'d" "$PMTILES_PLANET_SIZE") "$DIR/rss_update.sh" "$RSS_FILE" "Build Complete" "Tiles are up to date as of ${TIMESTAMP}Z. Render took ${HOURS} ${HOUR_TEXT} and ${MINUTES} ${MINUTE_TEXT}. The planet PMTiles file is ${PMTILES_PLANET_SIZE} bytes." else From 860e453040bad2a720f73561cdf9f2495f82a9dc Mon Sep 17 00:00:00 2001 From: Quincy Morgan <2046746+quincylvania@users.noreply.github.com> Date: Tue, 21 May 2024 19:09:11 -0400 Subject: [PATCH 05/13] Make _example.sh executable --- renderer/_example.sh | 0 1 file changed, 0 insertions(+), 0 deletions(-) mode change 100644 => 100755 renderer/_example.sh diff --git a/renderer/_example.sh b/renderer/_example.sh old mode 100644 new mode 100755 From 3be7ff6bc1b0089056d8a3f912aaa65ac242cf42 Mon Sep 17 00:00:00 2001 From: Quincy Morgan <2046746+quincylvania@users.noreply.github.com> Date: Tue, 21 May 2024 19:17:49 -0400 Subject: [PATCH 06/13] Fix some lint warnings --- renderer/_example.sh | 2 +- renderer/render_seashells.sh | 5 ++--- 2 files changed, 3 insertions(+), 4 deletions(-) diff --git a/renderer/_example.sh b/renderer/_example.sh index d54e224..08e5c68 100755 --- a/renderer/_example.sh +++ b/renderer/_example.sh @@ -19,7 +19,7 @@ else touch "${LOCKFILE}" # Ensure the lock file is removed when we exit and when we receive signals - trap "rm -f '${LOCKFILE}'; trap - SIGTERM && kill -- -'$$'" SIGINT SIGTERM EXIT + trap 'rm -f ${LOCKFILE}; trap - SIGTERM && kill -- -$$' SIGINT SIGTERM EXIT fi # The local path to the tileservice repo diff --git a/renderer/render_seashells.sh b/renderer/render_seashells.sh index 054b5d9..b42e808 100755 --- a/renderer/render_seashells.sh +++ b/renderer/render_seashells.sh @@ -33,10 +33,9 @@ PLANET="$WORKING_DIR/data/planet.pmtiles" run() { TIMESTAMP=$(date +"%Y-%m-%d %H:%M:%S") START_TIME=$(date +%s) - "$DIR/render_once.sh" "$WORKING_DIR" - # Check the exit status of render_once.sh - if [ $? -eq 0 ]; then + if "$DIR/render_once.sh" "$WORKING_DIR" + then # Record the end time END_TIME=$(date +%s) From c268a6502ef48ebb708d4dc2461336283f36e78a Mon Sep 17 00:00:00 2001 From: Quincy Morgan <2046746+quincylvania@users.noreply.github.com> Date: Tue, 21 May 2024 19:28:30 -0400 Subject: [PATCH 07/13] Turn off nitpicky linters --- .github/workflows/lint.yml | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/.github/workflows/lint.yml b/.github/workflows/lint.yml index 0d8d31f..df40474 100644 --- a/.github/workflows/lint.yml +++ b/.github/workflows/lint.yml @@ -27,4 +27,6 @@ jobs: uses: super-linter/super-linter@v6.5.0 # x-release-please-version env: # To report GitHub Actions status checks - GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} \ No newline at end of file + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + VALIDATE_JSCPD: false + VALIDATE_NATURAL_LANGUAGE: false \ No newline at end of file From 16bcb1f18abee619c31890e1355104b41f54cbdb Mon Sep 17 00:00:00 2001 From: Quincy Morgan <2046746+quincylvania@users.noreply.github.com> Date: Tue, 21 May 2024 19:36:08 -0400 Subject: [PATCH 08/13] Config yml --- .github/linters/.yaml-lint.yml | 6 ++++++ 1 file changed, 6 insertions(+) create mode 100644 .github/linters/.yaml-lint.yml diff --git a/.github/linters/.yaml-lint.yml b/.github/linters/.yaml-lint.yml new file mode 100644 index 0000000..3cc4c2a --- /dev/null +++ b/.github/linters/.yaml-lint.yml @@ -0,0 +1,6 @@ +--- +extends: default +rules: + line-length: disable + indentation: + spaces: 2 \ No newline at end of file From ccf35751ded545195920cd12813259075f38eccc Mon Sep 17 00:00:00 2001 From: Quincy Morgan <2046746+quincylvania@users.noreply.github.com> Date: Tue, 21 May 2024 19:50:55 -0400 Subject: [PATCH 09/13] Config yml linter --- .github/linters/.yaml-lint.yml | 9 +++++++-- .github/workflows/lint.yml | 3 +-- renderer/layers/water_trails_poi.yml | 2 +- 3 files changed, 9 insertions(+), 5 deletions(-) diff --git a/.github/linters/.yaml-lint.yml b/.github/linters/.yaml-lint.yml index 3cc4c2a..a042aba 100644 --- a/.github/linters/.yaml-lint.yml +++ b/.github/linters/.yaml-lint.yml @@ -1,6 +1,11 @@ --- extends: default rules: - line-length: disable indentation: - spaces: 2 \ No newline at end of file + spaces: 2 + line-length: disable + new-line-at-end-of-file: + level: warning + trailing-spaces: + level: warning + truthy: disable diff --git a/.github/workflows/lint.yml b/.github/workflows/lint.yml index df40474..1ae8f4e 100644 --- a/.github/workflows/lint.yml +++ b/.github/workflows/lint.yml @@ -3,7 +3,6 @@ name: Lint on: - push - pull_request -permissions: { } jobs: build: name: Lint @@ -29,4 +28,4 @@ jobs: # To report GitHub Actions status checks GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} VALIDATE_JSCPD: false - VALIDATE_NATURAL_LANGUAGE: false \ No newline at end of file + VALIDATE_NATURAL_LANGUAGE: false diff --git a/renderer/layers/water_trails_poi.yml b/renderer/layers/water_trails_poi.yml index f9f7bf9..715a97a 100644 --- a/renderer/layers/water_trails_poi.yml +++ b/renderer/layers/water_trails_poi.yml @@ -1,6 +1,6 @@ --- schema_name: OSM US Water Trail POIs -schema_description: Slipways, access points, dams, weirs, and waterfalls. +schema_description: Slipways, access points, dams, weirs, and waterfalls. attribution: © OpenStreetMap contributors sources: osm: From 570f36b839384b1c35988b04a20a2c5ac064123d Mon Sep 17 00:00:00 2001 From: Quincy Morgan <2046746+quincylvania@users.noreply.github.com> Date: Tue, 21 May 2024 19:55:43 -0400 Subject: [PATCH 10/13] Run shfmt --- renderer/_example.sh | 14 +++---- renderer/render.sh | 2 +- renderer/render_once.sh | 79 ++++++++++++++++++----------------- renderer/render_seashells.sh | 80 ++++++++++++++++++------------------ renderer/rss_update.sh | 42 +++++++++---------- 5 files changed, 110 insertions(+), 107 deletions(-) diff --git a/renderer/_example.sh b/renderer/_example.sh index 08e5c68..246d99e 100755 --- a/renderer/_example.sh +++ b/renderer/_example.sh @@ -12,14 +12,14 @@ LOCKFILE="/tmp/planet-render.lock" # Check if lock file exists if [ -e "${LOCKFILE}" ]; then - echo "A rendering process is already running." - exit 1 + echo "A rendering process is already running." + exit 1 else - # Create a lock file - touch "${LOCKFILE}" + # Create a lock file + touch "${LOCKFILE}" - # Ensure the lock file is removed when we exit and when we receive signals - trap 'rm -f ${LOCKFILE}; trap - SIGTERM && kill -- -$$' SIGINT SIGTERM EXIT + # Ensure the lock file is removed when we exit and when we receive signals + trap 'rm -f ${LOCKFILE}; trap - SIGTERM && kill -- -$$' SIGINT SIGTERM EXIT fi # The local path to the tileservice repo @@ -32,4 +32,4 @@ mkdir "$REPO_DIR" git clone https://github.com/osmus/tileservice.git "$REPO_DIR" # Start the render, assuming the directory format hasn't changed -"$REPO_DIR/renderer/render.sh" \ No newline at end of file +"$REPO_DIR/renderer/render.sh" diff --git a/renderer/render.sh b/renderer/render.sh index 55e5c39..e74d2a7 100755 --- a/renderer/render.sh +++ b/renderer/render.sh @@ -3,4 +3,4 @@ # Get the directory of the current script DIR="$(dirname "$0")" -"$DIR/render_seashells.sh" "$1" \ No newline at end of file +"$DIR/render_seashells.sh" "$1" diff --git a/renderer/render_once.sh b/renderer/render_once.sh index 9758449..875bc3b 100755 --- a/renderer/render_once.sh +++ b/renderer/render_once.sh @@ -26,10 +26,10 @@ docker system prune --force docker pull ghcr.io/onthegomap/planetiler:latest docker run -e JAVA_TOOL_OPTIONS='-Xmx2g' \ - -v "$WORKING_DIR/data":/data \ - -v "$DIR/layers":/layers \ - ghcr.io/onthegomap/planetiler:latest --area=planet \ - --download --download-only --only-fetch-wikidata + -v "$WORKING_DIR/data":/data \ + -v "$DIR/layers":/layers \ + ghcr.io/onthegomap/planetiler:latest --area=planet \ + --download --download-only --only-fetch-wikidata # Remove default downloaded OSM file rm -rf "$WORKING_DIR/data/sources/monaco.osm.pbf" @@ -37,25 +37,25 @@ rm -rf "$WORKING_DIR/data/sources/monaco.osm.pbf" PLANET="$WORKING_DIR/data/planet.pmtiles" docker run -e JAVA_TOOL_OPTIONS='-Xmx150g' \ - -v "$WORKING_DIR/data":/data \ - -v "$DIR/layers":/layers \ - ghcr.io/onthegomap/planetiler:latest --area=planet --bounds=world \ - --output="/data/planet.pmtiles" \ - --transportation_name_size_for_shield \ - --transportation_name_limit_merge \ - --boundary-osm-only \ - --storage=ram --nodemap-type=array \ - --max-point-buffer=4 \ - --building_merge_z13=false \ - --languages=ab,ace,af,als,am,an,ar,arz,as,ast,az,az-Arab,az-cyr,azb,ba,bar,bat-smg,be,be-tarask,ber,bg,bm,bn,bo,bpy,br,bs,bxr,ca,cdo,ce,ceb,cho,chr,chy,ckb,co,cr,crh,crh-cyr,crk,cs,csb,cv,cy,da,dak,de,dsb,dv,dz,ee,egl,el,en,eo,es,et,eu,fa,fi,fil,fit,fo,fr,frr,full,fur,fy,ga,gag,gan,gcf,gd,gl,gn,gr,grc,gsw,gu,gv,ha,hak,hak-HJ,haw,he,hi,hif,hr,hsb,ht,hu,hur,hy,ia,id,ie,ilo,int,io,is,it,iu,ja,ja_kana,ja_rm,ja-Hira,ja-Latn,jv,ka,kab,kbd,ki,kk,kk-Arab,kl,km,kn,ko,ko-Hani,ko-Latn,krc,krl,ks,ku,kv,kw,ky,la,lb,left,lez,li,lij,lld,lmo,ln,lo,lrc,lt,lv,lzh,md,mdf,mez,mg,mhr,mi,mia,mk,ml,mn,mo,moh,mr,mrj,ms,ms-Arab,mt,mwl,my,myv,mzn,nah,nan,nan-HJ,nan-POJ,nan-TL,nds,ne,nl,nn,no,nov,nv,oc,oj,old,or,os,ota,pa,pam,pcd,pfl,pl,pms,pnb,pot,ps,pt,pt-BR,pt-PT,qu,right,rm,ro,ru,rue,rw,sah,sat,sc,scn,sco,sd,se,sh,si,sju,sk,sl,sma,smj,so,sq,sr,sr-Latn,su,sv,sw,syc,szl,ta,te,TEC,tg,th,th-Latn,ti,tk,tl,tr,tt,tt-lat,udm,ug,uk,ur,uz,uz-Arab,uz-cyr,uz-Cyrl,uz-Latn,vec,vi,vls,vo,wa,war,win,wiy,wo,wuu,xmf,yi,yo,yue,yue-Hant,yue-Latn,za,zgh,zh,zh_pinyin,zh_zhuyin,zh-Hans,zh-Hant,zh-Latn-pinyin,zu,zza + -v "$WORKING_DIR/data":/data \ + -v "$DIR/layers":/layers \ + ghcr.io/onthegomap/planetiler:latest --area=planet --bounds=world \ + --output="/data/planet.pmtiles" \ + --transportation_name_size_for_shield \ + --transportation_name_limit_merge \ + --boundary-osm-only \ + --storage=ram --nodemap-type=array \ + --max-point-buffer=4 \ + --building_merge_z13=false \ + --languages=ab,ace,af,als,am,an,ar,arz,as,ast,az,az-Arab,az-cyr,azb,ba,bar,bat-smg,be,be-tarask,ber,bg,bm,bn,bo,bpy,br,bs,bxr,ca,cdo,ce,ceb,cho,chr,chy,ckb,co,cr,crh,crh-cyr,crk,cs,csb,cv,cy,da,dak,de,dsb,dv,dz,ee,egl,el,en,eo,es,et,eu,fa,fi,fil,fit,fo,fr,frr,full,fur,fy,ga,gag,gan,gcf,gd,gl,gn,gr,grc,gsw,gu,gv,ha,hak,hak-HJ,haw,he,hi,hif,hr,hsb,ht,hu,hur,hy,ia,id,ie,ilo,int,io,is,it,iu,ja,ja_kana,ja_rm,ja-Hira,ja-Latn,jv,ka,kab,kbd,ki,kk,kk-Arab,kl,km,kn,ko,ko-Hani,ko-Latn,krc,krl,ks,ku,kv,kw,ky,la,lb,left,lez,li,lij,lld,lmo,ln,lo,lrc,lt,lv,lzh,md,mdf,mez,mg,mhr,mi,mia,mk,ml,mn,mo,moh,mr,mrj,ms,ms-Arab,mt,mwl,my,myv,mzn,nah,nan,nan-HJ,nan-POJ,nan-TL,nds,ne,nl,nn,no,nov,nv,oc,oj,old,or,os,ota,pa,pam,pcd,pfl,pl,pms,pnb,pot,ps,pt,pt-BR,pt-PT,qu,right,rm,ro,ru,rue,rw,sah,sat,sc,scn,sco,sd,se,sh,si,sju,sk,sl,sma,smj,so,sq,sr,sr-Latn,su,sv,sw,syc,szl,ta,te,TEC,tg,th,th-Latn,ti,tk,tl,tr,tt,tt-lat,udm,ug,uk,ur,uz,uz-Arab,uz-cyr,uz-Cyrl,uz-Latn,vec,vi,vls,vo,wa,war,win,wiy,wo,wuu,xmf,yi,yo,yue,yue-Hant,yue-Latn,za,zgh,zh,zh_pinyin,zh_zhuyin,zh-Hans,zh-Hant,zh-Latn-pinyin,zu,zza # Check if the file exists and is at least 50GB if [[ ! -f "$PLANET" ]]; then - echo "Error: File $PLANET does not exist." - exit 1 -elif [[ $(stat -c %s "$PLANET") -lt $((50*1024*1024*1024)) ]]; then - echo "Error: File $PLANET is smaller than 50GB." - exit 1 + echo "Error: File $PLANET does not exist." + exit 1 +elif [[ $(stat -c %s "$PLANET") -lt $((50 * 1024 * 1024 * 1024)) ]]; then + echo "Error: File $PLANET is smaller than 50GB." + exit 1 fi echo 'Uploading planet to s3 bucket in background' @@ -63,23 +63,26 @@ aws s3 cp "$PLANET" s3://planet-pmtiles/ --only-show-errors & # Render optional layers for file in "$DIR/layers/"*.yml; do - # Get the base name of the file without the .yml extension - layer_name=$(basename "$file" .yml) - - echo "Processing layer: $layer_name" - - docker run -e JAVA_TOOL_OPTIONS='-Xmx24g' \ - -v "$WORKING_DIR/data":/data \ - -v "$DIR/layers":/layers \ - ghcr.io/onthegomap/planetiler:latest generate-custom \ - --area=planet --bounds=world \ - --output="/data/$layer_name.pmtiles" \ - --schema="/layers/$layer_name.yml" \ - --storage=mmap --nodemap-type=array \ - --max-point-buffer=4 - - echo "Uploading $layer_name to s3 bucket in background" - { aws s3 cp "$WORKING_DIR/data/$layer_name.pmtiles" s3://planet-pmtiles/ --only-show-errors; rm -rf "$WORKING_DIR/data/$layer_name.pmtiles"; } & + # Get the base name of the file without the .yml extension + layer_name=$(basename "$file" .yml) + + echo "Processing layer: $layer_name" + + docker run -e JAVA_TOOL_OPTIONS='-Xmx24g' \ + -v "$WORKING_DIR/data":/data \ + -v "$DIR/layers":/layers \ + ghcr.io/onthegomap/planetiler:latest generate-custom \ + --area=planet --bounds=world \ + --output="/data/$layer_name.pmtiles" \ + --schema="/layers/$layer_name.yml" \ + --storage=mmap --nodemap-type=array \ + --max-point-buffer=4 + + echo "Uploading $layer_name to s3 bucket in background" + { + aws s3 cp "$WORKING_DIR/data/$layer_name.pmtiles" s3://planet-pmtiles/ --only-show-errors + rm -rf "$WORKING_DIR/data/$layer_name.pmtiles" + } & done echo 'Waiting for all background jobs to finish' @@ -89,4 +92,4 @@ echo 'Invalidating the CDN cache' aws cloudfront create-invalidation --distribution-id E1E7N0LWX2WY4E --invalidation-batch "{\"Paths\": {\"Quantity\": 1, \"Items\": [\"/*\"]}, \"CallerReference\": \"invalidation-$DATE\"}" echo 'Render Complete' -date -u '+%Y-%m-%d %H:%M:%S' \ No newline at end of file +date -u '+%Y-%m-%d %H:%M:%S' diff --git a/renderer/render_seashells.sh b/renderer/render_seashells.sh index b42e808..fe3bbf7 100755 --- a/renderer/render_seashells.sh +++ b/renderer/render_seashells.sh @@ -17,7 +17,8 @@ LOG_TIMESTAMP=$(date +%Y%m%d%H%M%S) LOG_FILE="/var/log/render/logs_$LOG_TIMESTAMP.txt" touch "$LOG_FILE" -tail -f "$LOG_FILE" | nc seashells.io 1337 > /tmp/seashells_render & sleep 10 +tail -f "$LOG_FILE" | nc seashells.io 1337 >/tmp/seashells_render & +sleep 10 # Get the size of the file in bytes OSM_PLANET_SIZE=$(stat -c%s "$WORKING_DIR/data/sources/planet.osm.pbf") @@ -31,44 +32,43 @@ PLANET="$WORKING_DIR/data/planet.pmtiles" "$DIR/rss_update.sh" "$RSS_FILE" "Build Started." "The OSM planet file is ${OSM_PLANET_SIZE} bytes." run() { - TIMESTAMP=$(date +"%Y-%m-%d %H:%M:%S") - START_TIME=$(date +%s) - - if "$DIR/render_once.sh" "$WORKING_DIR" - then - # Record the end time - END_TIME=$(date +%s) - - # Calculate the time difference - TIME_DIFF=$((END_TIME - START_TIME)) - - # Convert the time difference to hours and minutes - HOURS=$((TIME_DIFF / 3600)) - MINUTES=$(((TIME_DIFF / 60) % 60)) - - # Format hours and minutes with singular or plural as appropriate - HOUR_TEXT="hours" - MINUTE_TEXT="minutes" - if [ "$HOURS" -eq 1 ]; then - HOUR_TEXT="hour" - fi - if [ "$MINUTES" -eq 1 ]; then - MINUTE_TEXT="minute" - fi - - # Get the size of the file in bytes - PMTILES_PLANET_SIZE=$(stat -c%s "$PLANET") - - # Print the size with comma separators - PMTILES_PLANET_SIZE=$(printf "%'d" "$PMTILES_PLANET_SIZE") - - "$DIR/rss_update.sh" "$RSS_FILE" "Build Complete" "Tiles are up to date as of ${TIMESTAMP}Z. Render took ${HOURS} ${HOUR_TEXT} and ${MINUTES} ${MINUTE_TEXT}. The planet PMTiles file is ${PMTILES_PLANET_SIZE} bytes." - else - "$DIR/rss_update.sh" "$RSS_FILE" "Build Failed" "Review the build log to find out why." - fi - - echo 'Removing local planet file' - rm -rf "$PLANET" + TIMESTAMP=$(date +"%Y-%m-%d %H:%M:%S") + START_TIME=$(date +%s) + + if "$DIR/render_once.sh" "$WORKING_DIR"; then + # Record the end time + END_TIME=$(date +%s) + + # Calculate the time difference + TIME_DIFF=$((END_TIME - START_TIME)) + + # Convert the time difference to hours and minutes + HOURS=$((TIME_DIFF / 3600)) + MINUTES=$(((TIME_DIFF / 60) % 60)) + + # Format hours and minutes with singular or plural as appropriate + HOUR_TEXT="hours" + MINUTE_TEXT="minutes" + if [ "$HOURS" -eq 1 ]; then + HOUR_TEXT="hour" + fi + if [ "$MINUTES" -eq 1 ]; then + MINUTE_TEXT="minute" + fi + + # Get the size of the file in bytes + PMTILES_PLANET_SIZE=$(stat -c%s "$PLANET") + + # Print the size with comma separators + PMTILES_PLANET_SIZE=$(printf "%'d" "$PMTILES_PLANET_SIZE") + + "$DIR/rss_update.sh" "$RSS_FILE" "Build Complete" "Tiles are up to date as of ${TIMESTAMP}Z. Render took ${HOURS} ${HOUR_TEXT} and ${MINUTES} ${MINUTE_TEXT}. The planet PMTiles file is ${PMTILES_PLANET_SIZE} bytes." + else + "$DIR/rss_update.sh" "$RSS_FILE" "Build Failed" "Review the build log to find out why." + fi + + echo 'Removing local planet file' + rm -rf "$PLANET" } -run 2>&1 | tee -a "$LOG_FILE" \ No newline at end of file +run 2>&1 | tee -a "$LOG_FILE" diff --git a/renderer/rss_update.sh b/renderer/rss_update.sh index b5b0219..702507c 100755 --- a/renderer/rss_update.sh +++ b/renderer/rss_update.sh @@ -2,8 +2,8 @@ # Check for input parameter (the message to be added) if [ $# -lt 1 ]; then - echo "Usage: $0 " - exit 1 + echo "Usage: $0 " + exit 1 fi RSS_FILE=$1 @@ -16,19 +16,19 @@ MESSAGE=$(echo "$MESSAGE" | sed 's/&/\&/g; s//\>/g; s/"/\&q # Check if the RSS file exists if [ -f "$RSS_FILE" ]; then - # Get the size of the RSS file - FILE_SIZE=$(stat -c%s "$RSS_FILE") - - # If the size of the RSS file is more than 100KB, remove it - if [ "$FILE_SIZE" -gt 102400 ]; then - echo "The RSS file is larger than 100KB. Removing the file..." - rm "$RSS_FILE" - fi + # Get the size of the RSS file + FILE_SIZE=$(stat -c%s "$RSS_FILE") + + # If the size of the RSS file is more than 100KB, remove it + if [ "$FILE_SIZE" -gt 102400 ]; then + echo "The RSS file is larger than 100KB. Removing the file..." + rm "$RSS_FILE" + fi fi # If the RSS file doesn't exist, create a new one if [ ! -f "$RSS_FILE" ]; then - echo ' + echo ' OpenStreetMap US @@ -36,7 +36,7 @@ if [ ! -f "$RSS_FILE" ]; then https://tile.ourmap.us/ Planet build status -' > "$RSS_FILE" +' >"$RSS_FILE" fi # Read the URL from /tmp/seashells_render and extract it using grep @@ -44,14 +44,14 @@ URL=$(grep -o 'https://[^ ]*' /tmp/seashells_render) # Add the new item entry to the RSS file xmlstarlet ed --inplace \ - -s "//channel" -t elem -n "item" -v "" \ - -s "//channel/item[last()]" -t elem -n "title" -v "$TITLE" \ - -s "//channel/item[last()]" -t elem -n "author" -v "tech@openstreetmap.us" \ - -s "//channel/item[last()]" -t elem -n "link" -v "$URL" \ - -s "//channel/item[last()]" -t elem -n "pubDate" -v "$CURRENT_DATE" \ - -s "//channel/item[last()]" -t elem -n "content:encoded" -v "$MESSAGE" \ - -i "//channel/item[last()]/content:encoded" -t attr -n "type" -v "html" \ - "$RSS_FILE" + -s "//channel" -t elem -n "item" -v "" \ + -s "//channel/item[last()]" -t elem -n "title" -v "$TITLE" \ + -s "//channel/item[last()]" -t elem -n "author" -v "tech@openstreetmap.us" \ + -s "//channel/item[last()]" -t elem -n "link" -v "$URL" \ + -s "//channel/item[last()]" -t elem -n "pubDate" -v "$CURRENT_DATE" \ + -s "//channel/item[last()]" -t elem -n "content:encoded" -v "$MESSAGE" \ + -i "//channel/item[last()]/content:encoded" -t attr -n "type" -v "html" \ + "$RSS_FILE" echo "New entry added to $RSS_FILE" -aws s3 cp "$RSS_FILE" s3://tileserver-static/ \ No newline at end of file +aws s3 cp "$RSS_FILE" s3://tileserver-static/ From 65c0e7314470d3b674e852f5a0acda5dd664b5dc Mon Sep 17 00:00:00 2001 From: Quincy Morgan <2046746+quincylvania@users.noreply.github.com> Date: Tue, 21 May 2024 20:02:39 -0400 Subject: [PATCH 11/13] No top level permissions --- .github/workflows/lint.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/lint.yml b/.github/workflows/lint.yml index 1ae8f4e..3ecdfd0 100644 --- a/.github/workflows/lint.yml +++ b/.github/workflows/lint.yml @@ -3,6 +3,7 @@ name: Lint on: - push - pull_request +permissions: {} jobs: build: name: Lint From 51b594aca3650f0685d72344917852b6720491cb Mon Sep 17 00:00:00 2001 From: Quincy Morgan <2046746+quincylvania@users.noreply.github.com> Date: Tue, 21 May 2024 20:02:50 -0400 Subject: [PATCH 12/13] Config yaml linter --- .github/linters/.yaml-lint.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/linters/.yaml-lint.yml b/.github/linters/.yaml-lint.yml index a042aba..17c51fb 100644 --- a/.github/linters/.yaml-lint.yml +++ b/.github/linters/.yaml-lint.yml @@ -3,6 +3,7 @@ extends: default rules: indentation: spaces: 2 + indent-sequences: consistent line-length: disable new-line-at-end-of-file: level: warning From 46bd08da5b078dff6e2e44e3ce061ee1bf25cc27 Mon Sep 17 00:00:00 2001 From: Quincy Morgan <2046746+quincylvania@users.noreply.github.com> Date: Tue, 21 May 2024 20:08:41 -0400 Subject: [PATCH 13/13] whatever --- .github/linters/.yaml-lint.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/linters/.yaml-lint.yml b/.github/linters/.yaml-lint.yml index 17c51fb..44b75a8 100644 --- a/.github/linters/.yaml-lint.yml +++ b/.github/linters/.yaml-lint.yml @@ -3,7 +3,7 @@ extends: default rules: indentation: spaces: 2 - indent-sequences: consistent + indent-sequences: whatever line-length: disable new-line-at-end-of-file: level: warning