diff --git a/.github/linters/.yaml-lint.yml b/.github/linters/.yaml-lint.yml new file mode 100644 index 0000000..44b75a8 --- /dev/null +++ b/.github/linters/.yaml-lint.yml @@ -0,0 +1,12 @@ +--- +extends: default +rules: + indentation: + spaces: 2 + indent-sequences: whatever + line-length: disable + new-line-at-end-of-file: + level: warning + trailing-spaces: + level: warning + truthy: disable diff --git a/.github/workflows/lint.yml b/.github/workflows/lint.yml new file mode 100644 index 0000000..3ecdfd0 --- /dev/null +++ b/.github/workflows/lint.yml @@ -0,0 +1,32 @@ +--- +name: Lint +on: + - push + - pull_request +permissions: {} +jobs: + build: + name: Lint + runs-on: ubuntu-latest + + permissions: + contents: read + packages: read + # To report GitHub Actions status checks + statuses: write + + steps: + - name: Checkout code + uses: actions/checkout@v4 + with: + # super-linter needs the full git history to get the + # list of files that changed across commits + fetch-depth: 0 + + - name: Super-linter + uses: super-linter/super-linter@v6.5.0 # x-release-please-version + env: + # To report GitHub Actions status checks + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + VALIDATE_JSCPD: false + VALIDATE_NATURAL_LANGUAGE: false diff --git a/README.md b/README.md index 3822a26..7b0916c 100644 --- a/README.md +++ b/README.md @@ -1,2 +1,2 @@ # osmus/tileservice -Central repo for the OSM US vector tile service +Central repository for the OSM US vector tile service diff --git a/renderer/_example.sh b/renderer/_example.sh old mode 100644 new mode 100755 index e0c4751..246d99e --- a/renderer/_example.sh +++ b/renderer/_example.sh @@ -12,14 +12,14 @@ LOCKFILE="/tmp/planet-render.lock" # Check if lock file exists if [ -e "${LOCKFILE}" ]; then - echo "A rendering process is already running." - exit 1 + echo "A rendering process is already running." + exit 1 else - # Create a lock file - touch "${LOCKFILE}" + # Create a lock file + touch "${LOCKFILE}" - # Ensure the lock file is removed when we exit and when we receive signals - trap "rm -f ${LOCKFILE}; trap - SIGTERM && kill -- -$$" SIGINT SIGTERM EXIT + # Ensure the lock file is removed when we exit and when we receive signals + trap 'rm -f ${LOCKFILE}; trap - SIGTERM && kill -- -$$' SIGINT SIGTERM EXIT fi # The local path to the tileservice repo @@ -32,4 +32,4 @@ mkdir "$REPO_DIR" git clone https://github.com/osmus/tileservice.git "$REPO_DIR" # Start the render, assuming the directory format hasn't changed -"$REPO_DIR/renderer/render.sh" \ No newline at end of file +"$REPO_DIR/renderer/render.sh" diff --git a/renderer/layers/trails.yml b/renderer/layers/trails.yml index aa4b7e1..9426159 100644 --- a/renderer/layers/trails.yml +++ b/renderer/layers/trails.yml @@ -1,3 +1,4 @@ +--- schema_name: OSM US Trails schema_description: A map of trails attribution: © OpenStreetMap contributors @@ -296,4 +297,3 @@ args: area: description: Geofabrik area to download default: rhode-island - diff --git a/renderer/layers/water_trails.yml b/renderer/layers/water_trails.yml index e539762..369476e 100644 --- a/renderer/layers/water_trails.yml +++ b/renderer/layers/water_trails.yml @@ -1,3 +1,4 @@ +--- schema_name: OSM US Water Trails schema_description: Canoe and kayak waterways and portages. attribution: © OpenStreetMap contributors @@ -229,4 +230,3 @@ args: area: description: Geofabrik area to download default: rhode-island - diff --git a/renderer/layers/water_trails_poi.yml b/renderer/layers/water_trails_poi.yml index 84801fb..715a97a 100644 --- a/renderer/layers/water_trails_poi.yml +++ b/renderer/layers/water_trails_poi.yml @@ -1,5 +1,6 @@ +--- schema_name: OSM US Water Trail POIs -schema_description: Slipways, access points, dams, weirs, and waterfalls. +schema_description: Slipways, access points, dams, weirs, and waterfalls. attribution: © OpenStreetMap contributors sources: osm: @@ -100,4 +101,3 @@ args: area: description: Geofabrik area to download default: rhode-island - diff --git a/renderer/render.sh b/renderer/render.sh index 0ddd662..e74d2a7 100755 --- a/renderer/render.sh +++ b/renderer/render.sh @@ -3,4 +3,4 @@ # Get the directory of the current script DIR="$(dirname "$0")" -"$DIR/render_seashells.sh" $1 \ No newline at end of file +"$DIR/render_seashells.sh" "$1" diff --git a/renderer/render_once.sh b/renderer/render_once.sh index 9758449..875bc3b 100755 --- a/renderer/render_once.sh +++ b/renderer/render_once.sh @@ -26,10 +26,10 @@ docker system prune --force docker pull ghcr.io/onthegomap/planetiler:latest docker run -e JAVA_TOOL_OPTIONS='-Xmx2g' \ - -v "$WORKING_DIR/data":/data \ - -v "$DIR/layers":/layers \ - ghcr.io/onthegomap/planetiler:latest --area=planet \ - --download --download-only --only-fetch-wikidata + -v "$WORKING_DIR/data":/data \ + -v "$DIR/layers":/layers \ + ghcr.io/onthegomap/planetiler:latest --area=planet \ + --download --download-only --only-fetch-wikidata # Remove default downloaded OSM file rm -rf "$WORKING_DIR/data/sources/monaco.osm.pbf" @@ -37,25 +37,25 @@ rm -rf "$WORKING_DIR/data/sources/monaco.osm.pbf" PLANET="$WORKING_DIR/data/planet.pmtiles" docker run -e JAVA_TOOL_OPTIONS='-Xmx150g' \ - -v "$WORKING_DIR/data":/data \ - -v "$DIR/layers":/layers \ - ghcr.io/onthegomap/planetiler:latest --area=planet --bounds=world \ - --output="/data/planet.pmtiles" \ - --transportation_name_size_for_shield \ - --transportation_name_limit_merge \ - --boundary-osm-only \ - --storage=ram --nodemap-type=array \ - --max-point-buffer=4 \ - --building_merge_z13=false \ - --languages=ab,ace,af,als,am,an,ar,arz,as,ast,az,az-Arab,az-cyr,azb,ba,bar,bat-smg,be,be-tarask,ber,bg,bm,bn,bo,bpy,br,bs,bxr,ca,cdo,ce,ceb,cho,chr,chy,ckb,co,cr,crh,crh-cyr,crk,cs,csb,cv,cy,da,dak,de,dsb,dv,dz,ee,egl,el,en,eo,es,et,eu,fa,fi,fil,fit,fo,fr,frr,full,fur,fy,ga,gag,gan,gcf,gd,gl,gn,gr,grc,gsw,gu,gv,ha,hak,hak-HJ,haw,he,hi,hif,hr,hsb,ht,hu,hur,hy,ia,id,ie,ilo,int,io,is,it,iu,ja,ja_kana,ja_rm,ja-Hira,ja-Latn,jv,ka,kab,kbd,ki,kk,kk-Arab,kl,km,kn,ko,ko-Hani,ko-Latn,krc,krl,ks,ku,kv,kw,ky,la,lb,left,lez,li,lij,lld,lmo,ln,lo,lrc,lt,lv,lzh,md,mdf,mez,mg,mhr,mi,mia,mk,ml,mn,mo,moh,mr,mrj,ms,ms-Arab,mt,mwl,my,myv,mzn,nah,nan,nan-HJ,nan-POJ,nan-TL,nds,ne,nl,nn,no,nov,nv,oc,oj,old,or,os,ota,pa,pam,pcd,pfl,pl,pms,pnb,pot,ps,pt,pt-BR,pt-PT,qu,right,rm,ro,ru,rue,rw,sah,sat,sc,scn,sco,sd,se,sh,si,sju,sk,sl,sma,smj,so,sq,sr,sr-Latn,su,sv,sw,syc,szl,ta,te,TEC,tg,th,th-Latn,ti,tk,tl,tr,tt,tt-lat,udm,ug,uk,ur,uz,uz-Arab,uz-cyr,uz-Cyrl,uz-Latn,vec,vi,vls,vo,wa,war,win,wiy,wo,wuu,xmf,yi,yo,yue,yue-Hant,yue-Latn,za,zgh,zh,zh_pinyin,zh_zhuyin,zh-Hans,zh-Hant,zh-Latn-pinyin,zu,zza + -v "$WORKING_DIR/data":/data \ + -v "$DIR/layers":/layers \ + ghcr.io/onthegomap/planetiler:latest --area=planet --bounds=world \ + --output="/data/planet.pmtiles" \ + --transportation_name_size_for_shield \ + --transportation_name_limit_merge \ + --boundary-osm-only \ + --storage=ram --nodemap-type=array \ + --max-point-buffer=4 \ + --building_merge_z13=false \ + --languages=ab,ace,af,als,am,an,ar,arz,as,ast,az,az-Arab,az-cyr,azb,ba,bar,bat-smg,be,be-tarask,ber,bg,bm,bn,bo,bpy,br,bs,bxr,ca,cdo,ce,ceb,cho,chr,chy,ckb,co,cr,crh,crh-cyr,crk,cs,csb,cv,cy,da,dak,de,dsb,dv,dz,ee,egl,el,en,eo,es,et,eu,fa,fi,fil,fit,fo,fr,frr,full,fur,fy,ga,gag,gan,gcf,gd,gl,gn,gr,grc,gsw,gu,gv,ha,hak,hak-HJ,haw,he,hi,hif,hr,hsb,ht,hu,hur,hy,ia,id,ie,ilo,int,io,is,it,iu,ja,ja_kana,ja_rm,ja-Hira,ja-Latn,jv,ka,kab,kbd,ki,kk,kk-Arab,kl,km,kn,ko,ko-Hani,ko-Latn,krc,krl,ks,ku,kv,kw,ky,la,lb,left,lez,li,lij,lld,lmo,ln,lo,lrc,lt,lv,lzh,md,mdf,mez,mg,mhr,mi,mia,mk,ml,mn,mo,moh,mr,mrj,ms,ms-Arab,mt,mwl,my,myv,mzn,nah,nan,nan-HJ,nan-POJ,nan-TL,nds,ne,nl,nn,no,nov,nv,oc,oj,old,or,os,ota,pa,pam,pcd,pfl,pl,pms,pnb,pot,ps,pt,pt-BR,pt-PT,qu,right,rm,ro,ru,rue,rw,sah,sat,sc,scn,sco,sd,se,sh,si,sju,sk,sl,sma,smj,so,sq,sr,sr-Latn,su,sv,sw,syc,szl,ta,te,TEC,tg,th,th-Latn,ti,tk,tl,tr,tt,tt-lat,udm,ug,uk,ur,uz,uz-Arab,uz-cyr,uz-Cyrl,uz-Latn,vec,vi,vls,vo,wa,war,win,wiy,wo,wuu,xmf,yi,yo,yue,yue-Hant,yue-Latn,za,zgh,zh,zh_pinyin,zh_zhuyin,zh-Hans,zh-Hant,zh-Latn-pinyin,zu,zza # Check if the file exists and is at least 50GB if [[ ! -f "$PLANET" ]]; then - echo "Error: File $PLANET does not exist." - exit 1 -elif [[ $(stat -c %s "$PLANET") -lt $((50*1024*1024*1024)) ]]; then - echo "Error: File $PLANET is smaller than 50GB." - exit 1 + echo "Error: File $PLANET does not exist." + exit 1 +elif [[ $(stat -c %s "$PLANET") -lt $((50 * 1024 * 1024 * 1024)) ]]; then + echo "Error: File $PLANET is smaller than 50GB." + exit 1 fi echo 'Uploading planet to s3 bucket in background' @@ -63,23 +63,26 @@ aws s3 cp "$PLANET" s3://planet-pmtiles/ --only-show-errors & # Render optional layers for file in "$DIR/layers/"*.yml; do - # Get the base name of the file without the .yml extension - layer_name=$(basename "$file" .yml) - - echo "Processing layer: $layer_name" - - docker run -e JAVA_TOOL_OPTIONS='-Xmx24g' \ - -v "$WORKING_DIR/data":/data \ - -v "$DIR/layers":/layers \ - ghcr.io/onthegomap/planetiler:latest generate-custom \ - --area=planet --bounds=world \ - --output="/data/$layer_name.pmtiles" \ - --schema="/layers/$layer_name.yml" \ - --storage=mmap --nodemap-type=array \ - --max-point-buffer=4 - - echo "Uploading $layer_name to s3 bucket in background" - { aws s3 cp "$WORKING_DIR/data/$layer_name.pmtiles" s3://planet-pmtiles/ --only-show-errors; rm -rf "$WORKING_DIR/data/$layer_name.pmtiles"; } & + # Get the base name of the file without the .yml extension + layer_name=$(basename "$file" .yml) + + echo "Processing layer: $layer_name" + + docker run -e JAVA_TOOL_OPTIONS='-Xmx24g' \ + -v "$WORKING_DIR/data":/data \ + -v "$DIR/layers":/layers \ + ghcr.io/onthegomap/planetiler:latest generate-custom \ + --area=planet --bounds=world \ + --output="/data/$layer_name.pmtiles" \ + --schema="/layers/$layer_name.yml" \ + --storage=mmap --nodemap-type=array \ + --max-point-buffer=4 + + echo "Uploading $layer_name to s3 bucket in background" + { + aws s3 cp "$WORKING_DIR/data/$layer_name.pmtiles" s3://planet-pmtiles/ --only-show-errors + rm -rf "$WORKING_DIR/data/$layer_name.pmtiles" + } & done echo 'Waiting for all background jobs to finish' @@ -89,4 +92,4 @@ echo 'Invalidating the CDN cache' aws cloudfront create-invalidation --distribution-id E1E7N0LWX2WY4E --invalidation-batch "{\"Paths\": {\"Quantity\": 1, \"Items\": [\"/*\"]}, \"CallerReference\": \"invalidation-$DATE\"}" echo 'Render Complete' -date -u '+%Y-%m-%d %H:%M:%S' \ No newline at end of file +date -u '+%Y-%m-%d %H:%M:%S' diff --git a/renderer/render_seashells.sh b/renderer/render_seashells.sh index 0e8b3be..fe3bbf7 100755 --- a/renderer/render_seashells.sh +++ b/renderer/render_seashells.sh @@ -17,13 +17,14 @@ LOG_TIMESTAMP=$(date +%Y%m%d%H%M%S) LOG_FILE="/var/log/render/logs_$LOG_TIMESTAMP.txt" touch "$LOG_FILE" -tail -f "$LOG_FILE" | nc seashells.io 1337 > /tmp/seashells_render & sleep 10 +tail -f "$LOG_FILE" | nc seashells.io 1337 >/tmp/seashells_render & +sleep 10 # Get the size of the file in bytes OSM_PLANET_SIZE=$(stat -c%s "$WORKING_DIR/data/sources/planet.osm.pbf") # Print the size with comma separators -OSM_PLANET_SIZE=$(printf "%'d" $OSM_PLANET_SIZE) +OSM_PLANET_SIZE=$(printf "%'d" "$OSM_PLANET_SIZE") RSS_FILE="$WORKING_DIR/rss.xml" PLANET="$WORKING_DIR/data/planet.pmtiles" @@ -31,45 +32,43 @@ PLANET="$WORKING_DIR/data/planet.pmtiles" "$DIR/rss_update.sh" "$RSS_FILE" "Build Started." "The OSM planet file is ${OSM_PLANET_SIZE} bytes." run() { - TIMESTAMP=$(date +"%Y-%m-%d %H:%M:%S") - START_TIME=$(date +%s) - "$DIR/render_once.sh" "$WORKING_DIR" - - # Check the exit status of render_once.sh - if [ $? -eq 0 ]; then - # Record the end time - END_TIME=$(date +%s) - - # Calculate the time difference - TIME_DIFF=$((END_TIME - START_TIME)) - - # Convert the time difference to hours and minutes - HOURS=$((TIME_DIFF / 3600)) - MINUTES=$(((TIME_DIFF / 60) % 60)) - - # Format hours and minutes with singular or plural as appropriate - HOUR_TEXT="hours" - MINUTE_TEXT="minutes" - if [ "$HOURS" -eq 1 ]; then - HOUR_TEXT="hour" - fi - if [ "$MINUTES" -eq 1 ]; then - MINUTE_TEXT="minute" - fi - - # Get the size of the file in bytes - PMTILES_PLANET_SIZE=$(stat -c%s "$PLANET") - - # Print the size with comma separators - PMTILES_PLANET_SIZE=$(printf "%'d" $PMTILES_PLANET_SIZE) - - "$DIR/rss_update.sh" "$RSS_FILE" "Build Complete" "Tiles are up to date as of ${TIMESTAMP}Z. Render took ${HOURS} ${HOUR_TEXT} and ${MINUTES} ${MINUTE_TEXT}. The planet PMTiles file is ${PMTILES_PLANET_SIZE} bytes." - else - "$DIR/rss_update.sh" "$RSS_FILE" "Build Failed" "Review the build log to find out why." - fi - - echo 'Removing local planet file' - rm -rf "$PLANET" + TIMESTAMP=$(date +"%Y-%m-%d %H:%M:%S") + START_TIME=$(date +%s) + + if "$DIR/render_once.sh" "$WORKING_DIR"; then + # Record the end time + END_TIME=$(date +%s) + + # Calculate the time difference + TIME_DIFF=$((END_TIME - START_TIME)) + + # Convert the time difference to hours and minutes + HOURS=$((TIME_DIFF / 3600)) + MINUTES=$(((TIME_DIFF / 60) % 60)) + + # Format hours and minutes with singular or plural as appropriate + HOUR_TEXT="hours" + MINUTE_TEXT="minutes" + if [ "$HOURS" -eq 1 ]; then + HOUR_TEXT="hour" + fi + if [ "$MINUTES" -eq 1 ]; then + MINUTE_TEXT="minute" + fi + + # Get the size of the file in bytes + PMTILES_PLANET_SIZE=$(stat -c%s "$PLANET") + + # Print the size with comma separators + PMTILES_PLANET_SIZE=$(printf "%'d" "$PMTILES_PLANET_SIZE") + + "$DIR/rss_update.sh" "$RSS_FILE" "Build Complete" "Tiles are up to date as of ${TIMESTAMP}Z. Render took ${HOURS} ${HOUR_TEXT} and ${MINUTES} ${MINUTE_TEXT}. The planet PMTiles file is ${PMTILES_PLANET_SIZE} bytes." + else + "$DIR/rss_update.sh" "$RSS_FILE" "Build Failed" "Review the build log to find out why." + fi + + echo 'Removing local planet file' + rm -rf "$PLANET" } -run 2>&1 | tee -a "$LOG_FILE" \ No newline at end of file +run 2>&1 | tee -a "$LOG_FILE" diff --git a/renderer/rss_update.sh b/renderer/rss_update.sh index b5b0219..702507c 100755 --- a/renderer/rss_update.sh +++ b/renderer/rss_update.sh @@ -2,8 +2,8 @@ # Check for input parameter (the message to be added) if [ $# -lt 1 ]; then - echo "Usage: $0 " - exit 1 + echo "Usage: $0 " + exit 1 fi RSS_FILE=$1 @@ -16,19 +16,19 @@ MESSAGE=$(echo "$MESSAGE" | sed 's/&/\&/g; s//\>/g; s/"/\&q # Check if the RSS file exists if [ -f "$RSS_FILE" ]; then - # Get the size of the RSS file - FILE_SIZE=$(stat -c%s "$RSS_FILE") - - # If the size of the RSS file is more than 100KB, remove it - if [ "$FILE_SIZE" -gt 102400 ]; then - echo "The RSS file is larger than 100KB. Removing the file..." - rm "$RSS_FILE" - fi + # Get the size of the RSS file + FILE_SIZE=$(stat -c%s "$RSS_FILE") + + # If the size of the RSS file is more than 100KB, remove it + if [ "$FILE_SIZE" -gt 102400 ]; then + echo "The RSS file is larger than 100KB. Removing the file..." + rm "$RSS_FILE" + fi fi # If the RSS file doesn't exist, create a new one if [ ! -f "$RSS_FILE" ]; then - echo ' + echo ' OpenStreetMap US @@ -36,7 +36,7 @@ if [ ! -f "$RSS_FILE" ]; then https://tile.ourmap.us/ Planet build status -' > "$RSS_FILE" +' >"$RSS_FILE" fi # Read the URL from /tmp/seashells_render and extract it using grep @@ -44,14 +44,14 @@ URL=$(grep -o 'https://[^ ]*' /tmp/seashells_render) # Add the new item entry to the RSS file xmlstarlet ed --inplace \ - -s "//channel" -t elem -n "item" -v "" \ - -s "//channel/item[last()]" -t elem -n "title" -v "$TITLE" \ - -s "//channel/item[last()]" -t elem -n "author" -v "tech@openstreetmap.us" \ - -s "//channel/item[last()]" -t elem -n "link" -v "$URL" \ - -s "//channel/item[last()]" -t elem -n "pubDate" -v "$CURRENT_DATE" \ - -s "//channel/item[last()]" -t elem -n "content:encoded" -v "$MESSAGE" \ - -i "//channel/item[last()]/content:encoded" -t attr -n "type" -v "html" \ - "$RSS_FILE" + -s "//channel" -t elem -n "item" -v "" \ + -s "//channel/item[last()]" -t elem -n "title" -v "$TITLE" \ + -s "//channel/item[last()]" -t elem -n "author" -v "tech@openstreetmap.us" \ + -s "//channel/item[last()]" -t elem -n "link" -v "$URL" \ + -s "//channel/item[last()]" -t elem -n "pubDate" -v "$CURRENT_DATE" \ + -s "//channel/item[last()]" -t elem -n "content:encoded" -v "$MESSAGE" \ + -i "//channel/item[last()]/content:encoded" -t attr -n "type" -v "html" \ + "$RSS_FILE" echo "New entry added to $RSS_FILE" -aws s3 cp "$RSS_FILE" s3://tileserver-static/ \ No newline at end of file +aws s3 cp "$RSS_FILE" s3://tileserver-static/