diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index c30606038..1c45c20af 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -23,6 +23,6 @@ jobs: compose_service: underpass compose_command: '"make check -j $(nproc)"' tag_override: ci - # TODO update postgis image to use github repo var ${{ vars.POSTGIS_TAG }} + # TODO update postgis image to use github repo var ${{ vars.UNDERPASSDB_TAG }} cache_extra_imgs: | "docker.io/postgis/postgis:15-3.3-alpine" diff --git a/.gitignore b/.gitignore index 853de149c..02ecae49c 100644 --- a/.gitignore +++ b/.gitignore @@ -43,3 +43,4 @@ unconfig.h.in **/__pycache__/ .vscode data +data_osm diff --git a/README.md b/README.md index ea28aeccf..73644d20a 100644 --- a/README.md +++ b/README.md @@ -9,12 +9,12 @@ It **updates a local copy of the OSM database** in near real-time, and provides ## Demo -We've deployed a rudimentary demo that keeps a database up-to-date for (some country), +We've deployed a basic demo that keeps a database up-to-date for (some country), rendering buildings and highlighting the ones identified as "un-squared": [https://underpass.live](https://underpass.live) -Screenshot 2023-11-22 at 10 32 56 +Screenshot 2024-06-05 at 15 51 57 ## Getting started @@ -26,17 +26,9 @@ Check the tasks board and roadmap [here](https://github.com/orgs/hotosm/projects ### Get involved! -We invite software designers and developers to contribute to the project, there are several issues -where we need help, some of them are: - -* Designs for data visualizations -* React UI components -* Data quality checks for the C++ core engine -* PostgreSQL queries for the Python `dbapi` module -* Endpoints for the Python `restapi` module -* Packages for Python, React and system binaries -* Data models for semantic validation -* Tests for everything +This is an exciting project, its core is made with C++ but it also includes a Python API and there are UI components for interacting and visualizing data. + +You can learn a lot contributing to Underpass, while helping the FOSS and mapping communities, get involved! ### License diff --git a/config/default.yaml b/config/default.yaml index e0a79e379..2d8a4851f 100644 --- a/config/default.yaml +++ b/config/default.yaml @@ -2,6 +2,8 @@ config: - underpass_db_url: - underpass:underpass@localhost:5432/underpass + - underpass_osm_db_url: + - underpass:underpass@localhost:5432/underpass - planet_servers: - planet.maps.mail.ru - destdir_base: diff --git a/docker-compose.yml b/docker-compose.yml index ed56bd05f..e5a258ed9 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -1,5 +1,5 @@ # -# Copyright (c) 2020, 2021, 2022, 2023 Humanitarian OpenStreetMap Team +# Copyright (c) 2020, 2021, 2022, 2023, 2024 Humanitarian OpenStreetMap Team # # This file is part of Underpass. # @@ -20,10 +20,10 @@ version: "3" services: - # Database - postgis: - image: postgis/postgis:${POSTGIS_TAG:-15-3.3-alpine} - container_name: "underpass_postgis" + # Database for Underpass + underpass_db: + image: postgis/postgis:${UNDERPASS_DB_TAG:-15-3.3-alpine} + container_name: "underpass_db" ports: - "${DB_PORT:-5439}:5432" environment: @@ -41,6 +41,28 @@ services: networks: internal: +# Un-comment for starting a second database +# Database for OSM Raw Data + # osm_db: + # image: postgis/postgis:${OSM_DB_TAG:-15-3.3-alpine} + # container_name: "osm_db" + # ports: + # - "${DB_PORT:-5440}:5432" + # environment: + # - POSTGRES_DB=osm + # - POSTGRES_USER=underpass + # - POSTGRES_PASSWORD=underpass + # volumes: + # - ./data_osm:/var/lib/postgresql/data + # restart: on-failure + # logging: + # driver: "json-file" + # options: + # max-size: "200k" + # max-file: "10" + # networks: + # internal: + # Underpass underpass: image: "ghcr.io/hotosm/underpass:${TAG_OVERRIDE:-debug}" @@ -51,13 +73,16 @@ services: target: ${TAG_OVERRIDE:-debug} args: APP_VERSION: ${APP_VERSION:-debug} - depends_on: [postgis] + depends_on: [underpass_db] environment: - - REPLICATOR_UNDERPASS_DB_URL=underpass:underpass@postgis/underpass + - REPLICATOR_UNDERPASS_DB_URL=underpass:underpass@underpass_db/underpass + - REPLICATOR_OSM_DB_URL=underpass:underpass@underpass_db/underpass + command: tail -f /dev/null + # Un-comment for debugging # volumes: - # - ${PWD}:/code - # - ./replication:/code/build/replication + # - ${PWD}:/code + # - ./replication:/usr/local/lib/underpass/data/replication networks: internal: @@ -79,26 +104,8 @@ services: networks: internal: environment: - - UNDERPASS_API_DB=postgresql://underpass:underpass@postgis/underpass - - # Underpass UI - ui: - image: "ghcr.io/hotosm/underpass/ui:${APP_VERSION:-debug}" - container_name: "underpass_ui" - build: - context: . - dockerfile: docker/underpass-ui.dockerfile - target: debug - args: - APP_VERSION: ${APP_VERSION:-debug} - # # Mount underpass-ui repo - # volumes: - # - ../underpass-ui/src:/code/src - # - ../underpass-ui/playground:/code/playground - ports: - - "${UI_PORT:-8080}:5000" - networks: - internal: + - UNDERPASS_API_DB=postgresql://underpass:underpass@underpass/underpass + - UNDERPASS_API_OSM_DB=postgresql://underpass:underpass@underpass/underpass networks: internal: diff --git a/docker/underpass-config.yaml b/docker/underpass-config.yaml index cdc52f084..2e5d4a187 100644 --- a/docker/underpass-config.yaml +++ b/docker/underpass-config.yaml @@ -1,7 +1,9 @@ # Underpass config file config: + - underpass_osm_db_url: + - underpass:underpass@underpass_db:5432/underpass - underpass_db_url: - - underpass@postgis/underpass + - underpass:underpass@underpass_db:5432/underpass - planet_servers: - planet.maps.mail.ru - destdir_base: diff --git a/docker/underpass.dockerfile b/docker/underpass.dockerfile index a49c53b2c..7461bc9c6 100644 --- a/docker/underpass.dockerfile +++ b/docker/underpass.dockerfile @@ -48,6 +48,7 @@ RUN set -ex \ "librange-v3-dev" \ "libtool" \ "osm2pgsql" \ + "rsync" \ && rm -rf /var/lib/apt/lists/* diff --git a/docs/Doxyfile.in b/docs/Doxyfile.in index 74b5134cc..0c402bee4 100644 --- a/docs/Doxyfile.in +++ b/docs/Doxyfile.in @@ -844,7 +844,7 @@ WARN_LOGFILE = # spaces. See also FILE_PATTERNS and EXTENSION_MAPPING # Note: If this tag is empty the current directory is searched. -INPUT = @SRCDIR@ +INPUT = ../.. # This tag can be used to specify the character encoding of the source files # that doxygen parses. Internally doxygen uses the UTF-8 encoding. Doxygen uses diff --git a/m4/ax_boost_timer.m4 b/m4/ax_boost_timer.m4 index 95c983352..35254128f 100644 --- a/m4/ax_boost_timer.m4 +++ b/m4/ax_boost_timer.m4 @@ -67,8 +67,8 @@ AC_DEFUN([AX_BOOST_TIMER], [AC_LANG_PUSH([C++]) CXXFLAGS_SAVE=$CXXFLAGS - AC_COMPILE_IFELSE([AC_LANG_PROGRAM([[@%:@include ]], - [[boost::timer timer;]])], + AC_COMPILE_IFELSE([AC_LANG_PROGRAM([[@%:@include ]], + [[boost::timer::auto_cpu_timer t;]])], ax_cv_boost_timer=yes, ax_cv_boost_timer=no) CXXFLAGS=$CXXFLAGS_SAVE AC_LANG_POP([C++]) diff --git a/python/dbapi/api/config.py b/python/dbapi/api/config.py new file mode 100644 index 000000000..2acad748c --- /dev/null +++ b/python/dbapi/api/config.py @@ -0,0 +1,6 @@ +# Results per page on raw featues queries +RESULTS_PER_PAGE = 500 +# Results per page on list featues queries +RESULTS_PER_PAGE_LIST = 10 +# Print debug messages +DEBUG=False \ No newline at end of file diff --git a/python/dbapi/api/db.py b/python/dbapi/api/db.py index db647593f..6c1687aed 100644 --- a/python/dbapi/api/db.py +++ b/python/dbapi/api/db.py @@ -19,21 +19,23 @@ import asyncpg import json +from .config import DEBUG -class UnderpassDB(): - # Default Underpass local DB configuration - # This might be replaced by an .ini config file +class DB(): + # Default DB configuration def __init__(self, connectionString = None): - self.connectionString = connectionString or "postgresql://underpass:underpass@postgis/underpass" + self.connectionString = connectionString or "postgresql://underpass:underpass@localhost:5432/underpass" self.pool = None + # Extract the name of the database + self.name = self.connectionString[self.connectionString.rfind('/') + 1:] async def __enter__(self): await self.connect() async def connect(self): """ Connect to the database """ - print("Connecting to DB ...") + print("Connecting to DB ... " + self.connectionString if DEBUG else "") if not self.pool: try: self.pool = await asyncpg.create_pool( @@ -50,20 +52,27 @@ def close(self): if self.pool is not None: self.pool.close() - async def run(self, query, singleObject = False): + async def run(self, query, singleObject = False, asJson=False): + if DEBUG: + print("Running query ...") if not self.pool: await self.connect() if self.pool: + result = None try: conn = await self.pool.acquire() result = await conn.fetch(query) - if singleObject: - return result[0] - return json.loads((result[0]['result'])) + data = None + if asJson: + data = result[0]['result'] + elif singleObject: + data = result[0] + else: + data = result + await self.pool.release(conn) + return data except Exception as e: print("\n******* \n" + query + "\n******* \n") print(e) return None - finally: - await self.pool.release(conn) - return None + diff --git a/python/dbapi/api/filters.py b/python/dbapi/api/filters.py index a26abf6ff..479e5c75a 100644 --- a/python/dbapi/api/filters.py +++ b/python/dbapi/api/filters.py @@ -1,3 +1,21 @@ +#!/usr/bin/python3 +# +# Copyright (c) 2023, 2024 Humanitarian OpenStreetMap Team +# +# This file is part of Underpass. +# +# Underpass is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# Underpass is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with Underpass. If not, see . def tagsQueryFilter(tagsQuery, table): query = "" @@ -5,17 +23,33 @@ def tagsQueryFilter(tagsQuery, table): keyValue = tags[0].split("=") if len(keyValue) == 2: - query += "{0}.tags->>'{1}' ~* '^{2}'".format(table, keyValue[0], keyValue[1]) + query += "{table}.tags->>'{key}' ~* '^{value}'".format( + table=table, + key=keyValue[0], + value=keyValue[1] + ) else: - query += "{0}.tags->>'{1}' IS NOT NULL".format(table, keyValue[0]) + query += "{table}.tags->>'{key}' IS NOT NULL".format( + table=table, + key=keyValue[0] + ) for tag in tags[1:]: keyValue = tag.split("=") if len(keyValue) == 2: - query += "OR {0}.tags->>'{1}' ~* '^{2}'".format(table, keyValue[0], keyValue[1]) + query += "OR {table}.tags->>'{key}' ~* '^{value}'".format( + table=table, + key=keyValue[0], + value=keyValue[1] + ) else: - query += "OR {0}.tags->>'{1}' IS NOT NULL".format(table, keyValue[0]) + query += "OR {table}.tags->>'{key}' IS NOT NULL".format( + table=table, + key=keyValue[0] + ) return query def hashtagQueryFilter(hashtag, table): - return "'{0}' = ANY (hashtags)".format(hashtag) + return "'{hashtag}' = ANY (hashtags)".format( + hashtag=hashtag + ) diff --git a/python/dbapi/api/queryHelper.py b/python/dbapi/api/queryHelper.py index 122848ae0..20d4b00de 100644 --- a/python/dbapi/api/queryHelper.py +++ b/python/dbapi/api/queryHelper.py @@ -1,6 +1,6 @@ #!/usr/bin/python3 # -# Copyright (c) 2023 Humanitarian OpenStreetMap Team +# Copyright (c) 2023, 2024 Humanitarian OpenStreetMap Team # # This file is part of Underpass. # @@ -17,14 +17,14 @@ # You should have received a copy of the GNU General Public License # along with Underpass. If not, see . -RESULTS_PER_PAGE = 25 - def hashtags(hashtagsList): - return "EXISTS ( SELECT * from unnest(hashtags) as h where {0} )".format( - ' OR '.join( - map(lambda x: "h ~* '^{0}'".format(x), hashtagsList) + return "EXISTS ( SELECT * from unnest(hashtags) as h where {condition} )".format( + condition=' OR '.join( + map(lambda x: "h ~* '^{hashtag}'".format(hashtag=x), hashtagsList) ) ) def bbox(wktMultipolygon): - return "ST_Intersects(bbox, ST_GeomFromText('{0}', 4326))".format(wktMultipolygon) + return "ST_Intersects(bbox, ST_GeomFromText('{area}', 4326))".format( + area=wktMultipolygon + ) diff --git a/python/dbapi/api/raw.py b/python/dbapi/api/raw.py index fdbd347e6..a7e7ae7ee 100644 --- a/python/dbapi/api/raw.py +++ b/python/dbapi/api/raw.py @@ -17,476 +17,352 @@ # You should have received a copy of the GNU General Public License # along with Underpass. If not, see . +from dataclasses import dataclass from .filters import tagsQueryFilter, hashtagQueryFilter +from enum import Enum +from .config import RESULTS_PER_PAGE, RESULTS_PER_PAGE_LIST, DEBUG +from .sharedTypes import Table, GeoType +from .serialization import deserializeTags +import json + +# Build and run queries for getting geometry features +# (Points, LinesStrings, Polygons) from the Raw OSM Data DB + +# Order by +class OrderBy(Enum): + closedAt = "closed_at" + id = "id" + timestamp = "timestamp" + +# DB table names +class Table(Enum): + nodes = "nodes" + lines = "ways_line" + polygons = "ways_poly" + relations = "relations" + +# OSM types +class OsmType(Enum): + nodes = "node" + lines = "way" + polygons = "way" + +# Raw Features parameters DTO +@dataclass +class RawFeaturesParamsDTO: + area: str = None + tags: list[str] = None + hashtag: str = "" + dateFrom: str = "" + dateTo: str = "" + table: Table = Table.nodes + +# List Features parameters DTO +@dataclass +class ListFeaturesParamsDTO(RawFeaturesParamsDTO): + orderBy: OrderBy = OrderBy.id + page: int = 0 + +# Build queries for getting geometry features +def geoFeaturesQuery(params: RawFeaturesParamsDTO, asJson: bool = False): + geoType:GeoType = GeoType[params.table.name] + query = "SELECT '{type}' as type, \ + osm_id as id, \n \ + timestamp, \n \ + ST_AsText(geom) as geometry, \n \ + tags, \n \ + hashtags, \n \ + editor, \n \ + closed_at \n \ + FROM {table} \n \ + LEFT JOIN changesets c ON c.id = {table}.changeset \n \ + WHERE{area}{tags}{hashtag}{date} {limit}; \n \ + ".format( + type=geoType.value, + table=params.table.value, + area=" AND ST_Intersects(\"geom\", ST_GeomFromText('MULTIPOLYGON((({area})))', 4326) ) \n" + .format(area=params.area) if params.area else "", + tags=" AND (" + tagsQueryFilter(params.tags, params.table.value) + ") \n" if params.tags else "", + hashtag=" AND " + hashtagQueryFilter(params.hashtag, params.table.value) if params.hashtag else "", + date=" AND closed_at >= {dateFrom} AND closed_at <= {dateTo}\n" + .format(dateFrom=params.dateFrom, dateTo=params.dateTo) + if params.dateFrom and params.dateTo else "\n", + limit=" LIMIT {limit}".format(limit=RESULTS_PER_PAGE) + ).replace("WHERE AND", "WHERE") + + if asJson: + return rawQueryToJSON(query, params) -RESULTS_PER_PAGE = 500 -RESULTS_PER_PAGE_LIST = 10 - -def getGeoType(table): - if table == "ways_poly": - return "Polygon" - elif table == "ways_line": - return "LineString" - return "Node" - -def geoFeaturesQuery( - area = None, - tags = None, - hashtag = None, - dateFrom = None, - dateTo = None, - page = 0, - status = None, - table = None): - - geoType = getGeoType(table) - query = "with t_ways AS ( \ - SELECT '" + geoType + "' as type, " + table + ".osm_id as id, " + table + ".timestamp, geom as geometry, tags, status, hashtags, editor, created_at FROM " + table + " \ - LEFT JOIN validation ON validation.osm_id = " + table + ".osm_id \ - LEFT JOIN changesets c ON c.id = " + table + ".changeset \ - WHERE \ - {0} {1} {2} {3} {4} {5} \ - ), \ - t_features AS ( \ - SELECT jsonb_build_object( 'type', 'Feature', 'id', id, 'properties', to_jsonb(t_ways) \ - - 'geometry' , 'geometry', ST_AsGeoJSON(geometry)::jsonb ) AS feature FROM t_ways \ - ) SELECT jsonb_build_object( 'type', 'FeatureCollection', 'features', jsonb_agg(t_features.feature) ) \ - as result FROM t_features;".format( - "ST_Intersects(\"geom\", ST_GeomFromText('MULTIPOLYGON((({0})))', 4326) )".format(area) if area else "1=1 ", - "AND (" + tagsQueryFilter(tags, table) + ")" if tags else "", - "AND " + hashtagQueryFilter(hashtag, table) if hashtag else "", - "AND created_at >= {0} AND created_at <= {1}".format(dateFrom, dateTo) if dateFrom and dateTo else "", - "AND status = '{0}'".format(status) if (status) else "", - "LIMIT " + str(RESULTS_PER_PAGE), - ) - return query + return query -def listAllFeaturesQuery( - area, - tags, - hashtag, - status, - orderBy, - page, - dateFrom, - dateTo, - table, + +# Build queries for getting list of features +def listFeaturesQuery( + params: ListFeaturesParamsDTO, + asJson: bool = False ): - geoType = getGeoType(table) - if table == "nodes": - osmType = "node" - else: - osmType = "way" - - query = "\ - ( \ - SELECT '" + osmType + "' as type, '" + geoType + "' as geotype, " + table + ".osm_id as id, ST_X(ST_Centroid(geom)) as lat, ST_Y(ST_Centroid(geom)) as lon, " + table + ".timestamp, tags, " + table + ".changeset, c.created_at, v.status FROM " + table + " \ - LEFT JOIN changesets c ON c.id = " + table + ".changeset \ - LEFT JOIN validation v ON v.osm_id = " + table + ".osm_id \ - WHERE {0} {1} {2} {3} {4} {5} {6} \ + geoType:GeoType = GeoType[params.table] + osmType:OsmType = OsmType[params.table] + table:Table = Table[params.table] + + query = "( \ + SELECT '{type}' as type, \n \ + '{geotype}' as geotype, \n \ + {table}.osm_id as id, \n \ + ST_X(ST_Centroid(geom)) as lat, \n \ + ST_Y(ST_Centroid(geom)) as lon, \n \ + {table}.timestamp, \n \ + tags, \n \ + {table}.changeset, \n \ + c.closed_at \n \ + FROM {table} \n \ + LEFT JOIN changesets c ON c.id = {table}.changeset \n \ + WHERE{fromDate}{toDate}{hashtag}{area}{tags}{order} \ )\ ".format( - "created_at >= '{0}'".format(dateFrom) if (dateFrom) else "1=1", - "AND created_at <= '{0}'".format(dateTo) if (dateTo) else "", - "AND status = '{0}'".format(status) if (status) else "", - "AND " + hashtagQueryFilter(hashtag, table) if hashtag else "", - "AND ST_Intersects(\"geom\", ST_GeomFromText('MULTIPOLYGON((({0})))', 4326) )".format(area) if area else "", - "AND (" + tagsQueryFilter(tags, table) + ")" if tags else "", - "AND " + orderBy + " IS NOT NULL ORDER BY " + orderBy + " DESC LIMIT " + str(RESULTS_PER_PAGE_LIST) + (" OFFSET {0}" \ - .format(page * RESULTS_PER_PAGE_LIST) if page else ""), - ).replace("WHERE 1=1 AND", "WHERE") + type=osmType.value, + geotype=geoType.value, + table=table.value, + fromDate=" AND closed_at >= '{dateFrom}'".format(dateFrom=params.dateFrom) if (params.dateFrom) else "", + toDate=" AND closed_at <= '{dateTo}'".format(dateTo=params.dateTo) if (params.dateTo) else "", + hashtag=" AND " + hashtagQueryFilter(params.hashtag, table.value) if params.hashtag else "", + area=" AND ST_Intersects(\"geom\", ST_GeomFromText('MULTIPOLYGON((({area})))', 4326) )" + .format( + area=params.area + ) if params.area else "", + tags=" AND (" + tagsQueryFilter(params.tags, table.value) + ")" if params.tags else "", + order=" AND {order} IS NOT NULL ORDER BY {order} DESC LIMIT {limit} OFFSET {offset}" + .format( + order=params.orderBy.value, + limit=RESULTS_PER_PAGE_LIST, + offset=params.page * RESULTS_PER_PAGE_LIST + ) if params.page is not None else " LIMIT {limit} OFFSET {offset}" + ).replace("WHERE AND", "WHERE") + if asJson: + return listQueryToJSON(query, params) return query -def queryToJSONAllFeatures(query, dateFrom, dateTo, orderBy): - query = "with predata AS (" + query + ") , \ - data as ( \ - select predata.type, geotype, predata.id, predata.timestamp, tags, status, predata.changeset, predata.created_at as created_at, lat, lon from predata \ - WHERE {0} {1} {2} \ - ),\ - t_features AS ( \ - SELECT to_jsonb(data) as feature from data \ +# Build queries for returning a list of features as a JSON response +def listQueryToJSON(query: str, params: ListFeaturesParamsDTO): + jsonQuery = "with predata AS \n ({query}) , \n \ + data as ( \n \ + select predata.type, \n \ + geotype, predata.id, \n \ + predata.timestamp, \n \ + tags, \n \ + predata.changeset, \n \ + predata.closed_at as closed_at, \n \ + lat, \n \ + lon \n \ + from predata \n \ + WHERE{date}{orderBy} \n \ + ),\n \ + t_features AS ( \n \ + SELECT to_jsonb(data) as feature from data \n \ ) SELECT jsonb_agg(t_features.feature) as result FROM t_features;" \ .format( - "created_at >= '{0}'".format(dateFrom) if (dateFrom) else "1=1", - "AND created_at <= '{0}'".format(dateTo) if (dateTo) else "", - "AND {0}{1} IS NOT NULL ORDER BY {0}{1} DESC".format("predata.",orderBy) if orderBy != "osm_id" else "ORDER BY id DESC", - ).replace("WHERE 1=1 AND", "WHERE") - return query + query=query, + date="closed_at >= '{dateFrom}' AND closed_at <= '{dateTo}'" + .format( + dateFrom=params.dateFrom if (params.dateFrom) else "", + dateTo=" AND closed_at <= '{dateTo}'".format(dateTo=params.dateTo) if (params.dateTo) else "" + ) if params.dateFrom and params.dateTo else "", + orderBy=" AND {orderBy} IS NOT NULL ORDER BY {orderBy} DESC" + .format( + orderBy=".".join(["predata",params.orderBy.value]) + ) if params.orderBy else "ORDER BY id DESC", + ).replace("WHERE AND", "WHERE") + if DEBUG: + print(jsonQuery) + return jsonQuery + +# Build queries for returning a raw features as a JSON (GeoJSON) response +def rawQueryToJSON(query: str, params: RawFeaturesParamsDTO): + jsonQuery = "with predata AS \n ({query}) , \n \ + t_features AS ( \ + SELECT jsonb_build_object( 'type', 'Feature', 'id', id, 'properties', to_jsonb(predata) \ + - 'geometry' , 'geometry', ST_AsGeoJSON(geometry)::jsonb ) AS feature FROM predata \ + ) SELECT jsonb_build_object( 'type', 'FeatureCollection', 'features', jsonb_agg(t_features.feature) ) \ + as result FROM t_features;" \ + .format( + query=query.replace(";","") + ) + if DEBUG: + print(jsonQuery) + return jsonQuery +# This class build and run queries for OSM Raw Data class Raw: def __init__(self,db): - self.underpassDB = db + self.db = db + # Get list of features def getList( self, - area, - tags, - hashtag, - dateFrom, - dateTo, - status, - orderBy, - page, - featureType + params: ListFeaturesParamsDTO, + featureType: GeoType = None, + asJson: bool = False ): - if featureType == "line": - return self.getLinesList( - area, - tags, - hashtag, - dateFrom, - dateTo, - status, - orderBy, - page - ) - elif featureType == "node": - return self.getNodesList( - area, - tags, - hashtag, - dateFrom, - dateTo, - status, - orderBy, - page - ) - elif featureType == "polygon": - return self.getPolygonsList( - area, - tags, - hashtag, - dateFrom, - dateTo, - status, - orderBy, - page - ) + if featureType == GeoType.lines: + return self.getLinesList(params, asJson=asJson) + elif featureType == GeoType.nodes: + return self.getNodesList(params, asJson=asJson) + elif featureType == GeoType.polygons: + return self.getPolygonsList(params, asJson=asJson) else: - return self.getAllList( - area, - tags, - hashtag, - dateFrom, - dateTo, - status, - orderBy, - page - ) + return self.getAllList(params, asJson=asJson) - def getFeatures( + # Get geometry features (lines, nodes, polygons or all) + async def getFeatures( self, - area, - tags, - hashtag, - dateFrom, - dateTo, - status, - page, - featureType + params: RawFeaturesParamsDTO, + featureType: GeoType = None, + asJson: bool = False ): if featureType == "line": - return self.getLines( - area, - tags, - hashtag, - dateFrom, - dateTo, - status, - page - ) + return self.getLines(params, asJson) elif featureType == "node": - return self.getNodes( - area, - tags, - hashtag, - dateFrom, - dateTo, - status, - page - ) + return self.getNodes(params, asJson) elif featureType == "polygon": - return self.getPolygons( - area, - tags, - hashtag, - dateFrom, - dateTo, - status, - page - ) + return self.getPolygons(params, asJson) else: - return self.getAll( - area, - tags, - hashtag, - dateFrom, - dateTo, - status, - page - ) + return await self.getAll(params, asJson) + # Get polygon features async def getPolygons( self, - area, - tags, - hashtag, - dateFrom, - dateTo, - status, - page + params: RawFeaturesParamsDTO, + asJson: bool = False ): + params.table = Table.polygons + result = await self.db.run(geoFeaturesQuery(params, asJson), asJson=asJson) + if asJson: + return result or {} + return deserializeTags(result) - return await self.underpassDB.run(geoFeaturesQuery( - area, - tags, - hashtag, - dateFrom, - dateTo, - page, - status, - "ways_poly" - )) - + # Get line features async def getLines( self, - area, - tags, - hashtag, - dateFrom, - dateTo, - status, - page + params: RawFeaturesParamsDTO, + asJson: bool = False ): - - return await self.underpassDB.run(geoFeaturesQuery( - area, - tags, - hashtag, - dateFrom, - dateTo, - page, - status, - "ways_line" - )) + params.table = Table.lines + result = await self.db.run(geoFeaturesQuery(params, asJson), asJson=asJson) + if asJson: + return result or {} + return deserializeTags(result) + # Get node features async def getNodes( self, - area, - tags, - hashtag, - dateFrom, - dateTo, - status, - page + params: RawFeaturesParamsDTO, + asJson: bool = False ): - - return await self.underpassDB.run(geoFeaturesQuery( - area, - tags, - hashtag, - dateFrom, - dateTo, - page, - status, - "nodes" - ), True) - - def getAll( + params.table = Table.nodes + result = await self.db.run(geoFeaturesQuery(params, asJson), asJson=asJson) + if asJson: + return result or {} + return deserializeTags(result) + + # Get all (polygon, line, node) features + async def getAll( self, - area, - tags, - hashtag, - dateFrom, - dateTo, - status, - page + params: RawFeaturesParamsDTO, + asJson: bool = False ): + if asJson: - polygons = self.getPolygons( - area, - tags, - hashtag, - dateFrom, - dateTo, - status, - page) - - lines = self.getLines( - area, - tags, - hashtag, - dateFrom, - dateTo, - status, - page) - - nodes = self.getNodes( - area, - tags, - hashtag, - dateFrom, - dateTo, - status, - page) - - result = {'type': 'FeatureCollection', 'features': []} - - if polygons and "features" in polygons and polygons['features']: - result['features'] = result['features'] + polygons['features'] - - if lines and "features" in lines and lines['features']: - result['features'] = result['features'] + lines['features'] - - elif nodes and "features" in nodes and nodes['features']: - result['features'] = result['features'] + nodes['features'] - - return result + polygons = json.loads(await self.getPolygons(params, asJson)) + lines = json.loads(await self.getLines(params, asJson)) + nodes = json.loads(await self.getNodes(params, asJson)) - async def getPolygonsList( - self, - area, - tags, - hashtag, - dateFrom, - dateTo, - status, - orderBy, - page - ): + jsonResult = {'type': 'FeatureCollection', 'features': []} - queryPolygons = listAllFeaturesQuery( - area, - tags, - hashtag, - status, - orderBy or "ways_poly.osm_id", - page or 0, - dateFrom, - dateTo, - "ways_poly") - - query = queryToJSONAllFeatures( - " UNION ".join([queryPolygons]), - dateFrom, - dateTo, - orderBy or "osm_id" - ) - return await self.underpassDB.run(query) + if polygons and "features" in polygons and polygons['features']: + jsonResult['features'] = jsonResult['features'] + polygons['features'] + + if lines and "features" in lines and lines['features']: + jsonResult['features'] = jsonResult['features'] + lines['features'] + + elif nodes and "features" in nodes and nodes['features']: + jsonResult['features'] = jsonResult['features'] + nodes['features'] + + # elif relations and "features" in relations and relations['features']: + # result['features'] = result['features'] + relations['features'] + + result = json.dumps(jsonResult) + return result + + else: + polygons = await self.getPolygons(params) + lines = await self.getLines(params) + nodes = await self.getNodes(params) + result = [polygons, lines, nodes] + return result + # Get a list of line features async def getLinesList( self, - area, - tags, - hashtag, - dateFrom, - dateTo, - status, - orderBy, - page + params: ListFeaturesParamsDTO, + asJson: bool = False ): + params.table = "lines" + result = await self.db.run(listFeaturesQuery(params, asJson), asJson=asJson) + if asJson: + return result + return deserializeTags(result) - queryLines = listAllFeaturesQuery( - area, - tags, - hashtag, - status, - orderBy or "ways_line.osm_id", - page or 0, - dateFrom, - dateTo, - "ways_line") - - query = queryToJSONAllFeatures( - " UNION ".join([queryLines]), - dateFrom, - dateTo, - orderBy or "osm_id" - ) - return await self.underpassDB.run(query) - + # Get a list of node features async def getNodesList( self, - area, - tags, - hashtag, - dateFrom, - dateTo, - status, - orderBy, - page + params: ListFeaturesParamsDTO, + asJson: bool = False ): + params.table = "nodes" + result = await self.db.run(listFeaturesQuery(params, asJson), asJson=asJson) + if asJson: + return result + return deserializeTags(result) + + # Get a list of polygon features + async def getPolygonsList( + self, + params: ListFeaturesParamsDTO, + asJson: bool = False + ): + params.table = "polygons" + result = await self.db.run(listFeaturesQuery(params, asJson), asJson=asJson) + if asJson: + return result + return deserializeTags(result) - queryNodes = listAllFeaturesQuery( - area, - tags, - hashtag, - status, - orderBy or "nodes.osm_id", - page or 0, - dateFrom, - dateTo, - "nodes") - - query = queryToJSONAllFeatures( - " UNION ".join([queryNodes]), - dateFrom, - dateTo, - orderBy or "osm_id" - ) - return await self.underpassDB.run(query) + # Get a list of all features async def getAllList( self, - area, - tags, - hashtag, - dateFrom, - dateTo, - status, - orderBy, - page + params: ListFeaturesParamsDTO, + asJson: bool = False ): - queryPolygons = listAllFeaturesQuery( - area, - tags, - hashtag, - status, - orderBy or "ways_poly.osm_id", - page or 0, - dateFrom, - dateTo, - "ways_poly") - - queryLines = listAllFeaturesQuery( - area, - tags, - hashtag, - status, - orderBy or "ways_line.osm_id", - page or 0, - dateFrom, - dateTo, - "ways_line") - - queryNodes = listAllFeaturesQuery( - area, - tags, - hashtag, - status, - orderBy or "nodes.osm_id", - page or 0, - dateFrom, - dateTo, - "nodes") - - query = queryToJSONAllFeatures( - " UNION ".join([queryPolygons, queryLines, queryNodes]), - dateFrom, - dateTo, - orderBy or "osm_id" - ) - return await self.underpassDB.run(query) + params.table = "polygons" + queryPolygons = listFeaturesQuery(params, asJson=False) + params.table = "lines" + queryLines = listFeaturesQuery(params, asJson=False) + params.table = "nodes" + queryNodes = listFeaturesQuery(params, asJson=False) + + # Combine queries for each geometry in a single query + if asJson: + query = listQueryToJSON( + " UNION ".join([queryPolygons, queryLines, queryNodes]), + params + ) + else: + query = " UNION ".join([queryPolygons, queryLines, queryNodes]) + + result = await self.db.run(query, asJson=asJson) + if asJson: + return result + return deserializeTags(result) diff --git a/python/dbapi/api/rawValidation.py b/python/dbapi/api/rawValidation.py new file mode 100644 index 000000000..0ac4d4fb9 --- /dev/null +++ b/python/dbapi/api/rawValidation.py @@ -0,0 +1,401 @@ + +#!/usr/bin/python3 +# +# Copyright (c) 2024 Humanitarian OpenStreetMap Team +# +# This file is part of Underpass. +# +# Underpass is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# Underpass is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with Underpass. If not, see . + +# Build and run queries for getting validation results +# from the Underpass validation table in combination with raw OSM data +# +# This file requires to have both OSM Raw Data and Underpass tables +# into the same database. + +from dataclasses import dataclass +from enum import Enum +from .sharedTypes import Table, GeoType +from .filters import tagsQueryFilter, hashtagQueryFilter +from .serialization import queryToJSON +from .config import RESULTS_PER_PAGE, RESULTS_PER_PAGE_LIST, DEBUG +from .raw import RawFeaturesParamsDTO, ListFeaturesParamsDTO, rawQueryToJSON, listQueryToJSON, OrderBy +from .serialization import deserializeTags +import json + + +# Validation errorrs +class ValidationError(Enum): + notags = "notags" + complete = "complete" + incomplete = "incomplete" + badvalue = "badvalue" + correct = "correct" + badgeom = "badgeom" + orphan = "orphan" + overlapping = "overlapping" + duplicate = "duplicate" + valid = "valid" + +# OSM types +class OsmType(Enum): + nodes = "node" + lines = "way" + polygons = "way" + +# Validation Count parameters DTO +@dataclass +class ValidationCountParamsDTO: + status: ValidationError + tags: list[str] = None + hashtag: str = "" + dateFrom: str = "" + dateTo: str = "" + area: str = None + table: Table = Table.nodes + +@dataclass +class RawValidationFeaturesParamsDTO(RawFeaturesParamsDTO): + status: ValidationError = None + +@dataclass +class ListValidationFeaturesParamsDTO(ListFeaturesParamsDTO): + status: ValidationError = None + +# Build queries for counting validation data +def countQuery( + params: ValidationCountParamsDTO, + asJson: bool = False): + + query = "with all_features as ( \ + select {table}.osm_id from {table} \ + left join changesets c on changeset = c.id \ + WHERE{dateFrom}{dateTo}{area}{tags}{hashtag} \ + ), \ + count_validated_features as ( \ + select count(distinct(all_features.osm_id)) as count from all_features \ + left join validation v on all_features.osm_id = v.osm_id \ + where v.status = '{status}' \ + ), count_features as (\ + select count(distinct(all_features.osm_id)) as total from all_features \ + ) \ + select count, total from count_validated_features, count_features".format( + table=params.table.value, + dateFrom=" AND closed_at >= '{dateFrom}'".format(dateFrom=params.dateFrom) if (params.dateFrom) else "", + dateTo=" AND closed_at <= '{dateTo}'".format(dateTo=params.dateTo) if (params.dateTo) else "", + area=" AND ST_Intersects(\"geom\", ST_GeomFromText('MULTIPOLYGON((({area})))', 4326) )".format(area=params.area) if params.area else "", + tags=" AND (" + tagsQueryFilter(params.tags, params.table.value) + ") \n" if params.tags else "", + hashtag=" AND " + hashtagQueryFilter(params.hashtag, params.table.value) if params.hashtag else "", + status=params.status.value + ).replace("WHERE AND", "WHERE") + + if asJson: + return queryToJSON(query) + return query + +# Build queries for getting geometry features +def geoFeaturesQuery(params: RawValidationFeaturesParamsDTO, asJson: bool = False): + geoType:GeoType = GeoType[params.table.name] + query = "SELECT '{type}' as type, \ + {table}.osm_id as id, \n \ + {table}.timestamp, \n \ + ST_AsText(geom) as geometry, \n \ + tags, \n \ + status, \n \ + hashtags, \n \ + editor, \n \ + closed_at \n \ + FROM {table} \n \ + LEFT JOIN changesets c ON c.id = {table}.changeset \n \ + LEFT JOIN validation ON validation.osm_id = {table}.osm_id \ + WHERE{area}{tags}{hashtag}{date}{status} {limit}; \n \ + ".format( + type=geoType.value, + table=params.table.value, + area=" AND ST_Intersects(\"geom\", ST_GeomFromText('MULTIPOLYGON((({area})))', 4326) ) \n" + .format(area=params.area) if params.area else "", + tags=" AND (" + tagsQueryFilter(params.tags, params.table.value) + ") \n" if params.tags else "", + hashtag=" AND " + hashtagQueryFilter(params.hashtag, params.table.value) if params.hashtag else "", + date=" AND closed_at >= {dateFrom} AND closed_at <= {dateTo}\n" + .format(dateFrom=params.dateFrom, dateTo=params.dateTo) + if params.dateFrom and params.dateTo else "\n", + status=" AND status = '{status}'".format(status=params.status.value) if (params.status) else "", + limit=" LIMIT {limit}".format(limit=RESULTS_PER_PAGE), + ).replace("WHERE AND", "WHERE") + if asJson: + return rawQueryToJSON(query, params) + + return query + + +# Build queries for getting list of features +def listFeaturesQuery( + params: ListValidationFeaturesParamsDTO, + asJson: bool = False + ): + + geoType:GeoType = GeoType[params.table] + osmType:OsmType = OsmType[params.table] + table:Table = Table[params.table] + orderBy:OrderBy = OrderBy[params.orderBy] + + query = "( \ + SELECT '{type}' as type, \n \ + '{geotype}' as geotype, \n \ + {table}.osm_id as id, \n \ + ST_X(ST_Centroid(geom)) as lat, \n \ + ST_Y(ST_Centroid(geom)) as lon, \n \ + {table}.timestamp, \n \ + tags, \n \ + {table}.changeset, \n \ + c.closed_at, \n \ + status \n \ + FROM {table} \n \ + LEFT JOIN changesets c ON c.id = {table}.changeset \n \ + LEFT JOIN validation v ON v.osm_id = {table}.osm_id \n \ + WHERE{fromDate}{toDate}{hashtag}{area}{tags}{status}{order} \ + )\ + ".format( + type=osmType.value, + geotype=geoType.value, + table=table.value, + fromDate=" AND closed_at >= '{dateFrom}'".format(dateFrom=params.dateFrom) if (params.dateFrom) else "", + toDate=" AND closed_at <= '{dateTo}'".format(dateTo=params.dateTo) if (params.dateTo) else "", + hashtag=" AND " + hashtagQueryFilter(params.hashtag, table.value) if params.hashtag else "", + area=" AND ST_Intersects(\"geom\", ST_GeomFromText('MULTIPOLYGON((({area})))', 4326) )" + .format( + area=params.area + ) if params.area else "", + tags=" AND (" + tagsQueryFilter(params.tags, table.value) + ")" if params.tags else "", + status=" AND status = '{status}'".format(status=params.status.value) if (params.status) else "", + order=" AND {order} IS NOT NULL ORDER BY {order} DESC LIMIT {limit} OFFSET {offset}" + .format( + order=orderBy.value, + limit=RESULTS_PER_PAGE_LIST, + offset=params.page * RESULTS_PER_PAGE_LIST + ) if params.page is not None else " LIMIT {limit} OFFSET {offset}" + ).replace("WHERE AND", "WHERE") + if asJson: + return listQueryToJSON(query, params) + return query + +# This class build and run queries for Validation Data +class RawValidation: + def __init__(self,db): + self.db = db + + # Get count of validation errors + async def getNodesCount( + self, + params: ValidationCountParamsDTO, + asJson: bool = False + ): + params.table = Table.nodes + query = countQuery(params,asJson=asJson) + return(await self.db.run(query, asJson=asJson, singleObject=True)) + + async def getLinesCount( + self, + params: ValidationCountParamsDTO, + asJson: bool = False + ): + params.table = Table.lines + query = countQuery(params,asJson=asJson) + return(await self.db.run(query, asJson=asJson, singleObject=True)) + + async def getPolygonsCount( + self, + params: ValidationCountParamsDTO, + asJson: bool = False + ): + params.table = Table.polygons + query = countQuery(params,asJson=asJson) + return(await self.db.run(query, asJson=asJson, singleObject=True)) + + async def getCount( + self, + params: ValidationCountParamsDTO, + asJson: bool = False + ): + + params.table = Table.nodes + queryNodes = countQuery(params) + nodesCount = await self.db.run(queryNodes, singleObject=True) + + params.table = Table.lines + queryLines = countQuery(params) + linesCount = await self.db.run(queryLines, singleObject=True) + + params.table = Table.polygons + queryPolygons = countQuery(params) + polygonsCount = await self.db.run(queryPolygons, singleObject=True) + + result = { + "total": nodesCount['total'] + linesCount['total'] + + polygonsCount['total'], + "count": nodesCount['count'] + linesCount['count'] + + polygonsCount['count'] + } + + return(result) + + # Get geometry features (lines, nodes, polygons or all) + async def getFeatures( + self, + params: RawFeaturesParamsDTO, + featureType: GeoType = None, + asJson: bool = False + ): + if featureType == "line": + return self.getLines(params, asJson) + elif featureType == "node": + return self.getNodes(params, asJson) + elif featureType == "polygon": + return self.getPolygons(params, asJson) + else: + return await self.getAll(params, asJson) + + # Get polygon features + async def getPolygons( + self, + params: RawValidationFeaturesParamsDTO, + asJson: bool = False + ): + params.table = Table.polygons + result = await self.db.run(geoFeaturesQuery(params, asJson), asJson=asJson) + if asJson: + return result + return deserializeTags(result) + + + # Get line features + async def getLines( + self, + params: RawValidationFeaturesParamsDTO, + asJson: bool = False + ): + params.table = Table.lines + result = await self.db.run(geoFeaturesQuery(params, asJson), asJson=asJson) + if asJson: + return result + return deserializeTags(result) + + # Get node features + async def getNodes( + self, + params: RawValidationFeaturesParamsDTO, + asJson: bool = False + ): + params.table = Table.nodes + result = await self.db.run(geoFeaturesQuery(params, asJson), asJson=asJson) + if asJson: + return result + return deserializeTags(result) + + # Get all (polygon, line, node) features + async def getAll( + self, + params: RawValidationFeaturesParamsDTO, + asJson: bool = False + ): + if asJson: + + polygons = json.loads(await self.getPolygons(params, asJson)) + lines = json.loads(await self.getLines(params, asJson)) + nodes = json.loads(await self.getNodes(params, asJson)) + + jsonResult = {'type': 'FeatureCollection', 'features': []} + + if polygons and "features" in polygons and polygons['features']: + jsonResult['features'] = jsonResult['features'] + polygons['features'] + + if lines and "features" in lines and lines['features']: + jsonResult['features'] = jsonResult['features'] + lines['features'] + + elif nodes and "features" in nodes and nodes['features']: + jsonResult['features'] = jsonResult['features'] + nodes['features'] + + # elif relations and "features" in relations and relations['features']: + # result['features'] = result['features'] + relations['features'] + + result = json.dumps(jsonResult) + + else: + polygons = await self.getPolygons(params, asJson) + lines = await self.getLines(params, asJson) + nodes = await self.getNodes(params, asJson) + result = polygons + lines + nodes + + return result + + # Get a list of line features + async def getLinesList( + self, + params: ListValidationFeaturesParamsDTO, + asJson: bool = False + ): + params.table = "lines" + result = await self.db.run(listFeaturesQuery(params, asJson), asJson=asJson) + if asJson: + return result + return deserializeTags(result) + + # Get a list of node features + async def getNodesList( + self, + params: ListValidationFeaturesParamsDTO, + asJson: bool = False + ): + params.table = "nodes" + result = await self.db.run(listFeaturesQuery(params, asJson), asJson=asJson) + if asJson: + return result + return deserializeTags(result) + + # Get a list of polygon features + async def getPolygonsList( + self, + params: ListValidationFeaturesParamsDTO, + asJson: bool = False + ): + params.table = "polygons" + result = await self.db.run(listFeaturesQuery(params, asJson), asJson=asJson) + if asJson: + return result + return deserializeTags(result) + + # Get a list of all features + async def getList( + self, + params: ListValidationFeaturesParamsDTO, + asJson: bool = False + ): + params.table = "polygons" + queryPolygons = listFeaturesQuery(params, asJson=False) + params.table = "lines" + queryLines = listFeaturesQuery(params, asJson=False) + params.table = "nodes" + queryNodes = listFeaturesQuery(params, asJson=False) + + # Combine queries for each geometry in a single query + if asJson: + query = listQueryToJSON( + " UNION ".join([queryPolygons, queryLines, queryNodes]), + params + ) + else: + query = " UNION ".join([queryPolygons, queryLines, queryNodes]) + + result = await self.db.run(query, asJson=asJson) + if asJson: + return result + return deserializeTags(result) diff --git a/python/dbapi/example/raw-data.py b/python/dbapi/api/serialization.py similarity index 58% rename from python/dbapi/example/raw-data.py rename to python/dbapi/api/serialization.py index a7000c1f4..c6029a66c 100644 --- a/python/dbapi/example/raw-data.py +++ b/python/dbapi/api/serialization.py @@ -1,5 +1,7 @@ + +#!/usr/bin/python3 # -# Copyright (c) 2023 Humanitarian OpenStreetMap Team +# Copyright (c) 2024 Humanitarian OpenStreetMap Team # # This file is part of Underpass. # @@ -16,23 +18,20 @@ # You should have received a copy of the GNU General Public License # along with Underpass. If not, see . -import sys,os -sys.path.append(os.path.realpath('..')) - -from api import raw -from api.db import UnderpassDB - -db = UnderpassDB("postgresql://localhost/underpass") -db.connect() -rawer = raw.Raw(db) +import json -results = rawer.getNodes( - area = "-180 90,180 90, 180 -90, -180 -90,-180 90", - tags = "building=yes", - hashtag = "", - dateFrom = "", - dateTo = "", - page = 0 -) +def queryToJSON(query: str): + jsonQuery = "with data AS \n ({query}) \n \ + SELECT to_jsonb(data) as result from data;" \ + .format(query=query) + return jsonQuery -print(results) +def deserializeTags(data): + result = [] + if data: + for row in data: + row_dict = dict(row) + if 'tags' in row: + row_dict['tags'] = json.loads(row['tags']) + result.append(row_dict) + return result diff --git a/python/dbapi/api/sharedTypes.py b/python/dbapi/api/sharedTypes.py new file mode 100644 index 000000000..847fae09b --- /dev/null +++ b/python/dbapi/api/sharedTypes.py @@ -0,0 +1,14 @@ +from enum import Enum + +# DB table names +class Table(Enum): + nodes = "nodes" + lines = "ways_line" + polygons = "ways_poly" + relations = "relations" + +# Geometry types +class GeoType(Enum): + polygons = "Polygon" + lines = "LineString" + nodes = "Node" \ No newline at end of file diff --git a/python/dbapi/api/stats.py b/python/dbapi/api/stats.py index 8338f2d7c..698d361c6 100644 --- a/python/dbapi/api/stats.py +++ b/python/dbapi/api/stats.py @@ -17,62 +17,101 @@ # You should have received a copy of the GNU General Public License # along with Underpass. If not, see . +# Build and run queries for getting statistics + +from dataclasses import dataclass from .filters import tagsQueryFilter, hashtagQueryFilter +from .sharedTypes import Table, GeoType +from .serialization import queryToJSON +import json + +# Stats parameters DTO +@dataclass +class StatsParamsDTO: + tags: list[str] = None + hashtag: str = "" + dateFrom: str = "" + dateTo: str = "" + area: str = None + table: Table = Table.nodes + +def featureCountQuery(params: StatsParamsDTO, asJson: bool = False): + query = "select count(distinct {table}.osm_id) AS count FROM {table} \ + LEFT JOIN changesets c ON changeset = c.id \ + WHERE{area}{tags}{hashtag}{date}".format( + table=params.table.value, + area=" AND ST_Intersects(\"geom\", ST_GeomFromText('MULTIPOLYGON((({area})))', 4326) ) \n" + .format(area=params.area) if params.area else "", + tags=" AND (" + tagsQueryFilter(params.tags, params.table.value) + ") \n" if params.tags else "", + hashtag=" AND " + hashtagQueryFilter(params.hashtag, params.table.value) if params.hashtag else "", + date=" AND closed_at >= {dateFrom} AND closed_at <= {dateTo}\n" + .format(dateFrom=params.dateFrom, dateTo=params.dateTo) + if params.dateFrom and params.dateTo else "\n" + ).replace("WHERE AND", "WHERE") + if asJson: + return queryToJSON(query) + return query class Stats: def __init__(self, db): - self.underpassDB = db + self.db = db + + async def getNodesCount( + self, + params: StatsParamsDTO, + asJson: bool = False + ): + params.table = Table.nodes + result = await self.db.run(featureCountQuery(params), singleObject=True) + if asJson: + return json.dumps(dict(result)) + return result + + async def getLinesCount( + self, + params: StatsParamsDTO, + asJson: bool = False + ): + params.table = Table.lines + result = await self.db.run(featureCountQuery(params), singleObject=True) + if asJson: + return json.dumps(dict(result)) + return result + + async def getPolygonsCount( + self, + params: StatsParamsDTO, + asJson: bool = False + ): + params.table = Table.polygons + result = await self.db.run(featureCountQuery(params), singleObject=True) + if asJson: + return json.dumps(dict(result)) + return result + async def getCount( self, - area = None, - tags = None, - hashtag = None, - dateFrom = None, - dateTo = None, - status = None, - featureType = None, + params: StatsParamsDTO, + asJson: bool = False ): - if featureType == "line": - table = "ways_line" - elif featureType == "node": - table = "nodes" - else: - table = "ways_poly" - if status: - query = "with all_features as ( \ - select {0}.osm_id from {0} \ - left join changesets c on changeset = c.id \ - where {1} {2} {3} {4} {5} \ - ), \ - count_validated_features as ( \ - select count(distinct(all_features.osm_id)) as count from all_features \ - left join validation v on all_features.osm_id = v.osm_id \ - where v.status = '{6}' \ - ), count_features as (\ - select count(distinct(all_features.osm_id)) as total from all_features \ - ) \ - select count, total from count_validated_features, count_features".format( - table, - "created_at >= '{0}'".format(dateFrom) if (dateFrom) else "1=1", - "AND created_at <= '{0}'".format(dateTo) if (dateTo) else "", - "AND ST_Intersects(\"geom\", ST_GeomFromText('MULTIPOLYGON((({0})))', 4326) )".format(area) if area else "", - "AND (" + tagsQueryFilter(tags, table) + ")" if tags else "", - "AND " + hashtagQueryFilter(hashtag, table) if hashtag else "", - status - ) - else: - query = "select count(distinct {0}.osm_id) as count from {0} \ - left join changesets c on changeset = c.id \ - where {1} {2} {3} {4} {5}".format( - table, - "created_at >= '{0}'".format(dateFrom) if (dateFrom) else "1=1", - "AND created_at <= '{0}'".format(dateTo) if (dateTo) else "", - "AND ST_Intersects(\"geom\", ST_GeomFromText('MULTIPOLYGON((({0})))', 4326) )".format(area) if area else "", - "AND (" + tagsQueryFilter(tags, table) + ")" if tags else "", - "AND " + hashtagQueryFilter(hashtag, table) if hashtag else "" - ) - return(await self.underpassDB.run(query, True)) + params.table = Table.nodes + queryNodes = featureCountQuery(params) + + params.table = Table.lines + queryLines = featureCountQuery(params) + + params.table = Table.polygons + queryPolygons = featureCountQuery(params) + + query = "SELECT ({queries}) AS count;".format(queries=" + ".join([ + "({queryPolygons})".format(queryPolygons=queryPolygons), + "({queryLines})".format(queryLines=queryLines), + "({queryNodes})".format(queryNodes=queryNodes) + ])) - \ No newline at end of file + result = await self.db.run(query, singleObject=True) + if asJson: + return json.dumps(dict(result)) + return result diff --git a/python/dbapi/examples/raw.py b/python/dbapi/examples/raw.py new file mode 100644 index 000000000..048d5f017 --- /dev/null +++ b/python/dbapi/examples/raw.py @@ -0,0 +1,42 @@ +# +# Copyright (c) 2023, 2024 Humanitarian OpenStreetMap Team +# +# This file is part of Underpass. +# +# Underpass is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# Underpass is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with Underpass. If not, see . + +import sys,os +import asyncio +sys.path.append(os.path.realpath('..')) + +from api import raw +from api.db import DB + +async def main(): + + db = DB() + await db.connect() + rawer = raw.Raw(db) + + # Get List of OSM features for Nodes + print( + await rawer.getNodes(raw.ListFeaturesParamsDTO( + area = "-64.28176188601022 -31.34986467833471,-64.10910770217268 -31.3479682248434,-64.10577675328835 -31.47636641835701,-64.28120672786282 -31.47873373712735,-64.28176188601022 -31.34986467833471", + tags = "amenity=hospital" + ), asJson=True) + ) + +asyncio.run(main()) + + diff --git a/python/dbapi/examples/rawValidation.py b/python/dbapi/examples/rawValidation.py new file mode 100644 index 000000000..0def2d4ed --- /dev/null +++ b/python/dbapi/examples/rawValidation.py @@ -0,0 +1,43 @@ +# +# Copyright (c) 2023, 2024 Humanitarian OpenStreetMap Team +# +# This file is part of Underpass. +# +# Underpass is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# Underpass is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with Underpass. If not, see . + +import sys,os +import asyncio +sys.path.append(os.path.realpath('..')) + +from api import rawValidation +from api.db import DB + +async def main(): + + db = DB() + await db.connect() + rawval = rawValidation.RawValidation(db) + + # Get List of Raw Validation OSM features for Polygons + print( + await rawval.getPolygons(rawValidation.ListValidationFeaturesParamsDTO( + area = "-64.28176188601022 -31.34986467833471,-64.10910770217268 -31.3479682248434,-64.10577675328835 -31.47636641835701,-64.28120672786282 -31.47873373712735,-64.28176188601022 -31.34986467833471", + tags = "building=yes", + status = rawValidation.ValidationError.badgeom + ), asJson=True) + ) + +asyncio.run(main()) + + diff --git a/python/dbapi/examples/stats.py b/python/dbapi/examples/stats.py new file mode 100644 index 000000000..8cca26b5b --- /dev/null +++ b/python/dbapi/examples/stats.py @@ -0,0 +1,42 @@ +# +# Copyright (c) 2024 Humanitarian OpenStreetMap Team +# +# This file is part of Underpass. +# +# Underpass is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# Underpass is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with Underpass. If not, see . + +import sys,os +import asyncio +sys.path.append(os.path.realpath('..')) + +from api import stats as StatsApi +from api.db import DB + +async def main(): + + db = DB() + await db.connect() + stats = StatsApi.Stats(db) + + # Get List of OSM features for Nodes + print( + await stats.getCount(StatsApi.StatsParamsDTO( + area = "-64.28176188601022 -31.34986467833471,-64.10910770217268 -31.3479682248434,-64.10577675328835 -31.47636641835701,-64.28120672786282 -31.47873373712735,-64.28176188601022 -31.34986467833471", + tags = "amenity=hospital" + ), asJson=True) + ) + +asyncio.run(main()) + + diff --git a/python/restapi/config-docker.py b/python/restapi/config-docker.py index 150ab6c99..099216afa 100644 --- a/python/restapi/config-docker.py +++ b/python/restapi/config-docker.py @@ -1,2 +1,3 @@ # ENABLE_UNDERPASS_CORE=True -UNDERPASS_DB="postgresql://underpass:underpass@postgis/underpass" +UNDERPASS_DB="postgresql://underpass:underpass@underpass_db/underpass" +UNDERPASS_API_OSM_DB="postgresql://underpass:underpass@osm_db/osm" diff --git a/python/restapi/config.py b/python/restapi/config.py index 8bdfe850c..2d31a5a1f 100644 --- a/python/restapi/config.py +++ b/python/restapi/config.py @@ -1,6 +1,27 @@ +#!/usr/bin/python3 +# +# Copyright (c) 2023, 2024 Humanitarian OpenStreetMap Team +# +# This file is part of Underpass. +# +# Underpass is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# Underpass is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with Underpass. If not, see . + import os UNDERPASS_DB=os.getenv("UNDERPASS_API_DB") or "postgresql://localhost/underpass" +UNDERPASS_OSM_DB=os.getenv("UNDERPASS_API_DB") or "postgresql://localhost/underpass" +# UNDERPASS_OSM_DB=os.getenv("UNDERPASS_API_OSM_DB") or "postgresql://localhost/osm" ORIGINS = os.getenv("UNDERPASS_API_ORIGINS").split(",") if os.getenv("UNDERPASS_API_ORIGINS") else [ "http://localhost", "http://localhost:5000", diff --git a/python/restapi/main.py b/python/restapi/main.py index c0018a519..a19ae6fd8 100644 --- a/python/restapi/main.py +++ b/python/restapi/main.py @@ -37,13 +37,12 @@ # --data '{"fromDate": "2023-03-01T00:00:00"}' import sys,os -sys.path.append(os.path.realpath('../dbapi')) from fastapi import FastAPI +from pydantic import BaseModel from fastapi.middleware.cors import CORSMiddleware -from models import * -from api import raw, stats -from api.db import UnderpassDB +from models import StatsRequest, RawRequest, RawListRequest, RawValidationRequest, RawValidationListRequest, RawValidationStatsRequest +import raw, stats, rawval import config app = FastAPI() @@ -56,97 +55,122 @@ allow_headers=["*"] ) -db = UnderpassDB(config.UNDERPASS_DB) -rawer = raw.Raw(db) -statser = stats.Stats(db) - -@app.get("/") -async def index(): - return {"message": "This is the Underpass REST API."} - -@app.post("/raw/polygons") -async def getPolygons(request: RawRequest): - results = await rawer.getPolygons( - area = request.area or None, - tags = request.tags or "", - hashtag = request.hashtag or "", - dateFrom = request.dateFrom or "", - dateTo = request.dateTo or "", - status = request.status or "", - page = request.page - ) - return results - -@app.post("/raw/nodes") -async def getNodes(request: RawRequest): - results = await rawer.getNodes( - area = request.area, - tags = request.tags or "", - hashtag = request.hashtag or "", - dateFrom = request.dateFrom or "", - dateTo = request.dateTo or "", - status = request.status or "", - page = request.page - ) - return results - -@app.post("/raw/lines") -async def getLines(request: RawRequest): - results = await rawer.getLines( - area = request.area, - tags = request.tags or "", - hashtag = request.hashtag or "", - dateFrom = request.dateFrom or "", - dateTo = request.dateTo or "", - status = request.status or "", - page = request.page - ) - return results - -@app.post("/raw/features") -async def getRawFeatures(request: RawRequest): - results = await rawer.getFeatures( - area = request.area or None, - tags = request.tags or "", - hashtag = request.hashtag or "", - dateFrom = request.dateFrom or "", - dateTo = request.dateTo or "", - status = request.status or "", - page = request.page, - featureType = request.featureType or None - ) - return results - -@app.post("/raw/list") -async def getRawList(request: RawRequest): - results = await rawer.getList( - area = request.area or None, - tags = request.tags or "", - hashtag = request.hashtag or "", - dateFrom = request.dateFrom or "", - dateTo = request.dateTo or "", - status = request.status or "", - orderBy = request.orderBy or None, - page = request.page, - featureType = request.featureType or None - ) - return results - -@app.post("/stats/count") -async def getStatsCount(request: StatsRequest): - results = await statser.getCount( - area = request.area or None, - tags = request.tags or "", - hashtag = request.hashtag or "", - dateFrom = request.dateFrom or "", - dateTo = request.dateTo or "", - status = request.status or "", - featureType = request.featureType or None - ) - return results - -@app.get("/availability") -async def getAvailability(): - return { - "countries": config.AVAILABILITY - } +class Index: + @app.get("/") + async def index(): + return {"message": "This is the Underpass REST API."} + +class Config: +# Availability (which countries this API provides data) + # Ex: ["nepal", "argentina"] + @app.get("/availability") + async def getAvailability(): + return { + "countries": config.AVAILABILITY + } + +# Raw OSM Data + +class Raw: + @app.post("/raw/polygons") + async def polygons(request: RawRequest): + return await raw.polygons(request) + + @app.post("/raw/nodes") + async def nodes(request: RawRequest): + return await raw.nodes(request) + + @app.post("/raw/lines") + async def lines(request: RawRequest): + return await raw.lines(request) + + @app.post("/raw/features") + async def features(request: RawRequest): + return await raw.features(request) + + @app.post("/raw/list") + async def list(request: RawListRequest): + return await raw.list(request) + + @app.post("/raw/polygons/list") + async def polygons(request: RawListRequest): + return await raw.polygonsList(request) + + @app.post("/raw/nodes/list") + async def nodes(request: RawListRequest): + return await raw.nodesList(request) + + @app.post("/raw/lines/list") + async def lines(request: RawListRequest): + return await raw.linesList(request) + + +# Raw OSM Data and Validation + +class RawValidation: + @app.post("/raw-validation/polygons") + async def polygons(request: RawValidationRequest): + return await rawval.polygons(request) + + @app.post("/raw-validation/nodes") + async def nodes(request: RawValidationRequest): + return await rawval.nodes(request) + + @app.post("/raw-validation/lines") + async def lines(request: RawValidationRequest): + return await rawval.lines(request) + + @app.post("/raw-validation/features") + async def features(request: RawValidationRequest): + return await rawval.features(request) + + @app.post("/raw-validation/list") + async def list(request: RawValidationListRequest): + return await rawval.list(request) + + @app.post("/raw-validation/polygons/list") + async def polygons(request: RawValidationListRequest): + return await rawval.polygonsList(request) + + @app.post("/raw-validation/nodes/list") + async def nodes(request: RawValidationListRequest): + return await rawval.nodesList(request) + + @app.post("/raw-validation/lines/list") + async def lines(request: RawValidationListRequest): + return await rawval.linesList(request) + + @app.post("/raw-validation/stats") + async def lines(request: RawValidationStatsRequest): + return await rawval.count(request) + + @app.post("/raw-validation/stats/nodes") + async def lines(request: RawValidationStatsRequest): + return await rawval.nodesCount(request) + + @app.post("/raw-validation/stats/polygons") + async def lines(request: RawValidationStatsRequest): + return await rawval.polygonsCount(request) + + @app.post("/raw-validation/stats/lines") + async def lines(request: RawValidationStatsRequest): + return await rawval.linesCount(request) + +# Statistics + +class Stats: + @app.post("/stats/nodes") + async def nodes(request: StatsRequest): + return await stats.nodes(request) + + @app.post("/stats/lines") + async def lines(request: StatsRequest): + return await stats.lines(request) + + @app.post("/stats/polygons") + async def polygons(request: StatsRequest): + return await stats.polygons(request) + + @app.post("/stats/features") + async def features(request: StatsRequest): + return await stats.features(request) diff --git a/python/restapi/models.py b/python/restapi/models.py index 6de2d5ed1..e79def910 100644 --- a/python/restapi/models.py +++ b/python/restapi/models.py @@ -19,24 +19,40 @@ from pydantic import BaseModel from typing import Union - -class RawRequest(BaseModel): - area: Union[str, None] = None + +class Item(BaseModel): + name: str + +class BaseRequest(BaseModel): + area: str = None tags: str = None hashtag: str = None dateFrom: str = None dateTo: str = None - status: str = None - orderBy: str = None - page: int = None featureType: str = None -class StatsRequest(BaseModel): - area: Union[str, None] = None - tags: str = None - hashtag: str = None - dateFrom: str = None - dateTo: str = None +class BaseListRequest(BaseRequest): + orderBy: str = "id" + page: int = None + +class BaseRawValidationRequest(BaseRequest): status: str = None - featureType: str = None +class RawValidationListRequest(BaseRawValidationRequest): + orderBy: str = "id" + page: int = None + +class RawRequest(BaseRequest): + pass + +class RawListRequest(BaseListRequest): + pass + +class RawValidationRequest(BaseRawValidationRequest): + pass + +class StatsRequest(BaseRequest): + pass + +class RawValidationStatsRequest(BaseRawValidationRequest): + pass diff --git a/python/restapi/raw.py b/python/restapi/raw.py new file mode 100644 index 000000000..d63a2fc0e --- /dev/null +++ b/python/restapi/raw.py @@ -0,0 +1,127 @@ +#!/usr/bin/python3 +# +# Copyright (c) 2023, 2024 Humanitarian OpenStreetMap Team +# +# This file is part of Underpass. +# +# Underpass is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# Underpass is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with Underpass. If not, see . + +import sys,os +sys.path.append(os.path.realpath('../dbapi')) + +from models import RawRequest, RawListRequest +from api import raw as RawApi +from api.db import DB +import config +import json + +db = DB(config.UNDERPASS_OSM_DB) +raw = RawApi.Raw(db) + +async def polygons(request: RawRequest): + return json.loads(await raw.getPolygons( + RawApi.RawFeaturesParamsDTO( + area = request.area, + tags = request.tags, + hashtag = request.hashtag, + dateFrom = request.dateFrom, + dateTo = request.dateTo, + ), asJson=True) + ) + +async def nodes(request: RawRequest): + return json.loads(await raw.getNodes( + RawApi.RawFeaturesParamsDTO( + area = request.area, + tags = request.tags, + hashtag = request.hashtag, + dateFrom = request.dateFrom, + dateTo = request.dateTo, + ), asJson=True) + ) + +async def lines(request: RawRequest): + return json.loads(await raw.getLines( + RawApi.RawFeaturesParamsDTO( + area = request.area, + tags = request.tags, + hashtag = request.hashtag, + dateFrom = request.dateFrom, + dateTo = request.dateTo, + ), asJson=True) + ) + +async def features(request: RawRequest): + return json.loads(await raw.getFeatures( + RawApi.RawFeaturesParamsDTO( + area = request.area, + tags = request.tags, + hashtag = request.hashtag, + dateFrom = request.dateFrom, + dateTo = request.dateTo, + ), asJson=True) + ) + +async def polygonsList(request: RawListRequest): + return await raw.getPolygonsList( + RawApi.ListFeaturesParamsDTO( + area = request.area, + tags = request.tags, + hashtag = request.hashtag, + dateFrom = request.dateFrom, + dateTo = request.dateTo, + orderBy = request.orderBy, + page = request.page + ) + ) + +async def nodesList(request: RawListRequest): + return await raw.getNodesList( + RawApi.ListFeaturesParamsDTO( + area = request.area, + tags = request.tags, + hashtag = request.hashtag, + dateFrom = request.dateFrom, + dateTo = request.dateTo, + orderBy = request.orderBy, + page = request.page + ) + ) + +async def linesList(request: RawListRequest): + return await raw.getLinesList( + RawApi.ListFeaturesParamsDTO( + area = request.area, + tags = request.tags, + hashtag = request.hashtag, + dateFrom = request.dateFrom, + dateTo = request.dateTo, + orderBy = request.orderBy, + page = request.page + ) + ) + +async def list(request: RawListRequest): + return await raw.getList( + RawApi.ListFeaturesParamsDTO( + area = request.area, + tags = request.tags, + hashtag = request.hashtag, + dateFrom = request.dateFrom, + dateTo = request.dateTo, + orderBy = request.orderBy, + page = request.page + ) + ) + diff --git a/python/restapi/rawval.py b/python/restapi/rawval.py new file mode 100644 index 000000000..4480eae84 --- /dev/null +++ b/python/restapi/rawval.py @@ -0,0 +1,182 @@ +#!/usr/bin/python3 +# +# Copyright (c) 2023, 2024 Humanitarian OpenStreetMap Team +# +# This file is part of Underpass. +# +# Underpass is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# Underpass is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with Underpass. If not, see . + +import sys,os +sys.path.append(os.path.realpath('../dbapi')) + +from models import RawValidationRequest, RawValidationListRequest, RawValidationStatsRequest +from api import rawValidation as RawValidationApi +from api.db import DB +import config +import json + +db = DB(config.UNDERPASS_DB) +rawval = RawValidationApi.RawValidation(db) + +async def polygons(request: RawValidationRequest): + return json.loads(await rawval.getPolygons( + RawValidationApi.RawValidationFeaturesParamsDTO( + area = request.area, + tags = request.tags, + hashtag = request.hashtag, + dateFrom = request.dateFrom, + dateTo = request.dateTo, + status = RawValidationApi.ValidationError(request.status) if request.status else None + ), asJson=True) + ) + +async def nodes(request: RawValidationRequest): + return json.loads(await rawval.getNodes( + RawValidationApi.RawValidationFeaturesParamsDTO( + area = request.area, + tags = request.tags, + hashtag = request.hashtag, + dateFrom = request.dateFrom, + dateTo = request.dateTo, + status = RawValidationApi.ValidationError(request.status) if request.status else None + ), asJson=True) + ) + +async def lines(request: RawValidationRequest): + return json.loads(await rawval.getLines( + RawValidationApi.RawValidationFeaturesParamsDTO( + area = request.area, + tags = request.tags, + hashtag = request.hashtag, + dateFrom = request.dateFrom, + dateTo = request.dateTo, + status = RawValidationApi.ValidationError(request.status) if request.status else None + ), asJson=True) + ) + +async def features(request: RawValidationRequest): + return json.loads(await rawval.getFeatures( + RawValidationApi.RawValidationFeaturesParamsDTO( + area = request.area, + tags = request.tags, + hashtag = request.hashtag, + dateFrom = request.dateFrom, + dateTo = request.dateTo, + status = RawValidationApi.ValidationError(request.status) if request.status else None + ), asJson=True) + ) + +async def polygonsList(request: RawValidationListRequest): + return await rawval.getPolygonsList( + RawValidationApi.ListValidationFeaturesParamsDTO( + area = request.area, + tags = request.tags, + hashtag = request.hashtag, + dateFrom = request.dateFrom, + dateTo = request.dateTo, + orderBy = request.orderBy, + page = request.page, + status = RawValidationApi.ValidationError(request.status) if request.status else None + ) + ) + +async def nodesList(request: RawValidationListRequest): + return await rawval.getNodesList( + RawValidationApi.ListValidationFeaturesParamsDTO( + area = request.area, + tags = request.tags, + hashtag = request.hashtag, + dateFrom = request.dateFrom, + dateTo = request.dateTo, + orderBy = request.orderBy, + page = request.page, + status = RawValidationApi.ValidationError(request.status) if request.status else None + ) + ) + +async def linesList(request: RawValidationListRequest): + return await rawval.getLinesList( + RawValidationApi.ListValidationFeaturesParamsDTO( + area = request.area, + tags = request.tags, + hashtag = request.hashtag, + dateFrom = request.dateFrom, + dateTo = request.dateTo, + orderBy = request.orderBy, + page = request.page, + status = RawValidationApi.ValidationError(request.status) if request.status else None + ) + ) + +async def list(request: RawValidationListRequest): + return await rawval.getList( + RawValidationApi.ListValidationFeaturesParamsDTO( + area = request.area, + tags = request.tags, + hashtag = request.hashtag, + dateFrom = request.dateFrom, + dateTo = request.dateTo, + orderBy = request.orderBy, + page = request.page, + status = RawValidationApi.ValidationError(request.status) if request.status else None + ) + ) + +async def count(request: RawValidationStatsRequest): + return await rawval.getCount( + RawValidationApi.ValidationCountParamsDTO( + area = request.area, + tags = request.tags, + hashtag = request.hashtag, + dateFrom = request.dateFrom, + dateTo = request.dateTo, + status = RawValidationApi.ValidationError(request.status) if request.status else None + ) + ) + +async def nodesCount(request: RawValidationStatsRequest): + return await rawval.getNodesCount( + RawValidationApi.ValidationCountParamsDTO( + area = request.area, + tags = request.tags, + hashtag = request.hashtag, + dateFrom = request.dateFrom, + dateTo = request.dateTo, + status = RawValidationApi.ValidationError(request.status) if request.status else None + ) + ) + +async def linesCount(request: RawValidationStatsRequest): + return await rawval.getLinesCount( + RawValidationApi.ValidationCountParamsDTO( + area = request.area, + tags = request.tags, + hashtag = request.hashtag, + dateFrom = request.dateFrom, + dateTo = request.dateTo, + status = RawValidationApi.ValidationError(request.status) if request.status else None + ) + ) + +async def polygonsCount(request: RawValidationStatsRequest): + return await rawval.getPolygonsCount( + RawValidationApi.ValidationCountParamsDTO( + area = request.area, + tags = request.tags, + hashtag = request.hashtag, + dateFrom = request.dateFrom, + dateTo = request.dateTo, + status = RawValidationApi.ValidationError(request.status) if request.status else None + ) + ) diff --git a/python/restapi/stats.py b/python/restapi/stats.py new file mode 100644 index 000000000..e9c694cf3 --- /dev/null +++ b/python/restapi/stats.py @@ -0,0 +1,84 @@ +#!/usr/bin/python3 +# +# Copyright (c) 2023, 2024 Humanitarian OpenStreetMap Team +# +# This file is part of Underpass. +# +# Underpass is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# Underpass is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with Underpass. If not, see . + +import sys,os +sys.path.append(os.path.realpath('../dbapi')) + +from models import StatsRequest +from api import stats as StatsApi +from api.db import DB +import config + +db = DB(config.UNDERPASS_DB) +stats = StatsApi.Stats(db) + +def nodes(request: StatsRequest): + return stats.getNodesCount( + StatsApi.StatsParamsDTO( + area = request.area, + tags = request.tags, + hashtag = request.hashtag, + dateFrom = request.dateFrom, + dateTo = request.dateTo + ) + ) + +async def lines(request: StatsRequest): + return await stats.getLinesCount( + StatsApi.StatsParamsDTO( + area = request.area, + tags = request.tags, + hashtag = request.hashtag, + dateFrom = request.dateFrom, + dateTo = request.dateTo + ) + ) + +async def lines(request: StatsRequest): + return await stats.getLinesCount( + StatsApi.StatsParamsDTO( + area = request.area, + tags = request.tags, + hashtag = request.hashtag, + dateFrom = request.dateFrom, + dateTo = request.dateTo + ) + ) + +async def polygons(request: StatsRequest): + return await stats.getPolygonsCount( + StatsApi.StatsParamsDTO( + area = request.area, + tags = request.tags, + hashtag = request.hashtag, + dateFrom = request.dateFrom, + dateTo = request.dateTo + ) + ) + +async def features(request: StatsRequest): + return await stats.getCount( + StatsApi.StatsParamsDTO( + area = request.area, + tags = request.tags, + hashtag = request.hashtag, + dateFrom = request.dateFrom, + dateTo = request.dateTo + ) + ) diff --git a/setup/bootstrap.sh b/setup/bootstrap.sh index 035401cb5..68e180334 100755 --- a/setup/bootstrap.sh +++ b/setup/bootstrap.sh @@ -110,9 +110,9 @@ then python3 ../utils/poly2geojson.py $COUNTRY.poly if "$use_docker"; then - docker cp $COUNTRY.geojson underpass:/etc/underpass/priority.geojson + docker cp $COUNTRY.geojson underpass:/usr/local/etc/underpass/priority.geojson else - sudo cp $COUNTRY.geojson /etc/underpass/priority.geojson + cp $COUNTRY.geojson /usr/local/etc/underpass/priority.geojson fi echo "Bootstrapping database ..." if "$use_docker"; diff --git a/setup/db/indexes.sql b/setup/db/indexes.sql index d4646450c..ee5c5d641 100644 --- a/setup/db/indexes.sql +++ b/setup/db/indexes.sql @@ -2,10 +2,6 @@ CREATE UNIQUE INDEX nodes_id_idx ON public.nodes (osm_id DESC); CREATE UNIQUE INDEX ways_poly_id_idx ON public.ways_poly (osm_id DESC); CREATE UNIQUE INDEX ways_line_id_idx ON public.ways_line(osm_id DESC); CREATE UNIQUE INDEX relations_id_idx ON public.relations(osm_id DESC); -CREATE INDEX way_refs_node_id_idx ON public.way_refs (node_id); -CREATE INDEX way_refs_way_id_idx ON public.way_refs (way_id); -CREATE INDEX rel_refs_rel_id_idx ON public.rel_refs (rel_id); -CREATE INDEX rel_refs_way_id_idx ON public.rel_refs (way_id); CREATE INDEX nodes_version_idx ON public.nodes (version); CREATE INDEX ways_poly_version_idx ON public.ways_poly (version); diff --git a/setup/db/underpass.sql b/setup/db/underpass.sql index 3d30dfd4f..8ba98ac26 100644 --- a/setup/db/underpass.sql +++ b/setup/db/underpass.sql @@ -109,25 +109,10 @@ CREATE TABLE IF NOT EXISTS public.relations ( ALTER TABLE ONLY public.relations ADD CONSTRAINT relations_pkey PRIMARY KEY (osm_id); -CREATE TABLE IF NOT EXISTS public.way_refs ( - way_id int8, - node_id int8 -); - -CREATE TABLE IF NOT EXISTS public.rel_refs ( - rel_id int8, - way_id int8 -); - CREATE UNIQUE INDEX nodes_id_idx ON public.nodes (osm_id DESC); CREATE UNIQUE INDEX ways_poly_id_idx ON public.ways_poly (osm_id DESC); CREATE UNIQUE INDEX ways_line_id_idx ON public.ways_line(osm_id DESC); CREATE UNIQUE INDEX relations_id_idx ON public.relations(osm_id DESC); -CREATE INDEX way_refs_node_id_idx ON public.way_refs (node_id); -CREATE INDEX way_refs_way_id_idx ON public.way_refs (way_id); - -CREATE INDEX rel_refs_rel_id_idx ON public.rel_refs (rel_id); -CREATE INDEX rel_refs_way_id_idx ON public.rel_refs (way_id); CREATE INDEX nodes_version_idx ON public.nodes (version); CREATE INDEX ways_poly_version_idx ON public.ways_poly (version); diff --git a/src/bootstrap/bootstrap.cc b/src/bootstrap/bootstrap.cc index 869a94c8d..5b484a658 100644 --- a/src/bootstrap/bootstrap.cc +++ b/src/bootstrap/bootstrap.cc @@ -49,21 +49,33 @@ namespace bootstrap { Bootstrap::Bootstrap(void) {} -std::string +BootstrapQueries Bootstrap::allTasksQueries(std::shared_ptr> tasks) { - std::string queries = ""; + BootstrapQueries queries; for (auto it = tasks->begin(); it != tasks->end(); ++it) { - queries += it->query ; + for (auto itt = it->query.begin(); itt != it->query.end(); ++itt) { + queries.underpass.push_back(*itt); + } + for (auto itt = it->osmquery.begin(); itt != it->osmquery.end(); ++itt) { + queries.osm.push_back(*itt); + } } return queries; } void Bootstrap::start(const underpassconfig::UnderpassConfig &config) { - std::cout << "Connecting to the database ... " << std::endl; + std::cout << "Connecting to OSM database ... " << std::endl; + osmdb = std::make_shared(); + if (!osmdb->connect(config.underpass_osm_db_url)) { + log_error("Could not connect to OSM DB, aborting bootstrapping thread!"); + return; + } + + std::cout << "Connecting to underpass database ... " << std::endl; db = std::make_shared(); if (!db->connect(config.underpass_db_url)) { - std::cout << "Could not connect to Underpass DB, aborting bootstrapping thread!" << std::endl; + log_error("Could not connect to Underpass DB, aborting bootstrapping thread!"); return; } @@ -86,14 +98,14 @@ Bootstrap::start(const underpassconfig::UnderpassConfig &config) { validator = creator(); queryvalidate = std::make_shared(db); - queryraw = std::make_shared(db); + queryraw = std::make_shared(osmdb); page_size = config.bootstrap_page_size; concurrency = config.concurrency; norefs = config.norefs; processWays(); processNodes(); - // processRelations(); + processRelations(); } @@ -144,7 +156,15 @@ Bootstrap::processWays() { pool.join(); - db->query(allTasksQueries(tasks)); + auto queries = allTasksQueries(tasks); + + for (auto it = queries.underpass.begin(); it != queries.underpass.end(); ++it) { + db->query(*it); + } + for (auto it = queries.osm.begin(); it != queries.osm.end(); ++it) { + osmdb->query(*it); + } + lastid = ways->back().id; for (auto it = tasks->begin(); it != tasks->end(); ++it) { count += it->processed; @@ -193,7 +213,13 @@ Bootstrap::processNodes() { pool.join(); - db->query(allTasksQueries(tasks)); + auto queries = allTasksQueries(tasks); + for (auto it = queries.underpass.begin(); it != queries.underpass.end(); ++it) { + db->query(*it); + } + for (auto it = queries.osm.begin(); it != queries.osm.end(); ++it) { + osmdb->query(*it); + } lastid = nodes->back().id; for (auto it = tasks->begin(); it != tasks->end(); ++it) { count += it->processed; @@ -241,7 +267,13 @@ Bootstrap::processRelations() { pool.join(); - db->query(allTasksQueries(tasks)); + auto queries = allTasksQueries(tasks); + for (auto it = queries.underpass.begin(); it != queries.underpass.end(); ++it) { + db->query(*it); + } + for (auto it = queries.osm.begin(); it != queries.osm.end(); ++it) { + osmdb->query(*it); + } lastid = relations->back().id; for (auto it = tasks->begin(); it != tasks->end(); ++it) { count += it->processed; @@ -275,16 +307,15 @@ Bootstrap::threadBootstrapWayTask(WayTask wayTask) if (i < ways->size()) { auto way = ways->at(i); wayval->push_back(validator->checkWay(way, "building")); - // Fill the way_refs table - if (!norefs) { - for (auto ref = way.refs.begin(); ref != way.refs.end(); ++ref) { - task.query += "INSERT INTO way_refs (way_id, node_id) VALUES (" + std::to_string(way.id) + "," + std::to_string(*ref) + "); "; - } - } ++processed; } } - queryvalidate->ways(wayval, task.query); + + auto result = queryvalidate->ways(wayval); + for (auto it = result->begin(); it != result->end(); ++it) { + task.query.push_back(*it); + } + task.processed = processed; const std::lock_guard lock(tasks_change_mutex); (*tasks)[taskIndex] = task; @@ -320,7 +351,12 @@ Bootstrap::threadBootstrapNodeTask(NodeTask nodeTask) ++processed; } } - queryvalidate->nodes(nodeval, task.query); + + auto result = queryvalidate->nodes(nodeval); + for (auto it = result->begin(); it != result->end(); ++it) { + task.query.push_back(*it); + } + task.processed = processed; const std::lock_guard lock(tasks_change_mutex); (*tasks)[taskIndex] = task; @@ -341,7 +377,7 @@ Bootstrap::threadBootstrapRelationTask(RelationTask relationTask) BootstrapTask task; int processed = 0; - // auto relationval = std::make_shared>>(); + auto relationval = std::make_shared>>(); // Proccesing relations for (size_t i = taskIndex * page_size; i < (taskIndex + 1) * page_size; ++i) { @@ -350,7 +386,7 @@ Bootstrap::threadBootstrapRelationTask(RelationTask relationTask) // relationval->push_back(validator->checkRelation(way, "building")); // Fill the rel_refs table for (auto mit = relation.members.begin(); mit != relation.members.end(); ++mit) { - task.query += "INSERT INTO rel_refs (rel_id, way_id) VALUES (" + std::to_string(relation.id) + "," + std::to_string(mit->ref) + "); "; + task.osmquery.push_back("INSERT INTO rel_refs (rel_id, way_id) VALUES (" + std::to_string(relation.id) + "," + std::to_string(mit->ref) + "); "); } ++processed; } diff --git a/src/bootstrap/bootstrap.hh b/src/bootstrap/bootstrap.hh index 4142e400b..bbe859c28 100644 --- a/src/bootstrap/bootstrap.hh +++ b/src/bootstrap/bootstrap.hh @@ -1,5 +1,5 @@ // -// Copyright (c) 2023 Humanitarian OpenStreetMap Team +// Copyright (c) 2023, 2024 Humanitarian OpenStreetMap Team // // This file is part of Underpass. // @@ -33,10 +33,18 @@ namespace bootstrap { /// \struct BootstrapTask /// \brief Represents a bootstrap task struct BootstrapTask { - std::string query = ""; + std::vector query; + std::vector osmquery; int processed = 0; }; +/// \struct BootstrapQueries +/// \brief Represents a bootstrap queries list +struct BootstrapQueries { + std::vector underpass; + std::vector osm; +}; + struct WayTask { int taskIndex; std::shared_ptr> tasks; @@ -72,12 +80,13 @@ class Bootstrap { void threadBootstrapWayTask(WayTask wayTask); void threadBootstrapNodeTask(NodeTask nodeTask); void threadBootstrapRelationTask(RelationTask relationTask); - std::string allTasksQueries(std::shared_ptr> tasks); + BootstrapQueries allTasksQueries(std::shared_ptr> tasks); std::shared_ptr validator; std::shared_ptr queryvalidate; std::shared_ptr queryraw; std::shared_ptr db; + std::shared_ptr osmdb; bool norefs; unsigned int concurrency; unsigned int page_size; @@ -85,4 +94,4 @@ class Bootstrap { static std::mutex tasks_change_mutex; -} \ No newline at end of file +} diff --git a/src/data/pq.cc b/src/data/pq.cc index 72c7e8862..ea7af0ad4 100644 --- a/src/data/pq.cc +++ b/src/data/pq.cc @@ -1,5 +1,5 @@ // -// Copyright (c) 2020, 2021, 2023 Humanitarian OpenStreetMap Team +// Copyright (c) 2020, 2021, 2023, 2024 Humanitarian OpenStreetMap Team // // This file is part of Underpass. // @@ -152,29 +152,22 @@ Pq::query(const std::string &query) { std::scoped_lock write_lock{pqxx_mutex}; pqxx::work worker(*sdb); - auto result = worker.exec(query); - worker.commit(); + pqxx::result result; + try { + result = worker.exec(query); + worker.commit(); + } catch (std::exception &e) { + log_error("ERROR executing query %1%", e.what()); + // Return an empty result so higher level code can handle the error + return result; + } return result; } std::string Pq::escapedString(const std::string &s) { - std::string newstr; - size_t i = 0; - while (i < s.size()) { - // Single quote (') - if (s[i] == '\'') { - newstr += "''"; - // Slash (\) - } else if (s[i] == '\\') { - newstr += "\\\\"; - } else { - newstr += s[i]; - } - i++; - } - return sdb->esc(newstr); + return sdb->esc(s); } std::string @@ -182,26 +175,26 @@ Pq::escapedJSON(const std::string &s) { std::ostringstream o; for (auto c = s.cbegin(); c != s.cend(); c++) { switch (*c) { - case '\x00': o << "\\u0000"; break; - case '\x01': o << " "; break; - case '\x02': o << " "; break; - case '\x03': o << " "; break; - case '\x04': o << " "; break; - case '\x05': o << " "; break; - case '\x06': o << " "; break; - case '\x07': o << " "; break; - case '\x08': o << " "; break; - case '\x09': o << " "; break; - case '\x0a': o << "\\n"; break; - case '\x0b': o << " "; break; - case '\x0c': o << " "; break; - case '\x0d': o << " "; break; - case '\x0e': o << " "; break; - case '\xfe': o << " "; break; - case '\x1f': o << "\\u001f"; break; - case '\x22': o << "\\\""; break; - case '\x5c': o << "\\\\"; break; - default: o << *c; + case '\x00': o << "\\u0000"; break; + case '\x01': o << " "; break; + case '\x02': o << " "; break; + case '\x03': o << " "; break; + case '\x04': o << " "; break; + case '\x05': o << " "; break; + case '\x06': o << " "; break; + case '\x07': o << " "; break; + case '\x08': o << " "; break; + case '\x09': o << " "; break; + case '\x0a': o << "\n"; break; + case '\x0b': o << " "; break; + case '\x0c': o << " "; break; + case '\x0d': o << " "; break; + case '\x0e': o << " "; break; + case '\xfe': o << " "; break; + case '\x1f': o << "\\u001f"; break; + case '\x22': o << "\""; break; + case '\x5c': o << "\\"; break; + default: o << *c; } } return o.str(); diff --git a/src/osm/changeset.cc b/src/osm/changeset.cc index 8cce25684..955da6ca3 100644 --- a/src/osm/changeset.cc +++ b/src/osm/changeset.cc @@ -1,5 +1,5 @@ // -// Copyright (c) 2020, 2021, 2022, 2023 Humanitarian OpenStreetMap Team +// Copyright (c) 2020, 2021, 2022, 2023, 2024 Humanitarian OpenStreetMap Team // // This file is part of Underpass. // diff --git a/src/osm/osmchange.cc b/src/osm/osmchange.cc index d9f87ebe7..e5f5a4b9f 100644 --- a/src/osm/osmchange.cc +++ b/src/osm/osmchange.cc @@ -49,11 +49,7 @@ using namespace boost::gregorian; #include #include #include -// #include -// #include -// #include -// using boost::multi_index_container; -// using namespace boost::multi_index; +#include #include "validate/validate.hh" #include "osm/osmobjects.hh" @@ -136,260 +132,288 @@ OsmChangeFile::buildGeometriesFromNodeCache() { for (auto wit = std::begin(change->ways); wit != std::end(change->ways); ++wit) { osmobjects::OsmWay *way = wit->get(); for (auto lit = std::begin(way->refs); lit != std::end(way->refs); ++lit) { - boost::geometry::append(way->linestring, nodecache[*lit]); + bg::append(way->linestring, nodecache[*lit]); } if (way->isClosed()) { way->polygon = { {std::begin(way->linestring), std::end(way->linestring)} }; } std::stringstream ss; if (way->isClosed()) { - ss << std::setprecision(12) << boost::geometry::wkt(way->polygon); + ss << std::setprecision(12) << bg::wkt(way->polygon); } else { - ss << std::setprecision(12) << boost::geometry::wkt(way->linestring); + ss << std::setprecision(12) << bg::wkt(way->linestring); } waycache.insert(std::pair(way->id, std::make_shared(*way))); } - for (auto rit = std::begin(change->relations); rit != std::end(change->relations); ++rit) { - osmobjects::OsmRelation *relation = rit->get(); - buildRelationGeometry(*relation); - } } } -// FIXME -// TODO: refactor, divide this function into multiple parts +struct RelationGeometry { + linestring_t linestring; + polygon_t polygon; +}; + void OsmChangeFile::buildRelationGeometry(osmobjects::OsmRelation &relation) { - - // A way is missing from cache - bool noWay = false; - // There are not Way relation members - bool noWayMembers = true; - - bool first = true; - std::string innerGeoStr; - std::string geometry_str; - std::string linestring_tmp; - bool using_linestring_tmp = false; + + std::vector parts_inner; + std::vector parts_outer; + linestring_t part; linestring_t lastLinestring; - point_t firstLinestringPoint; - bool isMultiPolygon = relation.isMultiPolygon(); - bool close = false; + bool justClosed = false; + bool first = true; + std::vector members; + + // Skip members that are not Way for (auto mit = relation.members.begin(); mit != relation.members.end(); ++mit) { - - // Process Way objects only, not Nodes or other Relations if (mit->type == osmobjects::way) { - noWayMembers = false; - - if (close == true) { - first = true; - close = false; - using_linestring_tmp = false; - } - - // If a reference is not on Way cache, skip this relation - if (!waycache.count(mit->ref)) { - noWay = true; - break; - } + members.push_back(*mit); + } + } - auto way = waycache.at(mit->ref); + for (auto mit = members.begin(); mit != members.end(); ++mit) { + // Process Way objects only, not Nodes or other Relations + if (!waycache.count(mit->ref)) { + // Way is not available in cache, + // possibily because Relation is not in the priority area + // or the way was deleted + return; + } + + auto way = std::make_shared(); - if (first) { - firstLinestringPoint = point_t( - boost::geometry::get<0>(way->linestring[0]), - boost::geometry::get<1>(way->linestring[0]) - ); - } + way = waycache.at(mit->ref); - // 1. When - // A) Relation is a LineString or MultiLineString but - // we want to save it as a MultiPolygon (this is the case for boundaries) - // B) the Relation is MultiPolygon but is composed of several LineStrings - if (using_linestring_tmp || (isMultiPolygon && - boost::geometry::num_points(way->linestring) > 0 && - boost::geometry::num_points(way->polygon) == 0) - ) { + if (bg::num_points(way->linestring) > 0 && + bg::num_points(way->polygon) == 0) + { - using_linestring_tmp = true; - std::stringstream ss; - std::string geometry; + // Linestrings - // If way's geometry is a polygon, save the outer - if (boost::geometry::num_points(way->polygon) > 0) { - ss << std::setprecision(12) << boost::geometry::wkt(way->polygon.outer()); - geometry = ss.str(); - geometry.erase(geometry.size() - 1); + if (!way->isClosed()) { - // Way's geometry is a LineString + // Reverse the line direction if it's necessary + if (first && (std::next(mit) != members.end())) { + auto nextWay = std::make_shared(); + auto nextWayId = std::next(mit)->ref; + if (!waycache.count(nextWayId)) { + // Way is not available in cache, + // possibily because Relation is not in the priority area + // or the way was deleted + return; + } + nextWay = waycache.at(nextWayId); + + if ( bg::num_points(nextWay->linestring) > 0 && + bg::num_points(way->linestring) > 0 && ( + bg::equals(way->linestring.front(), nextWay->linestring.front()) || + bg::equals(way->linestring.front(), nextWay->linestring.back()) + )) { + bg::reverse(way->linestring); + } } else { - - if (first) { - std::cout << "Check if first linestring have to be reversed" << std::endl; - auto nextWay = waycache.at(std::next(mit)->ref); - - auto first_node_way1 = way->refs.front(); - auto first_node_way2 = nextWay->refs.front(); - auto last_node_way2 = nextWay->refs.back(); - - if (first_node_way1 == first_node_way2 || - first_node_way1 == last_node_way2) { - std::cout << "Yes! reverse first line" << std::endl; + if ( bg::num_points(way->linestring) > 0 && + bg::num_points(lastLinestring) > 0 ) { + if (bg::equals(way->linestring.back(), lastLinestring.back())) { bg::reverse(way->linestring); - firstLinestringPoint = point_t( - boost::geometry::get<0>(way->linestring.front()), - boost::geometry::get<1>(way->linestring.front()) - ); } } + } - if (!first) { - // Reverse the line direction if it's necessary - if (lastLinestring.size() > 0 && way->linestring.size() > 0) { - bool reverse_line = bg::equals(lastLinestring.front(), way->linestring.front()) || - bg::equals(lastLinestring.front(), way->linestring.back()) || - bg::equals(lastLinestring.back(), way->linestring.front()) || - bg::equals(lastLinestring.back(), way->linestring.back()); - if (reverse_line) { - bg::reverse(way->linestring); - } - } - - // Check if object is closed - - // FIXME: - // If one linestring reaches the beginning of the multilinestring - // take it as a polygon and start with a new one, to solve cases - // like this one https://www.openstreetmap.org/relation/16193116 - - if (bg::equals(way->linestring.back(), firstLinestringPoint)) { - close = true; - } - + bg::append(part, way->linestring); + + // Check if object is closed + if (relation.isMultiPolygon() && bg::equals(part.back(), part.front())) { + // Convert LineString to Polygon + polygon_t polygon; + bg::append(polygon.outer(), part); + if (mit->role == "inner") { + parts_inner.push_back({ + linestring_t(), + { polygon } + }); + } else { + parts_outer.push_back({ + linestring_t(), + { polygon } + }); + } + part.clear(); + first = true; + justClosed = true; + lastLinestring.clear(); + } else if (std::next(mit) != members.end()) { + // Check if object is disconnected + auto nextWay = std::make_shared(); + auto nextWayId = std::next(mit)->ref; + if (!waycache.count(nextWayId)) { + // Way is not available in cache, + // possibily because Relation is not in the priority area + // or the way was deleted + return; + } + nextWay = waycache.at(nextWayId); + if ( (bg::num_points(way->linestring) > 0 && bg::num_points(nextWay->linestring) > 0 && + !bg::equals(way->linestring.back(), nextWay->linestring.front()) && + !bg::equals(way->linestring.back(), nextWay->linestring.back())) || + (bg::num_points(nextWay->linestring) == 0) + ) { + parts_outer.push_back({ + { part }, + polygon_t() + }); + part.clear(); + first = true; + justClosed = true; + lastLinestring.clear(); } - lastLinestring = way->linestring; - ss << std::setprecision(12) << boost::geometry::wkt(way->linestring); - geometry = ss.str(); } - // Erase "LINESTRING(" - geometry.erase(0,11); - geometry.erase(geometry.size() - 1); - // Get geometry coordinates as a string (lat lon, lat lon, ...), - if (close) { - std::cout << "***** [CLOSED OBJECT]" << std::endl; - std::cout << way->id << std::endl; - } - linestring_tmp += "(" + geometry + "),"; } else { + // Convert LineString to Polygon + if (mit->role == "inner") { + parts_inner.push_back({ + linestring_t(), + { way->polygon } + }); + } else { + parts_outer.push_back({ + linestring_t(), + { way->polygon } + }); + } + } - // 2. Any other MultiPolygon or MultiLineString + lastLinestring = way->linestring; - // When Relation is MultiLineString but way's geometry is a Polygon - if (!isMultiPolygon && boost::geometry::num_points(way->linestring) == 0 && - boost::geometry::num_points(way->polygon) > 0 - ) { - // Convert way's Polygon to LineString - bg::assign_points(way->linestring, way->polygon.outer()); - } + } else { - std::stringstream ss; - std::string geometry; + // Polygons - if (isMultiPolygon) { - ss << std::setprecision(12) << boost::geometry::wkt(way->polygon); - geometry = ss.str(); - // Erase "POLYGON(" - geometry.erase(0,8); + // When Relation is MultiLineString but way's geometry is a Polygon + if (!relation.isMultiPolygon() && bg::num_points(way->linestring) == 0 && + bg::num_points(way->polygon) > 0 + ) { + // Convert way's Polygon to LineString + bg::assign_points(way->linestring, way->polygon.outer()); + if (mit->role == "inner") { + parts_inner.push_back({ + { way->linestring }, + polygon_t() + }); } else { - ss << std::setprecision(12) << boost::geometry::wkt(way->linestring); - geometry = ss.str(); - // Erase "LINESTRING(" - geometry.erase(0,11); - } - // Erase ")" - geometry.erase(geometry.size() - 1); - - - // Get geometry coordinates as a string (lat lon, lat lon, ...) - if (!isMultiPolygon) { - geometry = "(" + geometry + ")"; + parts_outer.push_back({ + { way->linestring }, + polygon_t() + }); } - - // Add way's geometry to the final result - // FIXME CHECK - if (first && (mit->role != "inner")) { - geometry_str += geometry + ","; + } else { + if (mit->role == "inner") { + parts_inner.push_back({ + linestring_t(), + { way->polygon } + }); } else { - if (mit->role == "inner" && isMultiPolygon) { - innerGeoStr += geometry + ","; + if (way->polygon.outer().size() > 0) { + parts_outer.push_back({ + linestring_t(), + { way->polygon } + }); } else { - geometry_str += geometry + ","; - geometry_str += innerGeoStr; - innerGeoStr = ""; + parts_outer.push_back({ + { way->linestring }, + polygon_t() + }); } } } + + } + + if (first && !justClosed) { first = false; } + if (justClosed) { + justClosed = false; + } + } - // If the relation has way members and all ways were found in the ways cache - if (!noWay && !noWayMembers) { - - // FIXME CHECK - if (linestring_tmp.size() == 0) { - if (!isMultiPolygon) { + if (part.size() > 0) { + parts_outer.push_back({ + { part }, + polygon_t() + }); + } + + // Converts all geometries to WKT strings + + std::string geometry = ""; + int i = 0; + + // Inner parts + for (auto pit = parts_outer.begin(); pit != parts_outer.end(); ++pit) { + std::stringstream ss; + std::string geometry_str; + ++i; + if (relation.isMultiPolygon()) { + if (bg::num_points(pit->polygon.outer()) > 1) { + ss << std::setprecision(12) << bg::wkt(pit->polygon); + geometry_str = ss.str(); + // Erase "POLYGON(" + geometry_str.erase(0,8); geometry_str.erase(geometry_str.size() - 1); - geometry_str = "MULTILINESTRING(" + geometry_str; - } else { - geometry_str = "MULTIPOLYGON((" + geometry_str; + geometry += geometry_str + ","; } - if (innerGeoStr.size() > 0) { - geometry_str += innerGeoStr; + } else { + if (bg::num_points(pit->linestring) > 1) { + ss << std::setprecision(12) << bg::wkt(pit->linestring); + geometry_str = ss.str(); + // Erase "LINESTRING(" + geometry_str.erase(0,11); + geometry_str.erase(geometry_str.size() - 1); + geometry += "(" + geometry_str + "),"; } - geometry_str.erase(geometry_str.size() - 1); - geometry_str += "))"; + } + } - // FIXME CHECK + // Outer parts + for (auto pit = parts_inner.begin(); pit != parts_inner.end(); ++pit) { + std::stringstream ss; + std::string geometry_str; + ++i; + if (relation.isMultiPolygon()) { + if (bg::num_points(pit->polygon.outer()) > 1) { + ss << std::setprecision(12) << bg::wkt(pit->polygon); + geometry_str = ss.str(); + // Erase "POLYGON(" + geometry_str.erase(0,8); + geometry_str.erase(geometry_str.size() - 1); + geometry += geometry_str + ","; + } } else { - - std::cout << "HERE!" << std::endl; - - // Create a MultiLineString - // TODO: do this for each part - linestring_tmp.erase(linestring_tmp.size() - 1); - geometry_str = "MULTILINESTRING(" + linestring_tmp + ")"; - boost::geometry::read_wkt(geometry_str, relation.multilinestring); - linestring_t mergedLineString; - for (auto& line : relation.multilinestring) { - bg::append(mergedLineString, line); - } - - // Create a Polygon from the MultiLineString - polygon_t polygon; - bg::append(polygon.outer(), mergedLineString); - std::stringstream ss; - ss << std::setprecision(12) << boost::geometry::wkt(polygon); - geometry_str = ss.str(); - - // Erase "POLYGON" - geometry_str.erase(0, 8); - - // Create final MultiPolygon - geometry_str = "MULTIPOLYGON((" + geometry_str + ")"; - + if (bg::num_points(pit->linestring) > 1) { + ss << std::setprecision(12) << bg::wkt(pit->linestring); + geometry_str = ss.str(); + // Erase "LINESTRING(" + geometry_str.erase(0,11); + geometry_str.erase(geometry_str.size() - 1); + geometry += "(" + geometry_str + "),"; + } } + } - // Save the final geometry string - if (isMultiPolygon) { - boost::geometry::read_wkt(geometry_str, relation.multipolygon); + // Build the final multipolygon or multilinestring to store it as the + // relation's geometry + if (geometry.size() > 1) { + geometry.erase(geometry.size() - 1); + if (relation.isMultiPolygon()) { + bg::read_wkt("POLYGON(" + geometry + ")", relation.multipolygon); } else { - boost::geometry::read_wkt(geometry_str, relation.multilinestring); + bg::read_wkt("MULTILINESTRING(" + geometry + ")", relation.multilinestring); } - } } @@ -666,14 +690,14 @@ OsmChange::dump(void) way->dump(); } } - if (relations.size() > 0) { - for (auto it = std::begin(relations); it != std::end(relations); ++it) { - // std::cerr << "\tDumping relations: " << it->dump() << std::endl; - // std::shared_ptr rel = *it; - // rel->dump( << std::endl; - } - } - std::cerr << "Final timestamp: " << to_simple_string(final_entry) << std::endl; + // if (relations.size() > 0) { + // for (auto it = std::begin(relations); it != std::end(relations); ++it) { + // std::cerr << "\tDumping relations: " << it->dump() << std::endl; + // std::shared_ptr rel = *it; + // rel->dump(); + // } + // } + // std::cerr << "Final timestamp: " << to_simple_string(final_entry) << std::endl; } @@ -695,7 +719,7 @@ OsmChangeFile::dump(void) #if 0 std::cerr << "\tDumping nodecache:" << std::endl; for (auto it = std::begin(nodecache); it != std::end(nodecache); ++it) { - std::cerr << "\t\t: " << it->first << ": " << boost::geometry::wkt(it->second) << std::endl; + std::cerr << "\t\t: " << it->first << ": " << bg::wkt(it->second) << std::endl; } #endif } @@ -716,10 +740,10 @@ OsmChangeFile::areaFilter(const multipolygon_t &poly) // Filter nodes for (auto nit = std::begin(change->nodes); nit != std::end(change->nodes); ++nit) { OsmNode *node = nit->get(); - if (poly.empty() || boost::geometry::within(node->point, poly)) { + if (poly.empty() || bg::within(node->point, poly)) { node->priority = true; nodecache[node->id] = node->point; - } else if (!boost::geometry::within(node->point, poly)) { + } else if (!bg::within(node->point, poly)) { node->priority = false; } } @@ -732,7 +756,7 @@ OsmChangeFile::areaFilter(const multipolygon_t &poly) } else { way->priority = false; for (auto rit = std::begin(way->refs); rit != std::end(way->refs); ++rit) { - if (nodecache.count(*rit) && boost::geometry::within(nodecache[*rit], poly)) { + if (nodecache.count(*rit) && bg::within(nodecache[*rit], poly)) { way->priority = true; break; } @@ -747,19 +771,16 @@ OsmChangeFile::areaFilter(const multipolygon_t &poly) // Filter relations for (auto rit = std::begin(change->relations); rit != std::end(change->relations); ++rit) { OsmRelation *relation = rit->get(); - relation->priority = true; - if (!poly.empty()) { + if (poly.empty()) { + relation->priority = true; + } else { + relation->priority = false; for (auto mit = std::begin(relation->members); mit != std::end(relation->members); ++mit) { - if (waycache.count(mit->ref)) { - auto way = waycache.at(mit->ref); - if (!way->priority) { - relation->priority = false; - break; - } - } else { - relation->priority = false; + if (waycache.count(mit->ref) && waycache.at(mit->ref)->priority) { + relation->priority = true; break; } + } } } @@ -851,13 +872,13 @@ OsmChangeFile::collectStats(const multipolygon_t &poly) // Calculate length if ( (*hit == "highway" || *hit == "waterway") && way->action == osmobjects::create) { // Get the geometry behind each reference - boost::geometry::model::linestring globe; + bg::model::linestring globe; for (auto lit = std::begin(way->refs); lit != std::end(way->refs); ++lit) { double x = nodecache[*lit].get<0>(); double y = nodecache[*lit].get<1>(); if (x != 0 && y != 0) { globe.push_back(sphere_t(x,y)); - boost::geometry::append(way->linestring, nodecache[*lit]); + bg::append(way->linestring, nodecache[*lit]); } } std::string tag; @@ -867,8 +888,8 @@ OsmChangeFile::collectStats(const multipolygon_t &poly) if (*hit == "waterway") { tag = "waterway_km"; } - double length = boost::geometry::length(globe, - boost::geometry::strategy::distance::haversine(6371.0)); + double length = bg::length(globe, + bg::strategy::distance::haversine(6371.0)); // log_debug("LENGTH: %1% %2%", std::to_string(length), way->changeset); ostats->added[tag] += length; } diff --git a/src/osm/osmchange.hh b/src/osm/osmchange.hh index 5f6b9db63..8365f5a5e 100644 --- a/src/osm/osmchange.hh +++ b/src/osm/osmchange.hh @@ -248,7 +248,6 @@ class OsmChangeFile void buildGeometriesFromNodeCache(); void buildRelationGeometry(osmobjects::OsmRelation &relation); - #ifdef LIBXML /// Called by libxml++ for each element of the XML file void on_start_element(const Glib::ustring &name, diff --git a/src/osm/osmobjects.cc b/src/osm/osmobjects.cc index e87ffb68f..4135dea19 100644 --- a/src/osm/osmobjects.cc +++ b/src/osm/osmobjects.cc @@ -1,5 +1,5 @@ // -// Copyright (c) 2020, 2021, 2023 Humanitarian OpenStreetMap Team +// Copyright (c) 2020, 2021, 2023, 2024 Humanitarian OpenStreetMap Team // // This file is part of Underpass. // diff --git a/src/osm/osmobjects.hh b/src/osm/osmobjects.hh index 83ef74e8f..b58045b3a 100644 --- a/src/osm/osmobjects.hh +++ b/src/osm/osmobjects.hh @@ -59,7 +59,10 @@ namespace osmobjects { /// Relations contain multipe ways, and are often used for combining /// highway segments or administrative boundaries. -typedef enum { none, create, modify, remove } action_t; // delete is a reserved word +// delete is a reserved word +// modify_geom is a special action for modifying the geometry only +typedef enum { none, create, modify, remove, modify_geom } action_t; + typedef enum { empty, node, way, relation, member } osmtype_t; /// \class OsmObject @@ -219,7 +222,7 @@ class OsmRelation : public OsmObject { OsmRelation(void) { type = relation; }; multilinestring_t multilinestring; ///< Store the members as a multilinestring - multipolygon_t multipolygon; ///< Store the members as a multipolygon + polygon_t multipolygon; ///< Store the members as a multipolygon point_t center; ///< Store the centroid of the relation /// Add a member to this relation @@ -235,7 +238,7 @@ class OsmRelation : public OsmObject { bool isMultiPolygon(void) const { return (tags.count("type") && - (tags.at("type") == "multipolygon") + (tags.at("type") == "multipolygon" || tags.at("type") == "boundary") ); }; diff --git a/src/raw/queryraw.cc b/src/raw/queryraw.cc index eda7a4c6b..085d3c458 100644 --- a/src/raw/queryraw.cc +++ b/src/raw/queryraw.cc @@ -20,8 +20,8 @@ /// \file queryraw.cc /// \brief This file is used to work with the OSM Raw database /// -/// This manages the OSM Raw schema in a postgres database. This -/// includes querying existing data in the database, as well as +/// This manages the OSM Raw schema in a PostgreSQL database, +/// including querying existing data in the database, as well as /// updating the database. // This is generated by autoconf @@ -60,6 +60,8 @@ QueryRaw::QueryRaw(std::shared_ptr db) { dbconn = db; } +// Receives a dictionary of tags (key: value) and returns +// a JSONB string for doing an insert operation into the database. std::string QueryRaw::buildTagsQuery(std::map tags) const { if (tags.size() > 0) { @@ -67,7 +69,10 @@ QueryRaw::buildTagsQuery(std::map tags) const { int count = 0; for (auto it = std::begin(tags); it != std::end(tags); ++it) { ++count; - // PostgreSQL has an argument limit for functions + // PostgreSQL has an argument limit for functions (100 parameters max) + // Because of this, when the count of key/value pairs reaches 50, + // a concatenation of multiple calls to the jsonb_build_object() function + // is needed. if (count == 50) { tagsStr.erase(tagsStr.size() - 1); tagsStr += ") || jsonb_build_object("; @@ -86,13 +91,13 @@ QueryRaw::buildTagsQuery(std::map tags) const { } } +// Receives a list of Relation members and returns +// a JSONB string for doing an insert operation into the database. std::string buildMembersQuery(std::list members) { if (members.size() > 0) { std::string membersStr = "'["; - int count = 0; for (auto mit = std::begin(members); mit != std::end(members); ++mit) { - count++; membersStr += "{"; std::string member_format = "\"%s\": \"%s\","; boost::format member_fmt(member_format); @@ -100,7 +105,16 @@ buildMembersQuery(std::list members) { member_fmt % mit->role; membersStr += member_fmt.str(); member_fmt % "type"; - member_fmt % mit->type; + switch(mit->type) { + case osmobjects::osmtype_t::way: + member_fmt % "way"; break; + case osmobjects::osmtype_t::node: + member_fmt % "node"; break; + case osmobjects::osmtype_t::relation: + member_fmt % "relation"; break; + default: + member_fmt % ""; + } membersStr += member_fmt.str(); membersStr += "\"ref\":"; membersStr += std::to_string(mit->ref); @@ -114,6 +128,8 @@ buildMembersQuery(std::list members) { } } +// Parses a JSON object from a string and return a map of key/value. +// This function is useful for parsing tags from a query result. std::map parseJSONObjectStr(std::string input) { std::map obj; boost::property_tree::ptree pt; @@ -130,6 +146,8 @@ std::map parseJSONObjectStr(std::string input) { return obj; } +// Parses a JSON object from a string and return a vector of key/value maps +// This function is useful for parsing relation members from a query result. std::vector> parseJSONArrayStr(std::string input) { std::vector> arr; boost::property_tree::ptree pt; @@ -152,12 +170,17 @@ std::vector> parseJSONArrayStr(std::string in return arr; } -std::string + +// Apply the change for a Node. It will return a string of a query for +// insert, update or delete the Node in the database. +std::shared_ptr> QueryRaw::applyChange(const OsmNode &node) const { - std::string query; + auto queries = std::make_shared>(); + + // If create or modify, then insert or update if (node.action == osmobjects::create || node.action == osmobjects::modify) { - query = "INSERT INTO nodes as r (osm_id, geom, tags, timestamp, version, \"user\", uid, changeset) VALUES("; + std::string query = "INSERT INTO nodes as r (osm_id, geom, tags, timestamp, version, \"user\", uid, changeset) VALUES("; std::string format = "%d, ST_GeomFromText(\'%s\', 4326), %s, \'%s\', %d, \'%s\', %d, %d \ ) ON CONFLICT (osm_id) DO UPDATE SET geom = ST_GeomFromText(\'%s\', \ 4326), tags = %s, timestamp = \'%s\', version = %d, \"user\" = \'%s\', uid = %d, changeset = %d WHERE r.version < %d;"; @@ -168,7 +191,7 @@ QueryRaw::applyChange(const OsmNode &node) const // geometry std::stringstream ss; - ss << std::setprecision(12) << boost::geometry::wkt(node.point); + ss << std::setprecision(12) << bg::wkt(node.point); std::string geometry = ss.str(); fmt % geometry; @@ -187,7 +210,7 @@ QueryRaw::applyChange(const OsmNode &node) const // changeset fmt % node.changeset; - // ON CONFLICT + // ON CONFLICT (update) fmt % geometry; fmt % tags; fmt % timestamp; @@ -197,187 +220,271 @@ QueryRaw::applyChange(const OsmNode &node) const fmt % node.changeset; fmt % node.version; - query += fmt.str(); + query.append(fmt.str()); + queries->push_back(query); + // If remove, then delete the object } else if (node.action == osmobjects::remove) { - query = "DELETE from nodes where osm_id = " + std::to_string(node.id) + ";"; + queries->push_back("DELETE from nodes where osm_id = " + std::to_string(node.id) + ";"); } - return query; + return queries; } - const std::string QueryRaw::polyTable = "ways_poly"; const std::string QueryRaw::lineTable = "ways_line"; -std::string +// Apply the change for a Way. It will return a string of a query for +// insert, update or delete the Way in the database. +std::shared_ptr> QueryRaw::applyChange(const OsmWay &way) const { - std::string query = ""; + auto queries = std::make_shared>(); + std::string query; const std::string* tableName; + // Get a Polygon or LineString geometry string depending on the Way std::stringstream ss; if (way.refs.size() > 3 && (way.refs.front() == way.refs.back())) { tableName = &QueryRaw::polyTable; - ss << std::setprecision(12) << boost::geometry::wkt(way.polygon); + ss << std::setprecision(12) << bg::wkt(way.polygon); } else { tableName = &QueryRaw::lineTable; - ss << std::setprecision(12) << boost::geometry::wkt(way.linestring); + ss << std::setprecision(12) << bg::wkt(way.linestring); } std::string geostring = ss.str(); - if (way.refs.size() > 2 - && (way.action == osmobjects::create || way.action == osmobjects::modify)) { - if ((way.refs.front() != way.refs.back() && way.refs.size() == boost::geometry::num_points(way.linestring)) || - (way.refs.front() == way.refs.back() && way.refs.size() == boost::geometry::num_points(way.polygon)) + // Make sure we have what's needed to insert or update a Way: + // - At least 2 points + // - A LineString or a Polygon + // - A create, modify or "modify geometry" action. + if (way.refs.size() > 0 + && (way.action == osmobjects::create || way.action == osmobjects::modify || way.action == osmobjects::modify_geom)) { + if ((way.refs.front() != way.refs.back() && way.refs.size() == bg::num_points(way.linestring)) || + (way.refs.front() == way.refs.back() && way.refs.size() == bg::num_points(way.polygon)) ) { - query = "INSERT INTO " + *tableName + " as r (osm_id, tags, refs, geom, timestamp, version, \"user\", uid, changeset) VALUES("; - std::string format = "%d, %s, %s, %s, \'%s\', %d, \'%s\', %d, %d) \ - ON CONFLICT (osm_id) DO UPDATE SET tags = %s, refs = %s, geom = %s, timestamp = \'%s\', version = %d, \"user\" = \'%s\', uid = %d, changeset = %d WHERE r.version <= %d;"; - boost::format fmt(format); + // Insert or update the full Way, including id, tags, refs, geometry, timestamp, version, + // user, uid and changeset + if (way.action != osmobjects::modify_geom) { + + query = "INSERT INTO " + *tableName + " as r (osm_id, tags, refs, geom, timestamp, version, \"user\", uid, changeset) VALUES("; + std::string format = "%d, %s, %s, %s, \'%s\', %d, \'%s\', %d, %d) \ + ON CONFLICT (osm_id) DO UPDATE SET tags = %s, refs = %s, geom = %s, timestamp = \'%s\', version = %d, \"user\" = \'%s\', uid = %d, changeset = %d WHERE r.version <= %d;"; + boost::format fmt(format); + + // osm_id + fmt % way.id; + + // tags + auto tags = buildTagsQuery(way.tags); + fmt % tags; + + // refs + std::string refs = ""; + for (auto it = std::begin(way.refs); it != std::end(way.refs); ++it) { + refs += std::to_string(*it) + ","; + } + refs.erase(refs.size() - 1); + refs = "ARRAY[" + refs + "]"; + fmt % refs; + + // geometry + std::string geometry; + geometry = "ST_GeomFromText(\'" + geostring + "\', 4326)"; + fmt % geometry; + + // timestamp (now) + std::string timestamp = to_simple_string(boost::posix_time::microsec_clock::universal_time()); + fmt % timestamp; + // version + fmt % way.version; + // user + fmt % dbconn->escapedString(way.user); + // uid + fmt % way.uid; + // changeset + fmt % way.changeset; + + // ON CONFLICT (update) + fmt % tags; + fmt % refs; + fmt % geometry; + fmt % timestamp; + fmt % way.version; + fmt % dbconn->escapedString(way.user); + fmt % way.uid; + fmt % way.changeset; + fmt % way.version; + + query += fmt.str(); + queries->push_back(query); + + } else { + + // Update only the Way's geometry. This is the case when a Way was indirectly + // modified by a change on some referenced Node; the geometry of the Way will + // change but all other data (tags, version, etc) will remain the same. - // osm_id - fmt % way.id; + query = "UPDATE " + *tableName + " SET "; + std::string format = "geom=%s, timestamp=\'%s\' WHERE osm_id=%d;"; + boost::format fmt(format); - //tags - auto tags = buildTagsQuery(way.tags); - fmt % tags; + // Geometry + std::string geometry; + geometry = "ST_GeomFromText(\'" + geostring + "\', 4326)"; + fmt % geometry; - // refs - std::string refs = ""; - for (auto it = std::begin(way.refs); it != std::end(way.refs); ++it) { - refs += std::to_string(*it) + ","; + // Timestamp (now) + std::string timestamp = to_simple_string(boost::posix_time::microsec_clock::universal_time()); + fmt % timestamp; + + // osm_id + fmt % way.id; + + query += fmt.str(); + queries->push_back(query); } - refs.erase(refs.size() - 1); - refs = "ARRAY[" + refs + "]"; - fmt % refs; - - // geometry - std::string geometry; - geometry = "ST_GeomFromText(\'" + geostring + "\', 4326)"; - fmt % geometry; - - // timestamp - std::string timestamp = to_simple_string(boost::posix_time::microsec_clock::universal_time()); - fmt % timestamp; - // version - fmt % way.version; - // user - fmt % dbconn->escapedString(way.user); - // uid - fmt % way.uid; - // changeset - fmt % way.changeset; - - // ON CONFLICT - fmt % tags; - fmt % refs; - fmt % geometry; - fmt % timestamp; - fmt % way.version; - fmt % dbconn->escapedString(way.user); - fmt % way.uid; - fmt % way.changeset; - fmt % way.version; - - query += fmt.str(); - - query += "DELETE FROM way_refs WHERE way_id=" + std::to_string(way.id) + ";"; - for (auto ref = way.refs.begin(); ref != way.refs.end(); ++ref) { - query += "INSERT INTO way_refs (way_id, node_id) VALUES (" + std::to_string(way.id) + "," + std::to_string(*ref) + ");"; + + // If the Way's geometry is a LineString, remove all Polygons from the Polygons table. + // If the Way's geometry is a Polygon, remove all LineString from the LineStrings table. + // This is for preventing duplicated Way geometries. For example, when the Way was a + // LineString but it was then closed and converted to a Polygon. + std::string delquery = "DELETE FROM %s WHERE osm_id=%d;"; + boost::format delquery_fmt(delquery); + if (tableName == &QueryRaw::polyTable) { + delquery_fmt % QueryRaw::lineTable; + } else { + delquery_fmt % QueryRaw::polyTable; } + delquery_fmt % way.id; + queries->push_back(delquery_fmt.str()); } } else if (way.action == osmobjects::remove) { - query += "DELETE FROM way_refs WHERE way_id=" + std::to_string(way.id) + ";"; - query += "DELETE FROM " + QueryRaw::polyTable + " where osm_id = " + std::to_string(way.id) + ";"; - query += "DELETE FROM " + QueryRaw::lineTable + " where osm_id = " + std::to_string(way.id) + ";"; + // Delete a Way geometry and its references. + if (tableName == &QueryRaw::polyTable) { + queries->push_back("DELETE FROM " + QueryRaw::polyTable + " where osm_id = " + std::to_string(way.id) + ";"); + } else { + queries->push_back("DELETE FROM " + QueryRaw::lineTable + " where osm_id = " + std::to_string(way.id) + ";"); + } } - return query; + return queries; } -std::string +// Apply the change for a Relation. It will return a string of a query for +// insert, update or delete the Relation in the database. +std::shared_ptr> QueryRaw::applyChange(const OsmRelation &relation) const { - std::string query = ""; + auto queries = std::make_shared>(); + std::string query; - if (relation.action == osmobjects::create || relation.action == osmobjects::modify) { + // Create, modify or modify the geometry of a Relation + if (relation.action == osmobjects::create || relation.action == osmobjects::modify || relation.action == osmobjects::modify_geom) { + // Get a Polygon or LineString geometry string depending on the Relation std::stringstream ss; if (relation.isMultiPolygon()) { - ss << std::setprecision(12) << boost::geometry::wkt(relation.multipolygon); + ss << std::setprecision(12) << bg::wkt(relation.multipolygon); } else { - ss << std::setprecision(12) << boost::geometry::wkt(relation.multilinestring); + ss << std::setprecision(12) << bg::wkt(relation.multilinestring); } std::string geostring = ss.str(); // Ignore empty geometries - if (geostring != "MULTILINESTRING()" && geostring != "MULTIPOLYGON()") { - - query = "INSERT INTO relations as r (osm_id, tags, refs, geom, timestamp, version, \"user\", uid, changeset) VALUES("; - std::string format = "%d, %s, %s, %s, \'%s\', %d, \'%s\', %d, %d) \ - ON CONFLICT (osm_id) DO UPDATE SET tags = %s, refs = %s, geom = %s, timestamp = \'%s\', version = %d, \"user\" = \'%s\', uid = %d, changeset = %d WHERE r.version <= %d;"; - boost::format fmt(format); - - // osm_id - fmt % relation.id; - - // tags - auto tags = buildTagsQuery(relation.tags); - fmt % tags; - - // refs - auto refs = buildMembersQuery(relation.members); - fmt % refs; - - // geometry - std::string geometry; - geometry = "ST_GeomFromText(\'" + geostring + "\', 4326)"; - - std::cout << "Relation " << relation.id << " " << geostring << std::endl; - - fmt % geometry; - - // timestamp - std::string timestamp = to_simple_string(boost::posix_time::microsec_clock::universal_time()); - fmt % timestamp; - // version - fmt % relation.version; - // user - fmt % dbconn->escapedString(relation.user); - // uid - fmt % relation.uid; - // changeset - fmt % relation.changeset; - - // ON CONFLICT - fmt % tags; - fmt % refs; - fmt % geometry; - fmt % timestamp; - fmt % relation.version; - fmt % dbconn->escapedString(relation.user); - fmt % relation.uid; - fmt % relation.changeset; - fmt % relation.version; - - query += fmt.str(); - - for (auto it = std::begin(relation.members); it != std::end(relation.members); ++it) { - query += "INSERT INTO rel_refs (rel_id, way_id) VALUES (" + std::to_string(relation.id) + "," + std::to_string(it->ref) + ");"; + if (geostring != "MULTILINESTRING()" && geostring != "POLYGON()" + && geostring != "MULTILINESTRING(())" && geostring != "POLYGON(())") { + + // Insert or update the full Relation, including id, tags, refs, geometry, timestamp, + // version, user, uid and changeset + if (relation.action != osmobjects::modify_geom) { + + query = "INSERT INTO relations as r (osm_id, tags, refs, geom, timestamp, version, \"user\", uid, changeset) VALUES("; + std::string format = "%d, %s, %s, %s, \'%s\', %d, \'%s\', %d, %d) \ + ON CONFLICT (osm_id) DO UPDATE SET tags = %s, refs = %s, geom = %s, timestamp = \'%s\', version = %d, \"user\" = \'%s\', uid = %d, changeset = %d WHERE r.version <= %d;"; + boost::format fmt(format); + + // osm_id + fmt % relation.id; + + // tags + auto tags = buildTagsQuery(relation.tags); + fmt % tags; + + // refs + auto refs = buildMembersQuery(relation.members); + fmt % refs; + + // geometry + std::string geometry; + geometry = "ST_GeomFromText(\'" + geostring + "\', 4326)"; + fmt % geometry; + + // timestamp (now) + std::string timestamp = to_simple_string(boost::posix_time::microsec_clock::universal_time()); + fmt % timestamp; + // version + fmt % relation.version; + // user + fmt % dbconn->escapedString(relation.user); + // uid + fmt % relation.uid; + // changeset + fmt % relation.changeset; + + // ON CONFLICT + fmt % tags; + fmt % refs; + fmt % geometry; + fmt % timestamp; + fmt % relation.version; + fmt % dbconn->escapedString(relation.user); + fmt % relation.uid; + fmt % relation.changeset; + fmt % relation.version; + + query.append(fmt.str()); + queries->push_back(query); + + } else { + + // Update only the Relation's geometry. This is the case when a Relation was indirectly + // modified by a change on some referenced Way; the geometry of the Relation will + // change but all other data (tags, version, etc) will remain the same. + + query = "UPDATE relations SET "; + std::string format = "geom=%s, timestamp=\'%s\' WHERE osm_id=%d;"; + boost::format fmt(format); + + // Geometry + std::string geometry; + geometry = "ST_GeomFromText(\'" + geostring + "\', 4326)"; + fmt % geometry; + + // Timestamp + std::string timestamp = to_simple_string(boost::posix_time::microsec_clock::universal_time()); + fmt % timestamp; + + // osm_id + fmt % relation.id; + + query.append(fmt.str()); + queries->push_back(query); } - } else { - std::cout << "Relation " << relation.id << " geometry is empty" << std::endl; } } else if (relation.action == osmobjects::remove) { - query += "DELETE FROM relations where osm_id = " + std::to_string(relation.id) + ";"; + // Delete a Relation geometry and its references. + queries->push_back("DELETE FROM relations where osm_id = " + std::to_string(relation.id) + ";"); } - return query; + return queries; } -std::vector arrayStrToVector(std::string &refs_str) { +// Receives a string of comma separated values and +// returns a vector. This function is useful for +// getting a vector of references from a query result +std::vector arrayStrToVector(std::string refs_str) { refs_str.erase(0, 1); refs_str.erase(refs_str.size() - 1); std::vector refs; @@ -389,19 +496,25 @@ std::vector arrayStrToVector(std::string &refs_str) { return refs; } +// Get all Relations that have at least 1 reference to any Way +// of a list. This function receives a string of comma separated +// ids ("213213,328947,287313") and returns a list of Relation +// objects. This is useful for getting Relations that were +// indirectly modified by a change on a Way. std::list> QueryRaw::getRelationsByWaysRefs(std::string &wayIds) const { #ifdef TIMING_DEBUG boost::timer::auto_cpu_timer timer("getRelationsByWaysRefs(wayIds): took %w seconds\n"); #endif - // Get all relations that have references to ways + // Object to return std::list> rels; - std::string relsQuery = "SELECT distinct(osm_id), refs, version, tags, uid, changeset from rel_refs join relations r on r.osm_id = rel_id where way_id = any(ARRAY[" + wayIds + "])"; + // Query for getting Relations + std::string relsQuery = "SELECT distinct(osm_id), refs, version, tags, uid, changeset FROM relations WHERE EXISTS (SELECT 1 FROM jsonb_array_elements(refs) AS ref WHERE (ref->>'ref')::bigint IN (" + wayIds + "));"; auto rels_result = dbconn->query(relsQuery); - // Fill vector of OsmRelation objects + // Fill vector with OsmRelation objects for (auto rel_it = rels_result.begin(); rel_it != rels_result.end(); ++rel_it) { auto rel = std::make_shared(); rel->id = (*rel_it)[0].as(); @@ -440,31 +553,53 @@ QueryRaw::getRelationsByWaysRefs(std::string &wayIds) const return rels; } +// Receives a string with a list of Way ids, get them from the database and store them +// on a Way cache void QueryRaw::getWaysByIds(std::string &waysIds, std::map> &waycache) { #ifdef TIMING_DEBUG boost::timer::auto_cpu_timer timer("getWaysByIds(waysIds, waycache): took %w seconds\n"); #endif - // Get all ways that have references to nodes + // Get Ways and it's geometries (Polygon and LineString) std::string waysQuery = "SELECT distinct(osm_id), ST_AsText(geom, 4326), 'polygon' as type from ways_poly wp where osm_id = any(ARRAY[" + waysIds + "]) "; - waysQuery += "UNION SELECT distinct(osm_id), ST_AsText(geom, 4326), 'linestring' as type from ways_line wp where osm_id = any(ARRAY[" + waysIds + "])"; + waysQuery += "UNION SELECT distinct(osm_id), ST_AsText(geom, 4326), 'linestring' as type from ways_line wp where osm_id = any(ARRAY[" + waysIds + "]);"; auto ways_result = dbconn->query(waysQuery); + if (ways_result.size() == 0) { + log_debug("No results returned!"); + return; + } - // Fill vector of OsmWay objects + std::string resultIds = ""; + + // Insert Ways into waycache for (auto way_it = ways_result.begin(); way_it != ways_result.end(); ++way_it) { auto way = std::make_shared(); auto type = (*way_it)[2].as(); way->id = (*way_it)[0].as(); if (type == "polygon") { - boost::geometry::read_wkt((*way_it)[1].as(), way->polygon); + bg::read_wkt((*way_it)[1].as(), way->polygon); } else { - boost::geometry::read_wkt((*way_it)[1].as(), way->linestring); + bg::read_wkt((*way_it)[1].as(), way->linestring); } waycache.insert(std::pair(way->id, std::make_shared(*way))); } } +// Receives a list of Osm Changes and the priority area and completes the geometry of +// all objects (Nodes, Ways and Realations), including all indirectly modified objects. +// +// Incomplete geometries happens all the time on Ways and Relations because the data for +// they geometries (coordinates) can be not present on the OsmChange file. For example +// if the tags of a Way are modified, but not it's references, only the tag information +// will be on the OsmChange file, but not the coordinates for the referenced Nodes. +// +// An indirectly modified object is the one whose geometry was modified by a modification +// on the geometry of one of its references. For example, if a Node is modified and that +// Node is referenced on a Way, the Way's geometry must to be updated. Also, if that Way +// is referenced on a Relation, then the Relation's geometry must be updated too. +// // TODO: divide this function into multiple ones +// void QueryRaw::buildGeometries(std::shared_ptr osmchanges, const multipolygon_t &poly) { #ifdef TIMING_DEBUG @@ -481,87 +616,120 @@ void QueryRaw::buildGeometries(std::shared_ptr osmchanges, const for (auto wit = std::begin(change->ways); wit != std::end(change->ways); ++wit) { OsmWay *way = wit->get(); if (way->action != osmobjects::remove) { - // Save referenced nodes ids for later use + + // Save referenced Nodes ids for later use. The geometries of these + // Nodes will be needed later when building geometries for Ways for (auto rit = std::begin(way->refs); rit != std::end(way->refs); ++rit) { if (!osmchanges->nodecache.count(*rit)) { referencedNodeIds += std::to_string(*rit) + ","; } } - // Save ways for later use - if (way->isClosed()) { - // Save only ways with a geometry that are inside the priority area - // these are mostly created ways - if (poly.empty() || boost::geometry::within(way->linestring, poly)) { - osmchanges->waycache.insert(std::make_pair(way->id, std::make_shared(*way))); - } + // Save Ways in waycache, pre-filter by priority area + if (poly.empty() || bg::within(way->linestring, poly)) { + osmchanges->waycache.insert(std::make_pair(way->id, std::make_shared(*way))); } } else { + // Save removed Ways for later use. This list will be used to known + // which Ways will be skipped when building geometries removedWays.push_back(way->id); } } - // Save modified nodes for later use + // Save modified nodes for later use. This list will be used for getting + // indirectly modified Ways for (auto nit = std::begin(change->nodes); nit != std::end(change->nodes); ++nit) { OsmNode *node = nit->get(); if (node->action == osmobjects::modify) { // Get only modified nodes ids inside the priority area - if (poly.empty() || boost::geometry::within(node->point, poly)) { + if (poly.empty() || bg::within(node->point, poly)) { modifiedNodesIds += std::to_string(node->id) + ","; } } } - // for (auto rel_it = std::begin(change->relations); rel_it != std::end(change->relations); ++rel_it) { - // OsmRelation *relation = rel_it->get(); - // removedRelations.push_back(relation->id); - // } + // Save removed Relations for later use. This list will be used to known + // which Relations will be skipped when building geometries + for (auto rel_it = std::begin(change->relations); rel_it != std::end(change->relations); ++rel_it) { + OsmRelation *relation = rel_it->get(); + removedRelations.push_back(relation->id); + } } - // Add indirectly modified ways to osmchanges + // Add indirectly modified ways to osmchanges. An indirectly modified Way is a Way + // whose geoemtry was modified because one of it's referenced Nodes was modified if (modifiedNodesIds.size() > 1) { modifiedNodesIds.erase(modifiedNodesIds.size() - 1); + + // Get all Ways that have at least one reference to one of the modified Nodes auto modifiedWays = getWaysByNodesRefs(modifiedNodesIds); + + // Add a new change for the indirectly modified Way auto change = std::make_shared(none); for (auto wit = modifiedWays.begin(); wit != modifiedWays.end(); ++wit) { auto way = std::make_shared(*wit->get()); - // Save referenced nodes for later use - for (auto rit = std::begin(way->refs); rit != std::end(way->refs); ++rit) { - if (!osmchanges->nodecache.count(*rit)) { - referencedNodeIds += std::to_string(*rit) + ","; - } - } - // If the way is not marked as removed, mark it as modified + // If the Way is not removed if (std::find(removedWays.begin(), removedWays.end(), way->id) == removedWays.end()) { - way->action = osmobjects::modify; + + // Save referenced Nodes. This list will be used for getting the geometries of + // these Nodes, used when building the Way geometry + for (auto rit = std::begin(way->refs); rit != std::end(way->refs); ++rit) { + if (!osmchanges->nodecache.count(*rit)) { + referencedNodeIds += std::to_string(*rit) + ","; + } + } + + // Flag it as modified geometry. This means that only the geometry was modified, + // nor its tags, version, etc. + way->action = osmobjects::modify_geom; + + // Add the Way to the list of Ways in the OsmChange change->ways.push_back(way); + + // Save the id of the indirectly modified Way for later use. This will be used + // for identifying which Relations were indirectly modified by this change. modifiedWaysIds += std::to_string(way->id) + ","; } } osmchanges->changes.push_back(change); } - // Add indirectly modified relations to osmchanges - // if (modifiedWaysIds.size() > 1) { - // modifiedWaysIds.erase(modifiedWaysIds.size() - 1); - // auto modifiedRelations = getRelationsByWaysRefs(modifiedWaysIds); - // auto change = std::make_shared(none); - // for (auto rel_it = modifiedRelations.begin(); rel_it != modifiedRelations.end(); ++rel_it) { - // auto relation = std::make_shared(*rel_it->get()); - // // If the relation is not marked as removed, mark it as modified - // if (std::find(removedRelations.begin(), removedRelations.end(), relation->id) == removedRelations.end()) { - // relation->action = osmobjects::modify; - // change->relations.push_back(relation); - // } - // } - // osmchanges->changes.push_back(change); - // } - - // Fill nodecache with referenced nodes + // Add indirectly modified Relations to osmchanges. This is the case when a Way referenced + // in a Relation was modified (or indirectly modified by a change on one of its Nodes) + if (modifiedWaysIds.size() > 1) { + + // Get indirectly modified Relations from the DB, using the list of Ways + // that were modified + modifiedWaysIds.erase(modifiedWaysIds.size() - 1); + auto modifiedRelations = getRelationsByWaysRefs(modifiedWaysIds); + + // Create a new change for the indirecty modified Relation + auto change = std::make_shared(none); + for (auto rel_it = modifiedRelations.begin(); rel_it != modifiedRelations.end(); ++rel_it) { + auto relation = std::make_shared(*rel_it->get()); + // If the Relation is not removed + if (std::find(removedRelations.begin(), removedRelations.end(), relation->id) == removedRelations.end()) { + // Flag it as modified geometry. This means that only the geometry was modified, + // nor its tags, version, etc. + relation->action = osmobjects::modify_geom; + + // Add the Relation to the list of Relation in the OsmChange + change->relations.push_back(relation); + } + } + osmchanges->changes.push_back(change); + } + + // Fill nodecache with referenced Nodes. This will be used later when building the + // geometries of Ways if (referencedNodeIds.size() > 1) { referencedNodeIds.erase(referencedNodeIds.size() - 1); - // Get Nodes from DB + // Get Nodes geoemtries from DB std::string nodesQuery = "SELECT osm_id, st_x(geom) as lat, st_y(geom) as lon FROM nodes where osm_id in (" + referencedNodeIds + ");"; auto result = dbconn->query(nodesQuery); + if (result.size() == 0) { + log_debug("No results returned!"); + return; + } // Fill nodecache for (auto node_it = result.begin(); node_it != result.end(); ++node_it) { auto node_id = (*node_it)[0].as(); @@ -572,73 +740,78 @@ void QueryRaw::buildGeometries(std::shared_ptr osmchanges, const } } - // Build ways geometries using nodecache + // Build Ways geometries using nodecache for (auto it = std::begin(osmchanges->changes); it != std::end(osmchanges->changes); it++) { OsmChange *change = it->get(); for (auto wit = std::begin(change->ways); wit != std::end(change->ways); ++wit) { OsmWay *way = wit->get(); - way->linestring.clear(); - for (auto rit = way->refs.begin(); rit != way->refs.end(); ++rit) { - if (osmchanges->nodecache.count(*rit)) { - boost::geometry::append(way->linestring, osmchanges->nodecache.at(*rit)); + + // Only build geometries for Ways with incomplete geometries + if (bg::num_points(way->linestring) != way->refs.size()) { + way->linestring.clear(); + for (auto rit = way->refs.begin(); rit != way->refs.end(); ++rit) { + if (osmchanges->nodecache.count(*rit)) { + bg::append(way->linestring, osmchanges->nodecache.at(*rit)); + } + } + if (way->isClosed()) { + way->polygon = { {std::begin(way->linestring), std::end(way->linestring)} }; + way->linestring.clear(); } } - if (way->isClosed()) { - way->polygon = { {std::begin(way->linestring), std::end(way->linestring)} }; - } - // Save way pointer for later use - if (poly.empty() || boost::geometry::within(way->linestring, poly)) { + // Save Way pointer for later use. This will be used when building Relations geometries. + if (poly.empty() || bg::within(way->linestring, poly)) { if (osmchanges->waycache.count(way->id)) { - osmchanges->waycache.at(way->id)->polygon = way->polygon; + if (way->isClosed()) { + osmchanges->waycache.at(way->id)->polygon = way->polygon; + } else { + osmchanges->waycache.at(way->id)->linestring = way->linestring; + } } else { osmchanges->waycache.insert(std::make_pair(way->id, std::make_shared(*way))); } } + } } - // Relations - // std::string relsForWayCacheIds; - // bool debug = false; - // for (auto it = std::begin(osmchanges->changes); it != std::end(osmchanges->changes); it++) { - // OsmChange *change = it->get(); - // for (auto rel_it = std::begin(change->relations); rel_it != std::end(change->relations); ++rel_it) { - // OsmRelation *relation = rel_it->get(); - // if (relation->isMultiPolygon()) { - // bool getWaysForRelation = false; - // for (auto mit = relation->members.begin(); mit != relation->members.end(); ++mit) { - // if (osmchanges->waycache.count(mit->ref)) { - // getWaysForRelation = true; - // break; - // } - // } - // if (getWaysForRelation) { - // for (auto mit = relation->members.begin(); mit != relation->members.end(); ++mit) { - // if (!osmchanges->waycache.count(mit->ref)) { - // relsForWayCacheIds += std::to_string(mit->ref) + ","; - // } - // } - // } - // } - // } - // } - // // Get all missing ways geometries for relations - // if (relsForWayCacheIds != "") { - // relsForWayCacheIds.erase(relsForWayCacheIds.size() - 1); - // getWaysByIds(relsForWayCacheIds, osmchanges->waycache); - // } - - // Build relation geometries - // for (auto it = std::begin(osmchanges->changes); it != std::end(osmchanges->changes); it++) { - // OsmChange *change = it->get(); - // for (auto rel_it = std::begin(change->relations); rel_it != std::end(change->relations); ++rel_it) { - // OsmRelation *relation = rel_it->get(); - // osmchanges->buildRelationGeometry(*relation); - // } - // } + // Build list of Relations that have missing geometries. This list will be used for + // querying the database and get the geometries of the referenced Ways . + std::string relsForWayCacheIds; + for (auto it = std::begin(osmchanges->changes); it != std::end(osmchanges->changes); it++) { + OsmChange *change = it->get(); + for (auto rel_it = std::begin(change->relations); rel_it != std::end(change->relations); ++rel_it) { + OsmRelation *relation = rel_it->get(); + if (relation->action != osmobjects::remove) { + for (auto mit = relation->members.begin(); mit != relation->members.end(); ++mit) { + if (mit->type == osmobjects::way && !osmchanges->waycache.count(mit->ref)) { + relsForWayCacheIds += std::to_string(mit->ref) + ","; + } + } + } + } + } + // Get the geometries of the referenced Ways from the DB. + if (relsForWayCacheIds != "") { + relsForWayCacheIds.erase(relsForWayCacheIds.size() - 1); + getWaysByIds(relsForWayCacheIds, osmchanges->waycache); + } + + // Build geometries for Relations (Polygon or MultiLinestring) + for (auto it = std::begin(osmchanges->changes); it != std::end(osmchanges->changes); it++) { + OsmChange *change = it->get(); + for (auto rel_it = std::begin(change->relations); rel_it != std::end(change->relations); ++rel_it) { + OsmRelation *relation = rel_it->get(); + // Skip removed relations + if (relation->action != osmobjects::remove) { + osmchanges->buildRelationGeometry(*relation); + } + } + } } +// Fill Node cache with Nodes referenced from Ways void QueryRaw::getNodeCacheFromWays(std::shared_ptr> ways, std::map &nodecache) const { @@ -646,21 +819,26 @@ QueryRaw::getNodeCacheFromWays(std::shared_ptr> ways, std::m boost::timer::auto_cpu_timer timer("getNodeCacheFromWays(ways, nodecache): took %w seconds\n"); #endif - // Get all nodes ids referenced in ways + // Build a string list of all Nodes ids referenced from Ways std::string nodeIds; for (auto wit = ways->begin(); wit != ways->end(); ++wit) { for (auto rit = std::begin(wit->refs); rit != std::end(wit->refs); ++rit) { nodeIds += std::to_string(*rit) + ","; } } - if (nodeIds.size() > 1) { + if (nodeIds.size() > 1) { nodeIds.erase(nodeIds.size() - 1); - // Get Nodes from DB + // Get Nodes geometries from the DB std::string nodesQuery = "SELECT osm_id, st_x(geom) as lat, st_y(geom) as lon FROM nodes where osm_id in (" + nodeIds + ") and st_x(geom) is not null and st_y(geom) is not null;"; auto result = dbconn->query(nodesQuery); - // Fill nodecache + if (result.size() == 0) { + log_debug("No results returned!"); + return; + } + + // Fill nodecache with Nodes geometries (Points) for (auto node_it = result.begin(); node_it != result.end(); ++node_it) { auto node_id = (*node_it)[0].as(); auto node_lat = (*node_it)[1].as(); @@ -671,55 +849,70 @@ QueryRaw::getNodeCacheFromWays(std::shared_ptr> ways, std::m } } +// Recieve a string of comma separated values of Nodes ids +// and return a vector of Ways std::list> QueryRaw::getWaysByNodesRefs(std::string &nodeIds) const { #ifdef TIMING_DEBUG boost::timer::auto_cpu_timer timer("getWaysByNodesRefs(nodeIds): took %w seconds\n"); #endif - // Get all ways that have references to nodes std::list> ways; + std::vector queries; - std::string waysQuery = "SELECT distinct(osm_id), refs, version, tags, uid, changeset from way_refs join ways_poly wp on wp.osm_id = way_id where node_id = any(ARRAY[" + nodeIds + "])"; - waysQuery += " UNION SELECT distinct(osm_id), refs, version, tags, uid, changeset from way_refs join ways_line wl on wl.osm_id = way_id where node_id = any(ARRAY[" + nodeIds + "]);"; - auto ways_result = dbconn->query(waysQuery); + // Get all Ways that have references to Nodes from the DB, including Polygons and LineString geometries + // std::string waysQuery = "SELECT distinct(osm_id), refs, version, tags, uid, changeset from way_refs join ways_poly wp on wp.osm_id = way_id where node_id = any(ARRAY[" + nodeIds + "])"; + queries.push_back("SELECT distinct(osm_id), refs, version, tags, uid, changeset from ways_poly where refs @> '{" + nodeIds + "}';"); + queries.push_back("SELECT distinct(osm_id), refs, version, tags, uid, changeset from ways_line where refs @> '{" + nodeIds + "}';"); - // Fill vector of OsmWay objects - for (auto way_it = ways_result.begin(); way_it != ways_result.end(); ++way_it) { - auto way = std::make_shared(); - way->id = (*way_it)[0].as(); - std::string refs_str = (*way_it)[1].as(); - if (refs_str.size() > 1) { - way->refs = arrayStrToVector(refs_str); + for (auto it = queries.begin(); it != queries.end(); ++it) { + + auto ways_result = dbconn->query(*it); + if (ways_result.size() == 0) { + log_debug("No results returned!"); + return ways; } - way->version = (*way_it)[2].as(); - auto tags = (*way_it)[3]; - if (!tags.is_null()) { - auto tags = parseJSONObjectStr((*way_it)[3].as()); - for (auto const& [key, val] : tags) - { - way->addTag(key, val); + + // Create Ways objects and fill the vector + for (auto way_it = ways_result.begin(); way_it != ways_result.end(); ++way_it) { + auto way = std::make_shared(); + way->id = (*way_it)[0].as(); + std::string refs_str = (*way_it)[1].as(); + if (refs_str.size() > 1) { + way->refs = arrayStrToVector(refs_str); } + way->version = (*way_it)[2].as(); + auto tags = (*way_it)[3]; + if (!tags.is_null()) { + auto tags = parseJSONObjectStr((*way_it)[3].as()); + for (auto const& [key, val] : tags) { + way->addTag(key, val); + } + } + auto uid = (*way_it)[4]; + if (!uid.is_null()) { + way->uid = (*way_it)[4].as(); + } + auto changeset = (*way_it)[5]; + if (!changeset.is_null()) { + way->changeset = (*way_it)[5].as(); + } + ways.push_back(way); } - auto uid = (*way_it)[4]; - if (!uid.is_null()) { - way->uid = (*way_it)[4].as(); - } - auto changeset = (*way_it)[5]; - if (!changeset.is_null()) { - way->changeset = (*way_it)[5].as(); - } - ways.push_back(way); } return ways; } +// Get the count of objects for a table (nodes, ways_poly, ways_line, relations) int QueryRaw::getCount(const std::string &tableName) { std::string query = "select count(osm_id) from " + tableName; auto result = dbconn->query(query); return result[0][0].as(); } +// Get a page of Nodes from the DB, using an id for sorting +// and a page size. This is useful for batch processing of Nodes, +// like the Bootstraping process. std::shared_ptr> QueryRaw::getNodesFromDB(long lastid, int pageSize) { std::string nodesQuery = "SELECT osm_id, ST_AsText(geom, 4326)"; @@ -729,18 +922,22 @@ QueryRaw::getNodesFromDB(long lastid, int pageSize) { } else { nodesQuery += ", version, tags FROM nodes order by osm_id desc limit " + std::to_string(pageSize) + ";"; } - + auto nodes = std::make_shared>(); auto nodes_result = dbconn->query(nodesQuery); + if (nodes_result.size() == 0) { + log_debug("No results returned!"); + return nodes; + } + // Fill vector of OsmNode objects - auto nodes = std::make_shared>(); for (auto node_it = nodes_result.begin(); node_it != nodes_result.end(); ++node_it) { OsmNode node; node.id = (*node_it)[0].as(); point_t point; std::string point_str = (*node_it)[1].as(); - boost::geometry::read_wkt(point_str, point); - node.setPoint(boost::geometry::get<0>(point), boost::geometry::get<1>(point)); + bg::read_wkt(point_str, point); + node.setPoint(bg::get<0>(point), bg::get<1>(point)); node.version = (*node_it)[2].as(); auto tags = (*node_it)[3]; if (!tags.is_null()) { @@ -757,6 +954,9 @@ QueryRaw::getNodesFromDB(long lastid, int pageSize) { } +// Get a page of Ways from the DB, using an id for sorting +// and a page size. This is useful for batch processing of Ways, +// like the Bootstraping process. std::shared_ptr> QueryRaw::getWaysFromDB(long lastid, int pageSize, const std::string &tableName) { std::string waysQuery; @@ -771,9 +971,14 @@ QueryRaw::getWaysFromDB(long lastid, int pageSize, const std::string &tableName) waysQuery += ", version, tags FROM " + tableName + " order by osm_id desc limit " + std::to_string(pageSize) + ";"; } + auto ways = std::make_shared>(); auto ways_result = dbconn->query(waysQuery); + if (ways_result.size() == 0) { + log_debug("No results returned!"); + return ways; + } + // Fill vector of OsmWay objects - auto ways = std::make_shared>(); for (auto way_it = ways_result.begin(); way_it != ways_result.end(); ++way_it) { OsmWay way; way.id = (*way_it)[0].as(); @@ -782,7 +987,7 @@ QueryRaw::getWaysFromDB(long lastid, int pageSize, const std::string &tableName) way.refs = arrayStrToVector(refs_str); std::string poly = (*way_it)[2].as(); - boost::geometry::read_wkt(poly, way.linestring); + bg::read_wkt(poly, way.linestring); if (tableName == QueryRaw::polyTable) { way.polygon = { {std::begin(way.linestring), std::end(way.linestring)} }; @@ -803,6 +1008,10 @@ QueryRaw::getWaysFromDB(long lastid, int pageSize, const std::string &tableName) return ways; } +// Get a page of Ways from the DB, using an id for sorting +// and a page size, but without using Refs. This is useful +// for batch processing of Ways that are not from OSM, like +// third party geospatial databases. std::shared_ptr> QueryRaw::getWaysFromDBWithoutRefs(long lastid, int pageSize, const std::string &tableName) { std::string waysQuery; @@ -817,15 +1026,20 @@ QueryRaw::getWaysFromDBWithoutRefs(long lastid, int pageSize, const std::string waysQuery += ", tags FROM " + tableName + " order by osm_id desc limit " + std::to_string(pageSize) + ";"; } + auto ways = std::make_shared>(); auto ways_result = dbconn->query(waysQuery); + if (ways_result.size() == 0) { + log_debug("No results returned!"); + return ways; + } + // Fill vector of OsmWay objects - auto ways = std::make_shared>(); for (auto way_it = ways_result.begin(); way_it != ways_result.end(); ++way_it) { OsmWay way; way.id = (*way_it)[0].as(); std::string poly = (*way_it)[1].as(); - boost::geometry::read_wkt(poly, way.linestring); + bg::read_wkt(poly, way.linestring); if (tableName == QueryRaw::polyTable) { way.polygon = { {std::begin(way.linestring), std::end(way.linestring)} }; @@ -845,6 +1059,10 @@ QueryRaw::getWaysFromDBWithoutRefs(long lastid, int pageSize, const std::string return ways; } + +// Get a page of Relations from the DB, using an id for sorting +// and a page size. This is useful for batch processing of Relations, +// like the Bootstraping process. std::shared_ptr> QueryRaw::getRelationsFromDB(long lastid, int pageSize) { std::string relationsQuery = "SELECT osm_id, refs, ST_AsText(geom, 4326)"; @@ -854,9 +1072,13 @@ QueryRaw::getRelationsFromDB(long lastid, int pageSize) { relationsQuery += ", version, tags FROM relations order by osm_id desc limit " + std::to_string(pageSize) + ";"; } + auto relations = std::make_shared>(); auto relations_result = dbconn->query(relationsQuery); + if (relations_result.size() == 0) { + log_debug("No results returned!"); + return relations; + } // Fill vector of OsmRelation objects - auto relations = std::make_shared>(); for (auto rel_it = relations_result.begin(); rel_it != relations_result.end(); ++rel_it) { OsmRelation relation; relation.id = (*rel_it)[0].as(); @@ -877,10 +1099,10 @@ QueryRaw::getRelationsFromDB(long lastid, int pageSize) { ); } std::string geometry = (*rel_it)[2].as(); - if (geometry.substr(0, 12) == "MULTIPOLYGON") { - boost::geometry::read_wkt(geometry, relation.multipolygon); + if (geometry.substr(0, 7) == "POLYGON") { + bg::read_wkt(geometry, relation.multipolygon); } else if (geometry.substr(0, 15) == "MULTILINESTRING") { - boost::geometry::read_wkt(geometry, relation.multilinestring); + bg::read_wkt(geometry, relation.multilinestring); } relation.version = (*rel_it)[3].as(); } @@ -898,7 +1120,6 @@ QueryRaw::getRelationsFromDB(long lastid, int pageSize) { return relations; } - } // namespace queryraw // local Variables: diff --git a/src/raw/queryraw.hh b/src/raw/queryraw.hh index 8bee5c6c7..d9a5bb616 100644 --- a/src/raw/queryraw.hh +++ b/src/raw/queryraw.hh @@ -1,5 +1,5 @@ // -// Copyright (c) 2023 Humanitarian OpenStreetMap Team +// Copyright (c) 2023, 2024 Humanitarian OpenStreetMap Team // // This file is part of Underpass. // @@ -23,7 +23,7 @@ /// \file queryraw.hh /// \brief This build raw queries for the database /// -/// This manages the OSM Raw schema in a postgres database. This +/// This manages the OSM Raw Data schema in a PostgreSQL DB. This /// includes building queries for existing data in the database, /// as well for updating the database. @@ -45,45 +45,48 @@ using namespace osmchange; /// \namespace queryraw namespace queryraw { -/// \class QueryStats -/// \brief This handles all direct database access +/// \class QueryRaw +/// \brief This handles all raw data database access /// -/// This class handles all the queries to the OSM Stats database. +/// This class handles all the queries to the OSM Raw Data PostgreSQL DB. /// This includes querying the database for existing data, as -/// well as updating the data whenh applying a replication file. +/// well as updating the data (geometries and tags) applying a replication file. class QueryRaw { public: QueryRaw(void); ~QueryRaw(void){}; QueryRaw(std::shared_ptr db); + // Name of the table for storing polygons static const std::string polyTable; + // Name of the table for storing linestrings static const std::string lineTable; /// Build query for processed Node - std::string applyChange(const OsmNode &node) const; + std::shared_ptr> applyChange(const OsmNode &node) const; /// Build query for processed Way - std::string applyChange(const OsmWay &way) const; + std::shared_ptr> applyChange(const OsmWay &way) const; /// Build query for processed Relation - std::string applyChange(const OsmRelation &relation) const; - /// Build all geometries for osmchanges + std::shared_ptr> applyChange(const OsmRelation &relation) const; + /// Build all geometries for a OsmChange file void buildGeometries(std::shared_ptr osmchanges, const multipolygon_t &poly); - /// Get nodes for filling Node cache from ways refs + /// Get nodes for filling Node cache from refs on ways void getNodeCacheFromWays(std::shared_ptr> ways, std::map &nodecache) const; - // Get ways by refs + // Get ways by node refs (used for ways geometries) std::list> getWaysByNodesRefs(std::string &nodeIds) const; - // Get ways by ids (used for getting relations geometries) + // Get ways by ids (used for relations geometries) void getWaysByIds(std::string &relsForWayCacheIds, std::map> &waycache); - // Get relations by referenced ways + // Get relations by referenced ways (used for relations geometries) std::list> getRelationsByWaysRefs(std::string &wayIds) const; - // DB connection + // OSM DB connection std::shared_ptr dbconn; - // Get ways count + // Get object (nodes, ways or relations) count from the database int getCount(const std::string &tableName); - // Build tags query + // Build tags query for insert tags into the databse std::string buildTagsQuery(std::map tags) const; // Get ways by page std::shared_ptr> getWaysFromDB(long lastid, int pageSize, const std::string &tableName); + // Get ways by page, without refs (useful for non OSM databases) std::shared_ptr> getWaysFromDBWithoutRefs(long lastid, int pageSize, const std::string &tableName); // Get nodes by page std::shared_ptr> getNodesFromDB(long lastid, int pageSize); diff --git a/src/replicator/replication.cc b/src/replicator/replication.cc index 33416657b..c8ffc8bdc 100644 --- a/src/replicator/replication.cc +++ b/src/replicator/replication.cc @@ -1,5 +1,5 @@ // -// Copyright (c) 2020, 2021, 2022, 2023 Humanitarian OpenStreetMap Team +// Copyright (c) 2020, 2021, 2022, 2023, 2024 Humanitarian OpenStreetMap Team // // This file is part of Underpass. // diff --git a/src/replicator/threads.cc b/src/replicator/threads.cc index 22d702e16..5b09cee03 100644 --- a/src/replicator/threads.cc +++ b/src/replicator/threads.cc @@ -1,5 +1,5 @@ // -// Copyright (c) 2020, 2021, 2022, 2023 Humanitarian OpenStreetMap Team +// Copyright (c) 2020, 2021, 2022, 2023, 2024 Humanitarian OpenStreetMap Team // // This file is part of Underpass. // @@ -98,12 +98,26 @@ using namespace underpassconfig; namespace replicatorthreads { -std::string +std::shared_ptr> allTasksQueries(std::shared_ptr> tasks) { - std::string queries = ""; + auto queries = std::make_shared>(); + std::string osmsql; + std::string unsql; for (auto it = tasks->begin(); it != tasks->end(); ++it) { - queries += it->query; + for (auto itt = it->query.begin(); itt != it->query.end(); ++itt) { + if (itt->rfind(';') != itt->size() - 1) { + log_debug("HAS SEMI-COLON: %1%", *itt); + log_debug("HAS SEMI-COLON: %1% %2%", itt->size() - 1, itt->rfind(';')); + } + if (itt->find(" nodes ") != std::string::npos || itt->find(" ways_poly ") != std::string::npos || itt->find(" ways_line ") != std::string::npos || itt->find(" relations ") != std::string::npos) { + unsql.append(*itt); + } else { + osmsql.append(*itt); + } + } } + queries->push_back(osmsql); + queries->push_back(unsql); return queries; } @@ -139,14 +153,22 @@ startMonitorChangesets(std::shared_ptr &remote, assert(remote->frequency == frequency_t::changeset); auto db = std::make_shared(); - if (!db->connect(config.underpass_db_url)) { + if (!db->connect(config.underpass_osm_db_url)) { log_error("Could not connect to Underpass DB, aborting monitoring thread!"); return; } else { - log_debug("Connected to database: %1%", config.underpass_db_url); + log_debug("Connected to database: %1%", config.underpass_osm_db_url); } auto querystats = std::make_shared(db); + auto osmdb = std::make_shared(); + if (!osmdb->connect(config.underpass_osm_db_url)) { + log_error("Could not connect to Underpass DB, aborting monitoring thread!"); + return; + } else { + log_debug("Connected to database: %1%", config.underpass_osm_db_url); + } + int cores = config.concurrency; // Support multiple OSM planet servers @@ -200,7 +222,13 @@ startMonitorChangesets(std::shared_ptr &remote, remote->updateDomain(planets.front()->domain); } pool.join(); - db->query(allTasksQueries(tasks)); + auto result = allTasksQueries(tasks); + if (result->at(0).size() > 0) { + db->query(result->at(0)); + } + if (result->at(1).size() > 0) { + osmdb->query(result->at(1)); + } ptime now = boost::posix_time::second_clock::universal_time(); last_task = getClosest(tasks, now); @@ -275,7 +303,16 @@ startMonitorChanges(std::shared_ptr &remote, } auto querystats = std::make_shared(db); auto queryvalidate = std::make_shared(db); - auto queryraw = std::make_shared(db); + + // Connect to the raw OSM database, which is separate + auto osmdb = std::make_shared(); + if (!osmdb->connect(config.underpass_osm_db_url)) { + log_error("Could not connect to raw OSM DB, aborting monitoring thread!"); + return; + } else { + log_debug("Connected to database: %1%", config.underpass_osm_db_url); + } + auto queryraw = std::make_shared(osmdb); int cores = config.concurrency; @@ -339,7 +376,13 @@ startMonitorChanges(std::shared_ptr &remote, boost::asio::post(pool, task); } while (--i); pool.join(); - db->query(allTasksQueries(tasks)); + auto result = allTasksQueries(tasks); + if (result->at(0).size() > 0) { + db->query(result->at(0)); + } + if (result->at(1).size() > 0) { + osmdb->query(result->at(1)); + } ptime now = boost::posix_time::second_clock::universal_time(); last_task = getClosest(tasks, now); @@ -401,7 +444,7 @@ threadChangeSet(std::shared_ptr &remote, log_debug("ChangeSet last_closed_at: %1%", task.timestamp); changeset->areaFilter(poly); for (auto cit = std::begin(changeset->changes); cit != std::end(changeset->changes); ++cit) { - task.query += querystats->applyChange(*cit->get()); + task.query.push_back(querystats->applyChange(*cit->get())); } } const std::lock_guard lock(tasks_changeset_mutex); @@ -412,7 +455,9 @@ threadChangeSet(std::shared_ptr &remote, void threadOsmChange(OsmChangeTask osmChangeTask) { - +#ifdef TIMING_DEBUG + boost::timer::auto_cpu_timer timer("threadOsmChange: took %w seconds\n"); +#endif auto remote = osmChangeTask.remote; auto planet = osmChangeTask.planet; auto poly = osmChangeTask.poly; @@ -425,9 +470,6 @@ threadOsmChange(OsmChangeTask osmChangeTask) auto taskIndex = osmChangeTask.taskIndex; auto osmchanges = std::make_shared(); -#ifdef TIMING_DEBUG - boost::timer::auto_cpu_timer timer("threadOsmChange: took %w seconds\n"); -#endif log_debug("Processing OsmChange: %1%", remote->filespec); ReplicationTask task; task.url = remote->subpath; @@ -450,10 +492,11 @@ threadOsmChange(OsmChangeTask osmChangeTask) try { osmchanges->nodecache.clear(); + osmchanges->waycache.clear(); osmchanges->readXML(changes_xml); if (osmchanges->changes.size() > 0) { task.timestamp = osmchanges->changes.back()->final_entry; - log_debug("OsmChange final_entry: %1%", task.timestamp); + // log_debug("OsmChange final_entry: %1%", task.timestamp); } } catch (std::exception &e) { log_error("Couldn't parse: %1%", remote->filespec); @@ -469,10 +512,10 @@ threadOsmChange(OsmChangeTask osmChangeTask) } // - Fill node cache with nodes referenced in modified - // or created ways and also ways affected by modified nodes + // or created ways and also ways indirectly modified by modified nodes // - Add indirectly modified ways to osmchanges - // - Build ways geometries using nodecache - // - Build relation multipolyon geometries + // - Build ways polygon/linestring geometries using nodecache + // - Build relation multipolyon/multilinestring geometries using waycache if (!config->disable_raw) { queryraw->buildGeometries(osmchanges, poly); } @@ -487,13 +530,13 @@ threadOsmChange(OsmChangeTask osmChangeTask) if (it->second->added.size() == 0 && it->second->modified.size() == 0) { continue; } - task.query += querystats->applyChange(*it->second); + task.query.push_back(querystats->applyChange(*it->second)); } } auto removed_nodes = std::make_shared>(); auto removed_ways = std::make_shared>(); - // auto removed_relations = std::make_shared>(); + auto removed_relations = std::make_shared>(); auto validation_removals = std::make_shared>(); // Raw data and validation @@ -516,7 +559,10 @@ threadOsmChange(OsmChangeTask osmChangeTask) // Update nodes, ignore new ones outside priority area if (!config->disable_raw) { - task.query += queryraw->applyChange(*node); + auto changes = queryraw->applyChange(*node); + for (auto it = changes->begin(); it != changes->end(); ++it) { + task.query.push_back(*it); + } } } @@ -535,53 +581,70 @@ threadOsmChange(OsmChangeTask osmChangeTask) // Update ways, ignore new ones outside priority area if (!config->disable_raw) { - task.query += queryraw->applyChange(*way); + auto changes = queryraw->applyChange(*way); + for (auto it = changes->begin(); it != changes->end(); ++it) { + task.query.push_back(*it); + } } } - // // Relations - // for (auto rit = std::begin(change->relations); rit != std::end(change->relations); ++rit) { - // osmobjects::OsmRelation *relation = rit->get(); - - // if (relation->action != osmobjects::remove && !relation->priority) { - // continue; - // } - // // Remove deleted relations from validation table - // if (!config->disable_validation && relation->action == osmobjects::remove) { - // removed_relations->push_back(relation->id); - // } + // Relations + for (auto rit = std::begin(change->relations); rit != std::end(change->relations); ++rit) { + osmobjects::OsmRelation *relation = rit->get(); - // // Update relations, ignore new ones outside priority area - // if (!config->disable_raw) { - // task.query += queryraw->applyChange(*relation); - // } - // } + if (relation->action != osmobjects::remove && !relation->priority) { + // if (!relation->priority) { + // std::cout << "id << "> NOT PRIORITY" << std::endl; + // } + continue; + } + // Remove deleted relations from validation table + // if (!config->disable_validation && relation->action == osmobjects::remove) { + // removed_relations->push_back(relation->id); + // } + // Update relations, ignore new ones outside priority area + if (!config->disable_raw) { + auto changes = queryraw->applyChange(*relation); + for (auto it = changes->begin(); it != changes->end(); ++it) { + task.query.push_back(*it); + } + } + } } } - // // Update validation table + // Update validation table if (!config->disable_validation) { // Validate ways auto wayval = osmchanges->validateWays(poly, plugin); - queryvalidate->ways(wayval, task.query, validation_removals); + auto wayval_queries = queryvalidate->ways(wayval, validation_removals); + for (auto itt = wayval_queries->begin(); itt != wayval_queries->end(); ++itt) { + task.query.push_back(*itt); + } // Validate nodes auto nodeval = osmchanges->validateNodes(poly, plugin); - queryvalidate->nodes(nodeval, task.query, validation_removals); + auto nodeval_queries = queryvalidate->nodes(nodeval, validation_removals); + for (auto itt = nodeval_queries->begin(); itt != nodeval_queries->end(); ++itt) { + task.query.push_back(*itt); + } // Validate relations - // task.query += queryvalidate->rels(wayval, task.query, validation_removals); + // auto relval = osmchanges->validateRelations(poly, plugin); + // auto relval_queries = queryvalidate->relations(relval, validation_removals); + // for (auto itt = relval_queries->begin(); itt != relval_queries->end(); ++itt) { + // task.query.push_back(*itt); + // } // Remove validation entries for removed objects - task.query += queryvalidate->updateValidation(validation_removals); - task.query += queryvalidate->updateValidation(removed_nodes); - task.query += queryvalidate->updateValidation(removed_ways); + task.query.push_back(*queryvalidate->updateValidation(validation_removals)); + task.query.push_back(*queryvalidate->updateValidation(removed_nodes)); + task.query.push_back(*queryvalidate->updateValidation(removed_ways)); // task.query += queryvalidate->updateValidation(removed_relations); } - const std::lock_guard lock(tasks_change_mutex); (*tasks)[taskIndex] = task; diff --git a/src/replicator/threads.hh b/src/replicator/threads.hh index dca7727a0..fbd269aba 100644 --- a/src/replicator/threads.hh +++ b/src/replicator/threads.hh @@ -96,7 +96,7 @@ struct ReplicationTask { std::string url; ptime timestamp = not_a_date_time; replication::reqfile_t status = replication::reqfile_t::none; - std::string query = ""; + std::vector query; }; /// This monitors the planet server for new changesets files. diff --git a/src/stats/querystats.cc b/src/stats/querystats.cc index 437c53e0d..c5b71c50b 100644 --- a/src/stats/querystats.cc +++ b/src/stats/querystats.cc @@ -1,5 +1,5 @@ // -// Copyright (c) 2020, 2021, 2022, 2023 Humanitarian OpenStreetMap Team +// Copyright (c) 2020, 2021, 2022, 2023, 2024 Humanitarian OpenStreetMap Team // // This file is part of Underpass. // diff --git a/src/testsuite/libunderpass.all/change-test.cc b/src/testsuite/libunderpass.all/change-test.cc index e2976ee89..0b7905400 100644 --- a/src/testsuite/libunderpass.all/change-test.cc +++ b/src/testsuite/libunderpass.all/change-test.cc @@ -1,5 +1,5 @@ // -// Copyright (c) 2020, 2021, 2022, 2023 Humanitarian OpenStreetMap Team +// Copyright (c) 2020, 2021, 2022, 2023, 2024 Humanitarian OpenStreetMap Team // // This file is part of Underpass. // diff --git a/src/testsuite/libunderpass.all/geo-test.cc b/src/testsuite/libunderpass.all/geo-test.cc index 41060e6bf..0d9ef6616 100644 --- a/src/testsuite/libunderpass.all/geo-test.cc +++ b/src/testsuite/libunderpass.all/geo-test.cc @@ -1,5 +1,5 @@ // -// Copyright (c) 2020, 2021, 2022, 2023 Humanitarian OpenStreetMap Team +// Copyright (c) 2020, 2021, 2022, 2023, 2024 Humanitarian OpenStreetMap Team // // This file is part of Underpass. // diff --git a/src/testsuite/libunderpass.all/hashtags-test.cc b/src/testsuite/libunderpass.all/hashtags-test.cc index 663748b8c..83cf49762 100644 --- a/src/testsuite/libunderpass.all/hashtags-test.cc +++ b/src/testsuite/libunderpass.all/hashtags-test.cc @@ -1,5 +1,5 @@ // -// Copyright (c) 2020, 2021, 2022, 2023 Humanitarian OpenStreetMap Team +// Copyright (c) 2020, 2021, 2022, 2023, 2024 Humanitarian OpenStreetMap Team // // This file is part of Underpass. // diff --git a/src/testsuite/libunderpass.all/planetreplicator-test.cc b/src/testsuite/libunderpass.all/planetreplicator-test.cc index 8997eb870..592d76bca 100644 --- a/src/testsuite/libunderpass.all/planetreplicator-test.cc +++ b/src/testsuite/libunderpass.all/planetreplicator-test.cc @@ -1,5 +1,5 @@ // -// Copyright (c) 2020, 2021, 2022, 2023 Humanitarian OpenStreetMap Team +// Copyright (c) 2020, 2021, 2022, 2023, 2024 Humanitarian OpenStreetMap Team // // This file is part of Underpass. // diff --git a/src/testsuite/libunderpass.all/pq-test.cc b/src/testsuite/libunderpass.all/pq-test.cc index 4f4ed3dcd..3df083127 100644 --- a/src/testsuite/libunderpass.all/pq-test.cc +++ b/src/testsuite/libunderpass.all/pq-test.cc @@ -1,5 +1,5 @@ // -// Copyright (c) 2020, 2021, 2022, 2023 Humanitarian OpenStreetMap Team +// Copyright (c) 2020, 2021, 2022, 2023, 2024 Humanitarian OpenStreetMap Team // // This file is part of Underpass. // diff --git a/src/testsuite/libunderpass.all/raw-test.cc b/src/testsuite/libunderpass.all/raw-test.cc index 229f4bbe1..4890b841b 100644 --- a/src/testsuite/libunderpass.all/raw-test.cc +++ b/src/testsuite/libunderpass.all/raw-test.cc @@ -83,52 +83,73 @@ bool processFile(const std::string &filename, std::shared_ptr &db) { osmchanges->readChanges(destdir_base + "/testsuite/testdata/raw/" + filename); queryraw->buildGeometries(osmchanges, poly); osmchanges->areaFilter(poly); - std::string rawquery; + auto rawquery = std::make_shared>(); for (auto it = std::begin(osmchanges->changes); it != std::end(osmchanges->changes); ++it) { osmchange::OsmChange *change = it->get(); // Nodes for (auto nit = std::begin(change->nodes); nit != std::end(change->nodes); ++nit) { osmobjects::OsmNode *node = nit->get(); - rawquery += queryraw->applyChange(*node); + auto changes = *queryraw->applyChange(*node); + for (auto it = changes.begin(); it != changes.end(); ++it) { + rawquery->push_back(*it); + } } // Ways for (auto wit = std::begin(change->ways); wit != std::end(change->ways); ++wit) { osmobjects::OsmWay *way = wit->get(); - rawquery += queryraw->applyChange(*way); + auto changes = *queryraw->applyChange(*way); + for (auto it = changes.begin(); it != changes.end(); ++it) { + rawquery->push_back(*it); + } } // Relations for (auto rit = std::begin(change->relations); rit != std::end(change->relations); ++rit) { osmobjects::OsmRelation *relation = rit->get(); - rawquery += queryraw->applyChange(*relation); + auto changes = *queryraw->applyChange(*relation); + for (auto it = changes.begin(); it != changes.end(); ++it) { + rawquery->push_back(*it); + } } } - db->query(rawquery); + + for (auto rit = std::begin(*rawquery); rit != std::end(*rawquery); ++rit) { + db->query(*rit); + } } const std::vector expectedGeometries = { - "POLYGON((21.726001473 4.62042952837,21.726086573 4.62042742837,21.726084973 4.62036492836,21.725999873 4.62036702836,21.726001473 4.62042952837))", - "POLYGON((21.726001473 4.62042952837,21.726086573 4.62042742837,21.726084973 4.62036492836,21.725999873 4.62036702836,21.726001473 4.62042952837))", - "POLYGON((21.72600148 4.62042953,21.726086573 4.62042742837,21.726084973 4.62036492836,21.725999873 4.62036702836,21.72600148 4.62042953))", - "MULTIPOLYGON(((21.72600148 4.62042953,21.726086573 4.62042742837,21.726084973 4.62036492836,21.725999873 4.62036702836,21.72600148 4.62042953),(21.7260170728 4.62041343508,21.7260713875 4.62041326798,21.7260708846 4.62037684165,21.7260165699 4.62038035061,21.7260170728 4.62041343508)))", - "MULTIPOLYGON(((21.72600148 4.62042953,21.726086573 4.62042742837,21.7260807753 4.62037032501,21.725999873 4.62036702836,21.72600148 4.62042953),(21.7260170728 4.62041343508,21.7260713875 4.62041326798,21.7260708846 4.62037684165,21.7260165699 4.62038035061,21.7260170728 4.62041343508)))", - "MULTILINESTRING((21.726001473 4.62042952837,21.726086573 4.62042742837,21.726084973 4.62036492836),(21.726084973 4.62036492836,21.725999873 4.62036702836,21.726001473 4.62042952837))", - "MULTIPOLYGON(((-59.8355946 -36.7448782,-59.8353203 -36.7452085,-59.8368532 -36.7480926,-59.8370667 -36.7483355,-59.8408639 -36.7515154,-59.8482593 -36.745907,-59.8486299 -36.7452032,-59.848532 -36.7453269,-59.8483958 -36.7456167,-59.8482593 -36.745907,-59.8486299 -36.7452032,-59.8486213 -36.7451521,-59.8481455 -36.7432981,-59.8478834 -36.7425679,-59.8478326 -36.7423963,-59.8478326 -36.7423963,-59.8477902 -36.7422529,-59.8477764 -36.7422007,-59.8477733 -36.74215,-59.8477773 -36.742095,-59.8477972 -36.7420329,-59.8478247 -36.7419783,-59.8478609 -36.7419236,-59.8479126 -36.7418808,-59.84795 -36.7418607,-59.8480462 -36.7418147,-59.8480692 -36.7417819,-59.8480709 -36.7417548,-59.8480673 -36.7416893,-59.8479835 -36.741163,-59.8479787 -36.7411211,-59.8479562 -36.7410819,-59.847923 -36.7410414,-59.8478722 -36.7410094,-59.8475273 -36.7408127,-59.8474352 -36.7407496,-59.8473715 -36.7406912,-59.8473254 -36.7406329,-59.8472829 -36.7405588,-59.8472494 -36.7404962,-59.8472589 -36.7404251,-59.8472978 -36.7403582,-59.8473424 -36.74032,-59.8473864 -36.7402921,-59.8474375 -36.740274,-59.8474922 -36.740256,-59.8475468 -36.7402465,-59.8476241 -36.7402304,-59.8476784 -36.7402171,-59.8477563 -36.7401919,-59.8478265 -36.7401545,-59.8479128 -36.7400779,-59.8479937 -36.7399885,-59.8480801 -36.7398796,-59.8481504 -36.7397665,-59.848411 -36.7391216,-59.8484023 -36.739028,-59.8483557 -36.7389343,-59.8482783 -36.7388702,-59.84822 -36.7388242,-59.8481308 -36.7388079,-59.8480003 -36.7387583,-59.8478575 -36.7386841,-59.8477933 -36.738618,-59.8477365 -36.7385158,-59.8477106 -36.7384235,-59.8477053 -36.7382963,-59.8477134 -36.7381998,-59.8477433 -36.7381126,-59.8478321 -36.738022,-59.8479105 -36.7379644,-59.8480011 -36.7379216,-59.8482127 -36.7378122,-59.8482877 -36.7377698,-59.8483566 -36.7377126,-59.848393 -36.737632,-59.8484294 -36.7375366,-59.8485761 -36.7372026,-59.848605 -36.7370773,-59.8486246 -36.7369372,-59.8486196 -36.7368366,-59.8485807 -36.7367015,-59.848529 -36.7365836,-59.8484717 -36.7364835,-59.8483887 -36.7363497,-59.8477548 -36.7356502,-59.8477339 -36.7356248,-59.8477339 -36.7356248,-59.8475634 -36.7357007,-59.8474292 -36.7357691,-59.8473073 -36.7358571,-59.8469617 -36.7361243,-59.8447338 -36.737825,-59.8424572 -36.7395354,-59.8423067 -36.7396527,-59.8386641 -36.7424968,-59.838225 -36.7428388,-59.8355946 -36.7448782)))", - "MULTIPOLYGON(((-69.0344971 -33.6875005,-69.0354574 -33.6880228,-69.0356076 -33.6879112,-69.0360421 -33.6881656,-69.0362352 -33.6883218,-69.0369111 -33.6886075,-69.0375173 -33.6871078,-69.0367931 -33.686581,-69.0366161 -33.6866525,-69.0361923 -33.6865766,-69.0364122 -33.6860945,-69.0368253 -33.68634,-69.0370399 -33.6864739,-69.037512 -33.6861168,-69.0374959 -33.6859115,-69.0376568 -33.6857687,-69.037866 -33.6856838,-69.037351 -33.6853401,-69.0371311 -33.6842242,-69.0365088 -33.68376,-69.0362889 -33.6832065,-69.036144 -33.6823673,-69.0358865 -33.6818227,-69.0358973 -33.6817468,-69.03557 -33.6815816,-69.0359187 -33.6810459,-69.0351462 -33.6804031,-69.0349263 -33.6798541,-69.0345454 -33.67968,-69.0342611 -33.6794791,-69.0337515 -33.6794836,-69.0332151 -33.6793095,-69.0331185 -33.6788586,-69.0329737 -33.6786354,-69.0327162 -33.6783541,-69.0325767 -33.6782024,-69.0321851 -33.6780238,-69.0315521 -33.6776756,-69.0312892 -33.6774881,-69.0311068 -33.6773453,-69.0308118 -33.6771131,-69.0305275 -33.6769747,-69.0303397 -33.6769033,-69.0300447 -33.676948,-69.0298086 -33.6769256,-69.0288216 -33.6784256,-69.0287519 -33.6783742,-69.0296692 -33.6768676,-69.0292937 -33.6765819,-69.0289557 -33.6765596,-69.0286982 -33.6766355,-69.0284568 -33.6765685,-69.0282691 -33.6768363,-69.0279794 -33.6769122,-69.0277594 -33.6767917,-69.0276629 -33.6770015,-69.0274376 -33.6768988,-69.0273035 -33.6768899,-69.0270782 -33.6770328,-69.0268743 -33.6769301,-69.0265229 -33.6775261,-69.0258417 -33.6786086,-69.0233284 -33.6774703,-69.0232372 -33.6775238,-69.0232211 -33.6781176,-69.0231353 -33.6783408,-69.0233445 -33.6788229,-69.0233767 -33.6791979,-69.0232158 -33.6794925,-69.0233231 -33.6796264,-69.022733 -33.6803049,-69.0227705 -33.6806933,-69.0227598 -33.6811084,-69.0223092 -33.6817245,-69.0238702 -33.6824744,-69.0240419 -33.6822735,-69.0255761 -33.6830234,-69.0254581 -33.6832243,-69.0291756 -33.6849295,-69.0293527 -33.6846617,-69.0311283 -33.6854607,-69.0308976 -33.6858222,-69.0337139 -33.6871256,-69.0343523 -33.6862061,-69.0349907 -33.6865409,-69.0343738 -33.6874604,-69.0344971 -33.6875005),(-69.0305328 -33.683019,-69.0311444 -33.6821173,-69.0274215 -33.6803451,-69.0263647 -33.6819119,-69.0305328 -33.683019)))", - "MULTIPOLYGON(((-70.3434661 -38.5210725,-70.3434387 -38.5210955,-70.3434261 -38.5211299,-70.3434276 -38.5211716,-70.3434429 -38.5212067,-70.3434724 -38.5212373,-70.3435173 -38.5212595,-70.3435951 -38.5212788,-70.3436942 -38.5212955,-70.3438252 -38.52131479999999,-70.3439597 -38.521337,-70.3441048 -38.521359100000005,-70.3442463 -38.52138410000001,-70.3443808 -38.5214145,-70.3445082 -38.5214533,-70.3446177 -38.521495599999994,-70.3447356 -38.52154330000001,-70.3448535 -38.52158469999999,-70.3449795 -38.5216101,-70.3451015 -38.521622900000004,-70.3452113 -38.52164189999999,-70.3453088 -38.5216674,-70.3454192 -38.521701,-70.3455168 -38.5217296,-70.3456184 -38.5217582,-70.3457445 -38.5217805,-70.3458786 -38.52179,-70.3460209 -38.52179,-70.3471381 -38.5217576,-70.3472051 -38.52175339999999,-70.3472561 -38.52174300000001,-70.3473044 -38.52172410000001,-70.3473446 -38.521701,-70.3473795 -38.52166949999999,-70.3474116 -38.5216422,-70.3474492 -38.521612800000014,-70.3474921 -38.5215835,-70.3475323 -38.5215541,-70.3476101 -38.52149739999999,-70.3475739 -38.52162409999999,-70.3475456 -38.521693299999995,-70.3475137 -38.52176259999999,-70.3474677 -38.5218235,-70.3474253 -38.5218674,-70.3473686 -38.52190379999999,-70.3473085 -38.5219342,-70.3472413 -38.521954,-70.347174 -38.5219675,-70.3472817 -38.5219659,-70.3474005 -38.5219536,-70.3475173 -38.52192869999999,-70.3476447 -38.5218955,-70.3477721 -38.52184559999999,-70.3479455 -38.52176259999999,-70.3479066 -38.52183180000001,-70.3478641 -38.5218789,-70.3478075 -38.52192869999999,-70.3477355 -38.5219766,-70.3479018 -38.5219262,-70.3480467 -38.5218675,-70.3481915 -38.521791900000004,-70.3483042 -38.52172060000001,-70.3484135 -38.521645,-70.3486227 -38.5215149,-70.3488534 -38.52138899999999,-70.3491002 -38.5212799,-70.3493684 -38.5211918,-70.3496098 -38.5211162,-70.3498834 -38.5210491,-70.3501301 -38.5210029,-70.3503447 -38.5209693,-70.3505593 -38.520944100000015,-70.3507792 -38.5209204,-70.3507792 -38.5209204,-70.3505605 -38.52082279999999,-70.350236 -38.5206906,-70.3498659 -38.5205962,-70.3495735 -38.5205458,-70.3492637 -38.5204975,-70.3491282 -38.52049329999999,-70.3490075 -38.520505899999996,-70.3488064 -38.5205469,-70.3488064 -38.5205469,-70.3485167 -38.5206098,-70.3478837 -38.5207179,-70.3473875 -38.5207808,-70.3468376 -38.5208333,-70.3461456 -38.5208816,-70.3455448 -38.520934,-70.3449628 -38.5209949,-70.3443834 -38.52103060000001,-70.343796 -38.52105360000001,-70.3434661 -38.5210725)),((-70.3393449 -38.5210096,-70.3397998 -38.5209924,-70.3400091 -38.5209944,-70.3401816 -38.521013700000005,-70.3403726 -38.5210619,-70.340539 -38.52112459999999,-70.3406807 -38.5211969,-70.3408225 -38.5212789,-70.3409704 -38.5213898,-70.3410954 -38.521514500000016,-70.341237 -38.5216474,-70.3414069 -38.5217803,-70.3415626 -38.52188550000001,-70.3417466 -38.52196860000001,-70.3419731 -38.5220406,-70.3421855 -38.522096,-70.3423908 -38.5221458,-70.3426173 -38.5222012,-70.3428155 -38.522262099999985,-70.3430137 -38.52233410000001,-70.3432119 -38.5224338,-70.3433534 -38.5225058,-70.343495 -38.5225889,-70.3436649 -38.522660899999984,-70.3438772 -38.5226885,-70.3440683 -38.522682999999994,-70.3442524 -38.522660899999984,-70.3449444 -38.522532899999995,-70.3450571 -38.5224952,-70.3451644 -38.522428000000005,-70.3452341 -38.5223483,-70.3452663 -38.5222769,-70.3452717 -38.52220139999999,-70.3452448 -38.5221342,-70.3451966 -38.5220797,-70.3451 -38.5220335,-70.3449444 -38.52198729999999,-70.3447513 -38.52195369999999,-70.3445624 -38.52194130000001,-70.343915 -38.52190679999999,-70.3437005 -38.5218859,-70.3434859 -38.5218565,-70.3432337 -38.5217935,-70.3429923 -38.521717999999986,-70.3427402 -38.5216215,-70.3425364 -38.5215333,-70.3423486 -38.5214368,-70.3421877 -38.521336100000006,-70.3420589 -38.5212353,-70.341948 -38.5211271,-70.341948 -38.5211271,-70.3415966 -38.5210893,-70.3412157 -38.521032700000006,-70.3409931 -38.5209875,-70.3408737 -38.5209414,-70.3406967 -38.5208417,-70.3405653 -38.5207452,-70.3404205 -38.5206245,-70.3402944 -38.520521699999996,-70.3401375 -38.52040730000002,-70.3399175 -38.520231,-70.3397392 -38.5200925,-70.3396024 -38.5200075,-70.3394187 -38.519918299999986,-70.3392309 -38.519868,-70.3388715 -38.5197882,-70.3384933 -38.5197012,-70.3380534 -38.5196015,-70.3377664 -38.519549000000005,-70.3372943 -38.519500699999995,-70.3368531 -38.5194703,-70.3365098 -38.5194577,-70.3361759 -38.5194399,-70.3359184 -38.5194567,-70.3359184 -38.5194567,-70.3362585 -38.519638999999984,-70.3365107 -38.5197779,-70.3367348 -38.51993860000001,-70.3369777 -38.520114,-70.3372112 -38.5202456,-70.3376501 -38.5204356,-70.3381078 -38.520632899999995,-70.3384721 -38.5208009,-70.3387149 -38.520874,-70.3390231 -38.52093250000001,-70.3393449 -38.5210096)))" + "POLYGON((21.7260014 4.6204295,21.7260865 4.6204274,21.7260849 4.6203649,21.7259998 4.620367,21.7260014 4.6204295))", + "POLYGON((21.7260014 4.6204295,21.7260865 4.6204274,21.7260849 4.6203649,21.7259998 4.620367,21.7260014 4.6204295))", + "POLYGON((21.7260114 4.6204395,21.7260865 4.6204274,21.7260849 4.6203649,21.7259998 4.620367,21.7260114 4.6204395))", + "POLYGON((21.7260114 4.6204395,21.7260865 4.6204274,21.7260849 4.6203649,21.7259998 4.620367,21.7260114 4.6204395),(21.726017 4.6204134,21.72607138 4.6204132,21.72607088 4.6203768,21.72601656 4.6203803,21.726017 4.6204134))", + "POLYGON((21.7260114 4.6204395,21.7260865 4.6204274,21.7260807 4.6203703,21.7259998 4.620367,21.7260114 4.6204395),(21.726017 4.6204134,21.72607138 4.6204132,21.72607088 4.6203768,21.72601656 4.6203803,21.726017 4.6204134))", + "MULTILINESTRING((21.7260849 4.6203649,21.7260865 4.6204274,21.7260014 4.6204295,21.7260014 4.6204295,21.7259998 4.620367,21.7260849 4.6203649))", + "POLYGON((-59.8355946 -36.7448782,-59.8353203 -36.7452085,-59.8368532 -36.7480926,-59.8370667 -36.7483355,-59.8408639 -36.7515154,-59.8482593 -36.745907,-59.8482593 -36.745907,-59.8483958 -36.7456167,-59.848532 -36.7453269,-59.8486299 -36.7452032,-59.8486299 -36.7452032,-59.8486213 -36.7451521,-59.8481455 -36.7432981,-59.8478834 -36.7425679,-59.8478326 -36.7423963,-59.8478326 -36.7423963,-59.8477902 -36.7422529,-59.8477764 -36.7422007,-59.8477733 -36.74215,-59.8477773 -36.742095,-59.8477972 -36.7420329,-59.8478247 -36.7419783,-59.8478609 -36.7419236,-59.8479126 -36.7418808,-59.84795 -36.7418607,-59.8480462 -36.7418147,-59.8480692 -36.7417819,-59.8480709 -36.7417548,-59.8480673 -36.7416893,-59.8479835 -36.741163,-59.8479787 -36.7411211,-59.8479562 -36.7410819,-59.847923 -36.7410414,-59.8478722 -36.7410094,-59.8475273 -36.7408127,-59.8474352 -36.7407496,-59.8473715 -36.7406912,-59.8473254 -36.7406329,-59.8472829 -36.7405588,-59.8472494 -36.7404962,-59.8472589 -36.7404251,-59.8472978 -36.7403582,-59.8473424 -36.74032,-59.8473864 -36.7402921,-59.8474375 -36.740274,-59.8474922 -36.740256,-59.8475468 -36.7402465,-59.8476241 -36.7402304,-59.8476784 -36.7402171,-59.8477563 -36.7401919,-59.8478265 -36.7401545,-59.8479128 -36.7400779,-59.8479937 -36.7399885,-59.8480801 -36.7398796,-59.8481504 -36.7397665,-59.848411 -36.7391216,-59.8484023 -36.739028,-59.8483557 -36.7389343,-59.8482783 -36.7388702,-59.84822 -36.7388242,-59.8481308 -36.7388079,-59.8480003 -36.7387583,-59.8478575 -36.7386841,-59.8477933 -36.738618,-59.8477365 -36.7385158,-59.8477106 -36.7384235,-59.8477053 -36.7382963,-59.8477134 -36.7381998,-59.8477433 -36.7381126,-59.8478321 -36.738022,-59.8479105 -36.7379644,-59.8480011 -36.7379216,-59.8482127 -36.7378122,-59.8482877 -36.7377698,-59.8483566 -36.7377126,-59.848393 -36.737632,-59.8484294 -36.7375366,-59.8485761 -36.7372026,-59.848605 -36.7370773,-59.8486246 -36.7369372,-59.8486196 -36.7368366,-59.8485807 -36.7367015,-59.848529 -36.7365836,-59.8484717 -36.7364835,-59.8483887 -36.7363497,-59.8477548 -36.7356502,-59.8477339 -36.7356248,-59.8477339 -36.7356248,-59.8475634 -36.7357007,-59.8474292 -36.7357691,-59.8473073 -36.7358571,-59.8469617 -36.7361243,-59.8447338 -36.737825,-59.8424572 -36.7395354,-59.8423067 -36.7396527,-59.8386641 -36.7424968,-59.838225 -36.7428388,-59.8355946 -36.7448782))", + "POLYGON((-69.0344971 -33.6875005,-69.0354574 -33.6880228,-69.0356076 -33.6879112,-69.0360421 -33.6881656,-69.0362352 -33.6883218,-69.0369111 -33.6886075,-69.0375173 -33.6871078,-69.0367931 -33.686581,-69.0366161 -33.6866525,-69.0361923 -33.6865766,-69.0364122 -33.6860945,-69.0368253 -33.68634,-69.0370399 -33.6864739,-69.037512 -33.6861168,-69.0374959 -33.6859115,-69.0376568 -33.6857687,-69.037866 -33.6856838,-69.037351 -33.6853401,-69.0371311 -33.6842242,-69.0365088 -33.68376,-69.0362889 -33.6832065,-69.036144 -33.6823673,-69.0358865 -33.6818227,-69.0358973 -33.6817468,-69.03557 -33.6815816,-69.0359187 -33.6810459,-69.0351462 -33.6804031,-69.0349263 -33.6798541,-69.0345454 -33.67968,-69.0342611 -33.6794791,-69.0337515 -33.6794836,-69.0332151 -33.6793095,-69.0331185 -33.6788586,-69.0329737 -33.6786354,-69.0327162 -33.6783541,-69.0325767 -33.6782024,-69.0321851 -33.6780238,-69.0315521 -33.6776756,-69.0312892 -33.6774881,-69.0311068 -33.6773453,-69.0308118 -33.6771131,-69.0305275 -33.6769747,-69.0303397 -33.6769033,-69.0300447 -33.676948,-69.0298086 -33.6769256,-69.0288216 -33.6784256,-69.0287519 -33.6783742,-69.0296692 -33.6768676,-69.0292937 -33.6765819,-69.0289557 -33.6765596,-69.0286982 -33.6766355,-69.0284568 -33.6765685,-69.0282691 -33.6768363,-69.0279794 -33.6769122,-69.0277594 -33.6767917,-69.0276629 -33.6770015,-69.0274376 -33.6768988,-69.0273035 -33.6768899,-69.0270782 -33.6770328,-69.0268743 -33.6769301,-69.0265229 -33.6775261,-69.0258417 -33.6786086,-69.0233284 -33.6774703,-69.0232372 -33.6775238,-69.0232211 -33.6781176,-69.0231353 -33.6783408,-69.0233445 -33.6788229,-69.0233767 -33.6791979,-69.0232158 -33.6794925,-69.0233231 -33.6796264,-69.022733 -33.6803049,-69.0227705 -33.6806933,-69.0227598 -33.6811084,-69.0223092 -33.6817245,-69.0238702 -33.6824744,-69.0240419 -33.6822735,-69.0255761 -33.6830234,-69.0254581 -33.6832243,-69.0291756 -33.6849295,-69.0293527 -33.6846617,-69.0311283 -33.6854607,-69.0308976 -33.6858222,-69.0337139 -33.6871256,-69.0343523 -33.6862061,-69.0349907 -33.6865409,-69.0343738 -33.6874604,-69.0344971 -33.6875005),(-69.0305328 -33.683019,-69.0311444 -33.6821173,-69.0274215 -33.6803451,-69.0263647 -33.6819119,-69.0305328 -33.683019))", + "POLYGON((-70.3434661 -38.5210725,-70.3434387 -38.5210955,-70.3434261 -38.5211299,-70.3434276 -38.5211716,-70.3434429 -38.5212067,-70.3434724 -38.5212373,-70.3435173 -38.5212595,-70.3435951 -38.5212788,-70.3436942 -38.5212955,-70.3438252 -38.5213148,-70.3439597 -38.521337,-70.3441048 -38.5213591,-70.3442463 -38.5213841,-70.3443808 -38.5214145,-70.3445082 -38.5214533,-70.3446177 -38.5214956,-70.3447356 -38.5215433,-70.3448535 -38.5215847,-70.3449795 -38.5216101,-70.3451015 -38.5216229,-70.3452113 -38.5216419,-70.3453088 -38.5216674,-70.3454192 -38.521701,-70.3455168 -38.5217296,-70.3456184 -38.5217582,-70.3457445 -38.5217805,-70.3458786 -38.52179,-70.3460209 -38.52179,-70.3471381 -38.5217576,-70.3472051 -38.5217534,-70.3472561 -38.521743,-70.3473044 -38.5217241,-70.3473446 -38.521701,-70.3473795 -38.5216695,-70.3474116 -38.5216422,-70.3474492 -38.5216128,-70.3474921 -38.5215835,-70.3475323 -38.5215541,-70.3476101 -38.5214974,-70.3475739 -38.5216241,-70.3475456 -38.5216933,-70.3475137 -38.5217626,-70.3474677 -38.5218235,-70.3474253 -38.5218674,-70.3473686 -38.5219038,-70.3473085 -38.5219342,-70.3472413 -38.521954,-70.347174 -38.5219675,-70.3472817 -38.5219659,-70.3474005 -38.5219536,-70.3475173 -38.5219287,-70.3476447 -38.5218955,-70.3477721 -38.5218456,-70.3479455 -38.5217626,-70.3479066 -38.5218318,-70.3478641 -38.5218789,-70.3478075 -38.5219287,-70.3477355 -38.5219766,-70.3479018 -38.5219262,-70.3480467 -38.5218675,-70.3481915 -38.5217919,-70.3483042 -38.5217206,-70.3484135 -38.521645,-70.3486227 -38.5215149,-70.3488534 -38.521389,-70.3491002 -38.5212799,-70.3493684 -38.5211918,-70.3496098 -38.5211162,-70.3498834 -38.5210491,-70.3501301 -38.5210029,-70.3503447 -38.5209693,-70.3505593 -38.5209441,-70.3507792 -38.5209204,-70.3507792 -38.5209204,-70.3505605 -38.5208228,-70.350236 -38.5206906,-70.3498659 -38.5205962,-70.3495735 -38.5205458,-70.3492637 -38.5204975,-70.3491282 -38.5204933,-70.3490075 -38.5205059,-70.3488064 -38.5205469,-70.3488064 -38.5205469,-70.3485167 -38.5206098,-70.3478837 -38.5207179,-70.3473875 -38.5207808,-70.3468376 -38.5208333,-70.3461456 -38.5208816,-70.3455448 -38.520934,-70.3449628 -38.5209949,-70.3443834 -38.5210306,-70.343796 -38.5210536,-70.3434661 -38.5210725),(-70.3393449 -38.5210096,-70.3397998 -38.5209924,-70.3400091 -38.5209944,-70.3401816 -38.5210137,-70.3403726 -38.5210619,-70.340539 -38.5211246,-70.3406807 -38.5211969,-70.3408225 -38.5212789,-70.3409704 -38.5213898,-70.3410954 -38.5215145,-70.341237 -38.5216474,-70.3414069 -38.5217803,-70.3415626 -38.5218855,-70.3417466 -38.5219686,-70.3419731 -38.5220406,-70.3421855 -38.522096,-70.3423908 -38.5221458,-70.3426173 -38.5222012,-70.3428155 -38.5222621,-70.3430137 -38.5223341,-70.3432119 -38.5224338,-70.3433534 -38.5225058,-70.343495 -38.5225889,-70.3436649 -38.5226609,-70.3438772 -38.5226885,-70.3440683 -38.522683,-70.3442524 -38.5226609,-70.3449444 -38.5225329,-70.3450571 -38.5224952,-70.3451644 -38.522428,-70.3452341 -38.5223483,-70.3452663 -38.5222769,-70.3452717 -38.5222014,-70.3452448 -38.5221342,-70.3451966 -38.5220797,-70.3451 -38.5220335,-70.3449444 -38.5219873,-70.3447513 -38.5219537,-70.3445624 -38.5219413,-70.343915 -38.5219068,-70.3437005 -38.5218859,-70.3434859 -38.5218565,-70.3432337 -38.5217935,-70.3429923 -38.521718,-70.3427402 -38.5216215,-70.3425364 -38.5215333,-70.3423486 -38.5214368,-70.3421877 -38.5213361,-70.3420589 -38.5212353,-70.341948 -38.5211271,-70.341948 -38.5211271,-70.3415966 -38.5210893,-70.3412157 -38.5210327,-70.3409931 -38.5209875,-70.3408737 -38.5209414,-70.3406967 -38.5208417,-70.3405653 -38.5207452,-70.3404205 -38.5206245,-70.3402944 -38.5205217,-70.3401375 -38.5204073,-70.3399175 -38.520231,-70.3397392 -38.5200925,-70.3396024 -38.5200075,-70.3394187 -38.5199183,-70.3392309 -38.519868,-70.3388715 -38.5197882,-70.3384933 -38.5197012,-70.3380534 -38.5196015,-70.3377664 -38.519549,-70.3372943 -38.5195007,-70.3368531 -38.5194703,-70.3365098 -38.5194577,-70.3361759 -38.5194399,-70.3359184 -38.5194567,-70.3359184 -38.5194567,-70.3362585 -38.519639,-70.3365107 -38.5197779,-70.3367348 -38.5199386,-70.3369777 -38.520114,-70.3372112 -38.5202456,-70.3376501 -38.5204356,-70.3381078 -38.5206329,-70.3384721 -38.5208009,-70.3387149 -38.520874,-70.3390231 -38.5209325,-70.3393449 -38.5210096))", + "MULTILINESTRING((-61.7568258 -31.6740654,-61.7569584 -31.6740483,-61.7573608 -31.6739754,-61.7579382 -31.6738709,-61.758941 -31.6736955,-61.7599953 -31.6735111,-61.7603435 -31.6734503,-61.7605975 -31.6734047,-61.7621451 -31.6731352,-61.7621451 -31.6731352,-61.7621922 -31.6733343))", + "POLYGON((-68.5483882 -31.5039585,-68.5480409 -31.5039305,-68.5475132 -31.503888,-68.5475069 -31.503888,-68.5473788 -31.5038875,-68.5473722 -31.5038875,-68.5472674 -31.5038814,-68.5471701 -31.5038757,-68.5470843 -31.5038707,-68.5470683 -31.5038694,-68.5469779 -31.5038617,-68.5468804 -31.5038535,-68.5467856 -31.5038454,-68.5466804 -31.5038365,-68.5465891 -31.5038288,-68.5464757 -31.5038192,-68.5465458 -31.5035879,-68.5465473 -31.5035825,-68.54675 -31.5028187,-68.5468999 -31.5022372,-68.5469021 -31.5022284,-68.5469628 -31.5019979,-68.5469813 -31.5019276,-68.5469833 -31.5019199,-68.5469942 -31.5018781,-68.5469942 -31.5018781,-68.5470365 -31.5018696,-68.5470365 -31.5018696,-68.547483 -31.5017578,-68.5477639 -31.501702,-68.5480249 -31.5016557,-68.5485587 -31.5016466,-68.5490234 -31.5016055,-68.5490234 -31.5016055,-68.5489833 -31.5018285,-68.5489533 -31.5019571,-68.5489335 -31.5020433,-68.5488177 -31.5025468,-68.5487158 -31.5030088,-68.5486036 -31.5034996,-68.5485833 -31.5035724,-68.5484826 -31.5039664,-68.5484826 -31.5039664,-68.5483946 -31.503959,-68.5483882 -31.5039585))", + "POLYGON((-68.5482663 -31.5443404,-68.5480718 -31.5442838,-68.5480346 -31.544273,-68.5479679 -31.5442535,-68.547865 -31.5442235,-68.5477976 -31.5442039,-68.547794 -31.5442039,-68.547766 -31.5442043,-68.5477546 -31.5442044,-68.547652 -31.5442057,-68.5475929 -31.5442065,-68.5475307 -31.5442073,-68.5474691 -31.5442038,-68.547375 -31.5441984,-68.5472913 -31.5441936,-68.5471354 -31.5441956,-68.5470951 -31.5442013,-68.5468837 -31.5442374,-68.5468293 -31.5442467,-68.5467335 -31.5442581,-68.5467026 -31.5442618,-68.5466636 -31.5442665,-68.5465768 -31.5442768,-68.5464498 -31.5442919,-68.5464463 -31.54439,-68.5464427 -31.54449,-68.546441 -31.544537,-68.5464384 -31.5446868,-68.5464363 -31.5447315,-68.5464364 -31.544792,-68.5464364 -31.544807,-68.5464323 -31.5449592,-68.5464319 -31.5449777,-68.5464276 -31.5451235,-68.5464275 -31.5451625,-68.5464273 -31.5451953,-68.5464272 -31.5452255,-68.5464256 -31.5453554,-68.5464243 -31.5453784,-68.5464234 -31.5454414,-68.5464189 -31.5455607,-68.5465619 -31.5455611,-68.5465918 -31.5455614,-68.5467006 -31.5455617,-68.5468407 -31.5455622,-68.5469298 -31.5455623,-68.5469764 -31.5455626,-68.5470158 -31.5455629,-68.5471183 -31.5455631,-68.5472789 -31.5455637,-68.54736 -31.545564,-68.547401 -31.5455641,-68.547446 -31.5455643,-68.5475602 -31.5455647,-68.547728 -31.5455653,-68.547817 -31.5455656,-68.5478385 -31.5455657,-68.5478385 -31.5455657,-68.5478478 -31.5455678,-68.5478478 -31.5455678,-68.5478797 -31.5454672,-68.5479144 -31.5453579,-68.5479468 -31.5452559,-68.547975 -31.5451672,-68.5480045 -31.5450741,-68.5480327 -31.5449854,-68.5480647 -31.5448846,-68.5480958 -31.5447869,-68.548114 -31.5447294,-68.5481151 -31.5447209,-68.5481097 -31.5447145,-68.5481097 -31.5447145,-68.5481476 -31.5446143,-68.5481839 -31.5445183,-68.5481839 -31.5445183,-68.5481966 -31.5445115,-68.5482761 -31.5443436,-68.5482761 -31.5443436,-68.5482663 -31.5443404))", + "POLYGON((-58.3761403 -34.8049613,-58.3746555 -34.8043325,-58.3745757 -34.8043497,-58.3741564 -34.8050218,-58.3741564 -34.8050218,-58.3741704 -34.8050491,-58.3741619 -34.8050658,-58.3741513 -34.8050827,-58.374112 -34.8050927,-58.374112 -34.8050927,-58.3737147 -34.8057548,-58.3737396 -34.8058114,-58.3751384 -34.806525,-58.3751384 -34.806525,-58.3761403 -34.8049613))", + "MULTILINESTRING((-65.7819384 -28.4602624,-65.78194 -28.4602013,-65.7819774 -28.4602021,-65.7819821 -28.4600267,-65.78195 -28.460026,-65.7819522 -28.4599456,-65.7819904 -28.4599464,-65.7819951 -28.4597744,-65.7819717 -28.4597739,-65.7819734 -28.459716,-65.7823023 -28.4597233,-65.7822987 -28.4598488,-65.7823247 -28.4598494,-65.7823226 -28.4599249,-65.7821596 -28.4599213,-65.7821587 -28.4599523,-65.7822528 -28.4599544,-65.7822505 -28.4600365,-65.7821389 -28.460034,-65.7821351 -28.4601647,-65.7821705 -28.4601654,-65.7821712 -28.4601403,-65.7822232 -28.4601414,-65.7822246 -28.4600897,-65.7822633 -28.4600905,-65.7822667 -28.459963,-65.7823293 -28.4599643,-65.7823258 -28.4600981,-65.7822974 -28.4600975,-65.7822929 -28.4602696,-65.7819384 -28.4602624))", + "MULTILINESTRING((-60.3289264 -31.739336,-60.3292052 -31.7393546,-60.3294736 -31.7393725),(-60.4094365 -31.7562038,-60.4088564 -31.7561717,-60.4088554 -31.7561858,-60.4088547 -31.7561944,-60.4094348 -31.7562266,-60.4094365 -31.7562038),(-60.4490076 -31.7525825,-60.448439 -31.7526463),(-60.4546523 -31.7519354,-60.4540749 -31.7520026,-60.454079 -31.7520282,-60.4546564 -31.751961,-60.4546547 -31.7519499,-60.4546523 -31.7519354),(-60.4832498 -31.7697088,-60.4826784 -31.7696678,-60.4826772 -31.7696799,-60.4826763 -31.7696889,-60.4832477 -31.7697298,-60.4832498 -31.7697088),(-60.4955244 -31.7721043,-60.4950164 -31.7718769,-60.4950144 -31.77188,-60.4950072 -31.7718916,-60.4955153 -31.772119,-60.4955244 -31.7721043),(-60.5045121 -31.7734111,-60.5039414 -31.7733833,-60.503941 -31.7733884,-60.5039398 -31.7734072,-60.5045104 -31.773435,-60.5045121 -31.7734111),(-60.5127137 -31.7683306,-60.5127017 -31.7683237,-60.512696 -31.7683203,-60.5123721 -31.7687237,-60.5123899 -31.768734,-60.5127137 -31.7683306),(-60.5159017 -31.7643863,-60.5158879 -31.7643781,-60.5162173 -31.7639768,-60.5162208 -31.7639789,-60.516231 -31.7639849,-60.5159017 -31.7643863),(-60.5208546 -31.7580253,-60.520839 -31.7580154,-60.5208331 -31.7580117,-60.5205435 -31.758437,-60.5205649 -31.7584507,-60.5208546 -31.7580253),(-60.5331566 -31.7408199,-60.533206 -31.7408068,-60.5332684 -31.7408247,-60.533383 -31.7412668,-60.5339281 -31.7415692,-60.5309841 -31.7456282,-60.5309435 -31.7456843,-60.53044 -31.7463804,-60.5300264 -31.7461612,-60.5300312 -31.7461542,-60.5299422 -31.7461094,-60.529912 -31.7460935,-60.5298697 -31.7460714,-60.5297794 -31.7460242,-60.5297284 -31.7459975,-60.5296675 -31.7459652,-60.5296397 -31.7459394,-60.5295918 -31.7457893,-60.52956 -31.745762,-60.5293851 -31.7456733,-60.5296442 -31.7453201,-60.53048 -31.7442233,-60.53102 -31.7434927,-60.530924 -31.7434466,-60.5310456 -31.7432868,-60.5318423 -31.7426469,-60.5317027 -31.7425381,-60.5317238 -31.742426,-60.5317834 -31.7423184,-60.5322159 -31.7417072,-60.5322246 -31.741642,-60.5323013 -31.741637,-60.5324844 -31.7417341,-60.5329342 -31.7410775,-60.5331096 -31.7408541,-60.5331566 -31.7408199),(-60.3008178 -31.7269233,-60.2997508 -31.7264213,-60.2992341 -31.7262355,-60.2987165 -31.7260996,-60.2983246 -31.7260312,-60.2979304 -31.7259871,-60.2951509 -31.7257779,-60.2948654 -31.7257511),(-60.3008178 -31.7269233,-60.3014362 -31.7272238,-60.3014362 -31.7272238,-60.3024637 -31.7277236,-60.3024637 -31.7277236,-60.302867 -31.7279284,-60.302867 -31.7279284,-60.3074125 -31.7301641,-60.3074125 -31.7301641,-60.3086075 -31.7307464,-60.3086075 -31.7307464,-60.3108042 -31.7317669,-60.3127634 -31.7326402,-60.3172228 -31.7346731,-60.3253759 -31.7383794,-60.3264264 -31.7387431,-60.3274191 -31.7389943,-60.3287067 -31.739216,-60.3292215 -31.7392556,-60.330009 -31.7392862,-60.3308572 -31.7392541,-60.3317051 -31.7391853,-60.3332268 -31.7389584,-60.3375255 -31.7382145,-60.3422108 -31.7374445,-60.3432909 -31.7372748,-60.3440431 -31.7372228,-60.3445425 -31.7372449,-60.3451538 -31.737302,-60.3462993 -31.7374809,-60.3467372 -31.7375611,-60.3467372 -31.7375611,-60.3473804 -31.7376938,-60.3473804 -31.7376938,-60.3535773 -31.7390844,-60.3547467 -31.7393964,-60.3556076 -31.7396912,-60.3562877 -31.7399688,-60.3569581 -31.7403271,-60.3595767 -31.7418275,-60.3618068 -31.7431025,-60.3628968 -31.7436953,-60.3726941 -31.7491239,-60.3731728 -31.7493868,-60.3750034 -31.7503823,-60.3750034 -31.7503823,-60.3751797 -31.7504767,-60.3751797 -31.7504767,-60.3760144 -31.7509289,-60.3767292 -31.7513077,-60.3773037 -31.7516132,-60.3791301 -31.7526027,-60.3800814 -31.7531259,-60.3809362 -31.7535362,-60.3817033 -31.7538232,-60.3823594 -31.7540462,-60.3831037 -31.7542452,-60.383768 -31.7544097,-60.3844073 -31.7544919,-60.3852769 -31.7545847,-60.391759 -31.7550254,-60.3947969 -31.7552621,-60.3983017 -31.7555028,-60.4034161 -31.7557909,-60.4089999 -31.7561462,-60.4162907 -31.7566254,-60.4162907 -31.7566254,-60.4211499 -31.7569596,-60.4211499 -31.7569596,-60.4212419 -31.7569647,-60.4212419 -31.7569647,-60.4217818 -31.7569876,-60.422201 -31.7570016,-60.4225995 -31.7569933,-60.4229164 -31.7569808,-60.4231487 -31.7569695,-60.4234931 -31.7569398,-60.4237977 -31.7569098,-60.4240403 -31.7568805,-60.4244326 -31.7568285,-60.429089 -31.7561341,-60.4320429 -31.755709,-60.4320429 -31.755709,-60.4322796 -31.7556719,-60.4322796 -31.7556719,-60.433141 -31.7555326,-60.4335918 -31.7554554,-60.4338563 -31.7554001,-60.4342514 -31.7553105,-60.4398157 -31.7538303,-60.4405065 -31.7536485,-60.4408899 -31.7535552,-60.4413162 -31.7534647,-60.4418103 -31.7533804,-60.4425572 -31.7532894,-60.4435896 -31.753165,-60.4487216 -31.7525757,-60.4543628 -31.7519507,-60.4550763 -31.7518676,-60.455633 -31.7517995,-60.4563331 -31.7517481,-60.4565781 -31.751726,-60.4568612 -31.7517154,-60.4570931 -31.7517238,-60.4573522 -31.7517406,-60.4576725 -31.7517808,-60.45802 -31.7518464,-60.4583806 -31.751929,-60.4586472 -31.7520081,-60.4590053 -31.7521396,-60.4593369 -31.7522893,-60.4596348 -31.7524335,-60.4599121 -31.7525919,-60.4601533 -31.7527553,-60.4603896 -31.7529481,-60.4606191 -31.7531376,-60.4608146 -31.7533219,-60.4610099 -31.7535311,-60.4613042 -31.7539019,-60.4614993 -31.7542016,-60.4616335 -31.7544454,-60.4617322 -31.7546647,-60.4618385 -31.7549542,-60.4623042 -31.7563725,-60.4625074 -31.7569435,-60.4626164 -31.7572098,-60.4627576 -31.7574969,-60.4629091 -31.7577689,-60.4630744 -31.7580066,-60.4632542 -31.7582587,-60.4633955 -31.758441,-60.4636021 -31.7586751,-60.4637868 -31.7588719,-60.4642818 -31.7593688,-60.4642818 -31.7593688,-60.4657159 -31.7607494,-60.4657159 -31.7607494,-60.4659275 -31.7609647,-60.4659275 -31.7609647,-60.4676762 -31.7626358,-60.4676762 -31.7626358,-60.4704237 -31.765323,-60.4714476 -31.7662934,-60.4727492 -31.7675396,-60.4729745 -31.7677357,-60.4731879 -31.7679059,-60.4734519 -31.7680892,-60.4737067 -31.768242,-60.4741758 -31.7685163,-60.4744872 -31.768657,-60.4748118 -31.7687893,-60.475228 -31.7689283,-60.4755843 -31.7690173,-60.4758954 -31.7690903,-60.476293 -31.7691631,-60.476293 -31.7691631,-60.4766998 -31.7692137,-60.4822966 -31.7696248,-60.4822966 -31.7696248,-60.4829569 -31.7696704,-60.4882929 -31.7700365,-60.4900467 -31.7701689,-60.4904031 -31.7702105,-60.4907066 -31.7702578,-60.4909641 -31.770308,-60.4912993 -31.7703894,-60.4915193 -31.7704493,-60.4915778 -31.7704649,-60.4915778 -31.7704649,-60.4916427 -31.7704822,-60.4916427 -31.7704822,-60.4916909 -31.770495,-60.491994 -31.7706067,-60.4926391 -31.7708823,-60.4943447 -31.7716185,-60.4945109 -31.7716747,-60.4945109 -31.7716747,-60.4945838 -31.771718,-60.4952189 -31.772001,-60.4960156 -31.7723578,-60.4967335 -31.7726567,-60.49705 -31.7727752,-60.4973223 -31.7728665,-60.49775 -31.7729713,-60.4981407 -31.7730473,-60.4986835 -31.7731218,-60.4992668 -31.7731626,-60.5007513 -31.7732472,-60.5007513 -31.7732472,-60.5037251 -31.7734225,-60.5042092 -31.7734492,-60.5048177 -31.7734662,-60.5051449 -31.7734753,-60.5054104 -31.7734753,-60.5058064 -31.7734505,-60.5063707 -31.7733886,-60.5068671 -31.7733006,-60.507315 -31.7731792,-60.5077413 -31.7730512,-60.5080395 -31.7729346,-60.5084842 -31.7727228,-60.5088276 -31.772529,-60.5092397 -31.7722801,-60.5095732 -31.7720319,-60.5099031 -31.7717468,-60.5102817 -31.7713813,-60.5119563 -31.7692915,-60.5125696 -31.768537,-60.512967 -31.7680465,-60.5160883 -31.7641966,-60.5165764 -31.7635889,-60.5176685 -31.7622364,-60.5190501 -31.7605006,-60.5196965 -31.7596572,-60.5204974 -31.7585817,-60.5204974 -31.7585817,-60.5207303 -31.758246,-60.5211648 -31.7576076,-60.5212465 -31.7575002,-60.5219138 -31.7564919,-60.52226 -31.7559724,-60.5233517 -31.7543968,-60.5239439 -31.7535624,-60.5253634 -31.7515949,-60.5264399 -31.7500995,-60.5290275 -31.7465343,-60.5294917 -31.7458866,-60.5295443 -31.7458132,-60.531725 -31.742812,-60.531908 -31.7426207,-60.5322666 -31.7421254))", + "POLYGON((-71.4829087 -43.9783326,-71.4819861 -43.9796374,-71.4814603 -43.9799539,-71.4810741 -43.980502,-71.4814174 -43.9812123,-71.4815301 -43.9817681,-71.4813584 -43.9818299,-71.4803714 -43.982019,-71.4802587 -43.9816369,-71.4800012 -43.9811505,-71.4791107 -43.9803553,-71.4776623 -43.9799616,-71.4771795 -43.9799693,-71.4762461 -43.9800311,-71.4753986 -43.9801546,-71.4748407 -43.9801315,-71.4741862 -43.9798612,-71.4732099 -43.9794521,-71.4716971 -43.978873,-71.471386 -43.9785565,-71.4705277 -43.9781319,-71.469723 -43.9776764,-71.4689183 -43.9773289,-71.4679742 -43.9776146,-71.4668691 -43.9764797,-71.4666438 -43.9759855,-71.4662254 -43.9753987,-71.4657104 -43.974673,-71.4652062 -43.9740862,-71.4649058 -43.9737464,-71.4644551 -43.9731751,-71.4640689 -43.9725265,-71.4633608 -43.9720632,-71.4626742 -43.9722949,-71.4618802 -43.9720169,-71.4617944 -43.9711521,-71.460228 -43.970573,-71.4599168 -43.9704572,-71.4593482 -43.9699939,-71.4585865 -43.9695074,-71.4583504 -43.9689051,-71.4578676 -43.968326,-71.4572454 -43.9673222,-71.4570415 -43.967021,-71.456548 -43.9666195,-71.4561939 -43.9660017,-71.4560866 -43.9652063,-71.4564836 -43.9648356,-71.4567411 -43.9651291,-71.4568377 -43.9656619,-71.4573097 -43.9662102,-71.4586294 -43.9665114,-71.4592838 -43.9663492,-71.4589512 -43.9658086,-71.4584684 -43.9656233,-71.4579749 -43.9654303,-71.4572346 -43.9650441,-71.4570522 -43.9643569,-71.4571381 -43.9640634,-71.4565158 -43.9637622,-71.456945 -43.9635537,-71.456945 -43.9632371,-71.4563012 -43.9628973,-71.455518 -43.9623876,-71.4550889 -43.9615613,-71.4548636 -43.9611134,-71.4546919 -43.9608586,-71.4539302 -43.9605265,-71.4533508 -43.9601944,-71.4531362 -43.9601481,-71.4527607 -43.9603102,-71.4518595 -43.9605574,-71.4510656 -43.9608045,-71.4509583 -43.9610748,-71.4507437 -43.9615227,-71.4509368 -43.9620942,-71.4510548 -43.9624958,-71.4512909 -43.9628896,-71.4515054 -43.96377,-71.4513123 -43.9643723,-71.451087 -43.9646117,-71.4506471 -43.9647816,-71.4500034 -43.9649515,-71.4493919 -43.9652758,-71.4489198 -43.9654611,-71.4490485 -43.9657855,-71.4495421 -43.9660326,-71.4500141 -43.9663646,-71.4505291 -43.9666349,-71.4512479 -43.9670519,-71.451838 -43.9673067,-71.452471 -43.967631,-71.4525461 -43.9680712,-71.4525837 -43.968326,-71.4525783 -43.9686001,-71.4525622 -43.9688781,-71.4524978 -43.9692333,-71.4520794 -43.9691252,-71.4522135 -43.9684032,-71.452117 -43.9680866,-71.451956 -43.967855,-71.4515591 -43.9674689,-71.4506686 -43.9672372,-71.4496601 -43.9668202,-71.4484584 -43.9663801,-71.4477074 -43.9658704,-71.4475143 -43.9654843,-71.4472461 -43.964882,-71.4475787 -43.9644109,-71.4481258 -43.9641947,-71.4489627 -43.9642796,-71.4495957 -43.9643569,-71.4501536 -43.9639785,-71.4500356 -43.9632835,-71.4496011 -43.9621791,-71.4494348 -43.9616308,-71.4493382 -43.961291,-71.4490914 -43.9605728,-71.4490807 -43.9602562,-71.4492738 -43.9597619,-71.4495099 -43.9592445,-71.4491236 -43.9586421,-71.4491665 -43.9578003,-71.4497566 -43.9567576,-71.4498639 -43.9563792,-71.4496815 -43.9558926,-71.4490593 -43.9554369,-71.4482653 -43.9550585,-71.4475894 -43.954989,-71.447171 -43.9550353,-71.4464522 -43.9548577,-71.4451754 -43.9544483,-71.4448321 -43.9543325,-71.4442957 -43.9541548,-71.443888 -43.953869,-71.4434803 -43.9535215,-71.44274 -43.953143,-71.442461 -43.9527027,-71.4418817 -43.9527105,-71.4419461 -43.953143,-71.4411736 -43.9530812,-71.4404869 -43.9522625,-71.4397145 -43.9525251,-71.4392424 -43.9523552,-71.4396715 -43.9520153,-71.4396501 -43.9515828,-71.4391566 -43.9512584,-71.4387274 -43.9509031,-71.4378262 -43.950625,-71.4382339 -43.9498526,-71.4377189 -43.9494818,-71.4371181 -43.9487866,-71.4361525 -43.9477052,-71.4356804 -43.9470409,-71.4350796 -43.9461294,-71.4345432 -43.9456968,-71.434114 -43.9460213,-71.4337921 -43.9466238,-71.4339209 -43.9474271,-71.4343071 -43.9481687,-71.4345217 -43.9494046,-71.4331269 -43.9484931,-71.4321828 -43.9479833,-71.4316678 -43.9480451,-71.4310241 -43.9478443,-71.4302516 -43.9473499,-71.429286 -43.9467474,-71.4283633 -43.9461757,-71.4272905 -43.9456968,-71.426282 -43.9449707,-71.4255309 -43.9441827,-71.4248014 -43.9428077,-71.4238787 -43.9418189,-71.4233422 -43.9414017,-71.4226771 -43.9407064,-71.4219475 -43.9401502,-71.4208746 -43.9393776,-71.4198876 -43.9383424,-71.4201236 -43.9374153,-71.42066 -43.9361174,-71.4203274 -43.9355688,-71.4207459 -43.935337,-71.4214325 -43.9349662,-71.4215291 -43.9345258,-71.4213681 -43.9341781,-71.4210892 -43.9341704,-71.4205098 -43.9346958,-71.4199948 -43.9348812,-71.4198661 -43.9342863,-71.4194048 -43.9337532,-71.4197588 -43.9336991,-71.4202631 -43.9335368,-71.4207029 -43.9332664,-71.4208531 -43.9333823,-71.421057 -43.9337686,-71.4216256 -43.9338304,-71.4222801 -43.9333205,-71.4225912 -43.9331042,-71.4224088 -43.9327178,-71.4222479 -43.9325479,-71.4218187 -43.9323315,-71.4215183 -43.9319452,-71.4211106 -43.9316207,-71.4202952 -43.9312884,-71.4192224 -43.9308867,-71.41801 -43.9310257,-71.417259 -43.9304694,-71.4165187 -43.9305003,-71.4155531 -43.9300522,-71.4158321 -43.929519,-71.4157784 -43.9292486,-71.4152205 -43.9288777,-71.4154351 -43.9287,-71.4152956 -43.9284527,-71.4149845 -43.9279891,-71.4144802 -43.9268764,-71.4143086 -43.9255009,-71.4142871 -43.9245273,-71.4138901 -43.923542,-71.4135683 -43.92343,-71.4131713 -43.9231981,-71.4126778 -43.9230127,-71.4117873 -43.9231595,-71.4115405 -43.9233604,-71.4117766 -43.9237159,-71.4126563 -43.9238859,-71.4132249 -43.9241486,-71.4130747 -43.9248286,-71.4124954 -43.925045,-71.4118409 -43.9246818,-71.4115942 -43.9242877,-71.4108539 -43.9241023,-71.4097595 -43.9238859,-71.408279 -43.9234841,-71.4075172 -43.9232986,-71.4066911 -43.9228658,-71.406101 -43.922549,-71.4054465 -43.9220853,-71.4049637 -43.921753,-71.4050388 -43.9212198,-71.4046955 -43.9209571,-71.4037514 -43.9200452,-71.4033759 -43.9195428,-71.4029467 -43.9187623,-71.4028072 -43.9180165,-71.4023674 -43.9175876,-71.4018416 -43.9170505,-71.4016646 -43.9167838,-71.4010263 -43.9161308,-71.4008117 -43.9157482,-71.4006454 -43.9155936,-71.4004201 -43.9153965,-71.4002484 -43.9150062,-71.4002484 -43.9145464,-71.4000928 -43.9141909,-71.400125 -43.9138856,-71.3999614 -43.9134392,-71.3995001 -43.9130296,-71.3994035 -43.9128422,-71.3991112 -43.9128055,-71.39902 -43.9128402,-71.3983011 -43.9126219,-71.3977647 -43.9123127,-71.397711 -43.9121234,-71.3974804 -43.9118683,-71.3962734 -43.9111727,-71.3957906 -43.9106239,-71.395458 -43.9102761,-71.3952595 -43.9099862,-71.3945836 -43.9086529,-71.394074 -43.9079959,-71.393913 -43.9075939,-71.3940042 -43.9071186,-71.394133 -43.9067514,-71.3940632 -43.9063649,-71.3945782 -43.9061562,-71.3948572 -43.9059668,-71.3942081 -43.9049001,-71.3946211 -43.9045561,-71.3946801 -43.904301,-71.3944656 -43.9039222,-71.3945782 -43.9037135,-71.3948625 -43.9033618,-71.3950932 -43.9032574,-71.3953668 -43.903211,-71.3957638 -43.9032381,-71.3960427 -43.9034314,-71.3961875 -43.9035241,-71.3964343 -43.9037135,-71.3966328 -43.9039184,-71.3969386 -43.9041425,-71.3972175 -43.9046025,-71.3967025 -43.9048189,-71.3960159 -43.9053446,-71.3958228 -43.9057465,-71.3959837 -43.9061639,-71.3959193 -43.9068596,-71.3967133 -43.9071379,-71.3972122 -43.9076055,-71.3972068 -43.9077524,-71.3969654 -43.9080268,-71.3970566 -43.908363,-71.3970351 -43.9087147,-71.3973516 -43.9088886,-71.3978237 -43.9090471,-71.3978881 -43.9090819,-71.3980168 -43.9093988,-71.3981992 -43.9096539,-71.3984674 -43.9100287,-71.3988537 -43.9103418,-71.3989717 -43.910477,-71.3989502 -43.9107437,-71.3992989 -43.9108751,-71.3995403 -43.9111031,-71.3994277 -43.9114393,-71.3997656 -43.9116094,-71.4002752 -43.9116828,-71.4007312 -43.9117562,-71.4008975 -43.9121465,-71.4014554 -43.9124712,-71.4033437 -43.9137387,-71.4037514 -43.9141561,-71.4034724 -43.9151917,-71.4040947 -43.9157173,-71.404438 -43.916892,-71.4062405 -43.917804,-71.4068198 -43.9182368,-71.4075816 -43.919056,-71.4100921 -43.9205938,-71.4103067 -43.9212739,-71.4130426 -43.9215367,-71.4145446 -43.9226417,-71.4152312 -43.9237545,-71.4162183 -43.9255627,-71.4168406 -43.9260109,-71.4181924 -43.9266446,-71.4212609 -43.9267373,-71.42434 -43.9308635,-71.4249623 -43.9318525,-71.4254344 -43.9322543,-71.4261854 -43.9327024,-71.4270866 -43.9331119,-71.4280951 -43.9333282,-71.4289534 -43.9331814,-71.4304018 -43.9330926,-71.430772 -43.9327488,-71.4309812 -43.9321963,-71.431641 -43.9316748,-71.4322257 -43.9315164,-71.4319038 -43.9317868,-71.4320111 -43.932517,-71.4324403 -43.9324551,-71.4337063 -43.9322079,-71.4349723 -43.9316516,-71.435616 -43.9323779,-71.4364743 -43.9327642,-71.4367748 -43.9332432,-71.4384699 -43.9335368,-71.4390064 -43.9339695,-71.4392209 -43.9356384,-71.4402724 -43.9361328,-71.4417529 -43.9368745,-71.44274 -43.937678,-71.4436197 -43.9386669,-71.4441133 -43.9397176,-71.4446497 -43.9412936,-71.4447999 -43.9422051,-71.4452934 -43.9431785,-71.4459801 -43.9438583,-71.4465809 -43.9448316,-71.4472032 -43.9460985,-71.4473963 -43.946871,-71.447804 -43.9475662,-71.4479327 -43.9484931,-71.4483619 -43.9505864,-71.4485335 -43.9511193,-71.4488125 -43.9514978,-71.449939 -43.9514978,-71.4503145 -43.952023,-71.4501751 -43.9522857,-71.4503467 -43.9525869,-71.4506686 -43.952919,-71.4514947 -43.9531198,-71.4518273 -43.9535369,-71.451956 -43.9540235,-71.4522457 -43.9544638,-71.452471 -43.9547186,-71.4530718 -43.9550971,-71.4536619 -43.9555451,-71.4537478 -43.9560316,-71.4531362 -43.9565259,-71.4532328 -43.9567885,-71.4543647 -43.9575068,-71.4548045 -43.9577423,-71.4555556 -43.9582405,-71.4555636 -43.9583949,-71.4559364 -43.9587695,-71.4563441 -43.9589819,-71.4564085 -43.9591827,-71.4575672 -43.9595225,-71.4595628 -43.9599859,-71.460743 -43.9604183,-71.461258 -43.9616076,-71.46173 -43.9628742,-71.4618695 -43.963268,-71.4619339 -43.9636541,-71.4624918 -43.9641484,-71.4635003 -43.9646349,-71.4645195 -43.9649746,-71.4652491 -43.9653453,-71.4655495 -43.965994,-71.4656997 -43.9663569,-71.4658392 -43.9669824,-71.4661932 -43.967631,-71.4669335 -43.9681484,-71.4674914 -43.9690132,-71.4676523 -43.9693993,-71.4681888 -43.970102,-71.4692938 -43.9707815,-71.4716435 -43.971044,-71.4724052 -43.9712293,-71.4736819 -43.9715613,-71.474551 -43.9719551,-71.475184 -43.9723644,-71.4758384 -43.973067,-71.4766967 -43.9734839,-71.4778018 -43.9738545,-71.4788318 -43.974256,-71.479497 -43.9746884,-71.479733 -43.9754451,-71.4809776 -43.9760164,-71.4819861 -43.9764333,-71.4824581 -43.9772826,-71.4829087 -43.9783326),(-71.4346504 -43.937763,-71.4343929 -43.9372222,-71.4342105 -43.936635,-71.4340925 -43.9361483,-71.4340818 -43.9354761,-71.4339101 -43.9353216,-71.4337063 -43.9349585,-71.4336527 -43.934634,-71.4333522 -43.9339541,-71.4322043 -43.9335677,-71.4314854 -43.9333514,-71.4307881 -43.93356,-71.4304233 -43.9336527,-71.429919 -43.9340777,-71.4294362 -43.934464,-71.4296186 -43.9350512,-71.4298439 -43.935337,-71.4300478 -43.9355766,-71.4309597 -43.9362642,-71.4316785 -43.9367741,-71.4321292 -43.937199,-71.4328051 -43.9370986,-71.4332664 -43.937593,-71.4337278 -43.9379948,-71.4341891 -43.9379793,-71.4346504 -43.937763),(-71.4335775 -43.9412627,-71.4333952 -43.9408378,-71.433481 -43.9405442,-71.4325369 -43.9404283,-71.4321506 -43.9406369,-71.4320219 -43.9410232,-71.4323115 -43.9413245,-71.4327192 -43.9414403,-71.4331913 -43.9414094,-71.4335775 -43.9412627),(-71.4315713 -43.9442909,-71.4311421 -43.9438429,-71.4305842 -43.9433948,-71.4298868 -43.9432944,-71.4288783 -43.9436111,-71.4281273 -43.9438042,-71.4282668 -43.9442523,-71.4283311 -43.9444377,-71.4287603 -43.9449243,-71.4295864 -43.9451329,-71.4301336 -43.9452024,-71.4309383 -43.9451329,-71.4314103 -43.9447003,-71.4315713 -43.9442909),(-71.4306057 -43.9405365,-71.4304555 -43.9401502,-71.4302409 -43.9400111,-71.4293397 -43.9401966,-71.4285672 -43.9401579,-71.4280522 -43.9403433,-71.4283097 -43.9406137,-71.4286959 -43.9409845,-71.4292431 -43.9410618,-71.4293289 -43.9408532,-71.4296508 -43.9407064,-71.4299512 -43.9407682,-71.4306057 -43.9405365),(-71.4283955 -43.9387751,-71.4282775 -43.938551,-71.4278376 -43.9381956,-71.4272475 -43.938157,-71.4267433 -43.9382497,-71.4256168 -43.9379639,-71.4248765 -43.9376626,-71.4243186 -43.9376007,-71.4243722 -43.9380025,-71.4251125 -43.9385278,-71.4256704 -43.9388909,-71.4261854 -43.9392772,-71.4267004 -43.9395708,-71.427151 -43.9402429,-71.4277518 -43.9400498,-71.4275479 -43.9397253,-71.4269149 -43.9393313,-71.4275694 -43.9392386,-71.4281917 -43.9390686,-71.4283955 -43.9387751),(-71.4245868 -43.9351053,-71.4244044 -43.9347421,-71.4241362 -43.9344485,-71.4235568 -43.9342863,-71.4230204 -43.9340854,-71.4224517 -43.9339,-71.4219904 -43.9342168,-71.4219368 -43.9344099,-71.4223337 -43.9344949,-71.4228273 -43.9345413,-71.422441 -43.9348735,-71.4225054 -43.9351671,-71.4220548 -43.9352443,-71.4220762 -43.9359242,-71.4220119 -43.936326,-71.4222264 -43.9366041,-71.4228916 -43.9370522,-71.4233422 -43.9371604,-71.4239967 -43.9372994,-71.4242971 -43.9370986,-71.4241898 -43.9367586,-71.42434 -43.9363337,-71.4241362 -43.9360092,-71.4241898 -43.9357774,-71.4242327 -43.9354143,-71.4245868 -43.9351053),(-71.4242005 -43.9387905,-71.4240503 -43.9384506,-71.4234066 -43.9383656,-71.4229453 -43.9383656,-71.4224517 -43.9385665,-71.4223981 -43.9388523,-71.4228487 -43.939115,-71.4232028 -43.9391459,-71.4231384 -43.9387828,-71.4236641 -43.9387442,-71.4242005 -43.9387905))" }; std::string getWKT(const polygon_t &polygon) { std::stringstream ss; ss << std::setprecision(12) << boost::geometry::wkt(polygon); + // std::cout << ss.str() << std::endl; return ss.str(); } std::string getWKTFromDB(const std::string &table, const long id, std::shared_ptr &db) { - auto result = db->query("SELECT ST_AsText(geom, 4326), refs from relations where osm_id=" + std::to_string(id)); + auto result = db->query("SELECT ST_AsText(geom, 4326), refs from " + table + " where osm_id=" + std::to_string(id)); for (auto r_it = result.begin(); r_it != result.end(); ++r_it) { + // std::cout << (*r_it)[0].as() << std::endl; return (*r_it)[0].as(); } } @@ -153,93 +174,143 @@ main(int argc, char *argv[]) auto queryraw = std::make_shared(db); std::map> waycache; - // processFile("raw-case-1.osc", db); - // processFile("raw-case-2.osc", db); - - // std::string waysIds = "101874,101875"; - // queryraw->getWaysByIds(waysIds, waycache); - - // // 4 created Nodes, 1 created Way (same changeset) - // if ( getWKT(waycache.at(101874)->polygon).compare(expectedGeometries[0]) == 0) { - // runtest.pass("4 created Nodes, 1 created Ways (same changeset)"); - // } else { - // runtest.fail("4 created Nodes, 1 created Ways (same changeset)"); - // return 1; - // } - - // // 1 created Way, 4 existing Nodes (different changeset) - // if ( getWKT(waycache.at(101875)->polygon).compare(expectedGeometries[1]) == 0) { - // runtest.pass("1 created Way, 4 existing Nodes (different changesets)"); - // } else { - // runtest.fail("1 created Way, 4 existing Nodes (different changesets)"); - // return 1; - // } - - // // 1 modified node, indirectly modify other existing ways - // processFile("raw-case-3.osc", db); - // waycache.erase(101875); - // queryraw->getWaysByIds(waysIds, waycache); - // if ( getWKT(waycache.at(101875)->polygon).compare(expectedGeometries[2]) == 0) { - // runtest.pass("1 modified Node, indirectly modify other existing Ways (different changesets)"); - // } else { - // runtest.fail("1 modified Node, indirectly modify other existing Ways (different changesets)"); - // return 1; - // } - - // // 1 created Relation referencing 1 created Way and 1 existing Way - // processFile("raw-case-4.osc", db); - // if ( getWKTFromDB("relations", 211766, db).compare(expectedGeometries[3]) == 0) { - // runtest.pass("1 created Relation referencing 1 created Way and 1 existing Way (different changesets)"); - // } else { - // runtest.fail("1 created Relation referencing 1 created Way and 1 existing Way (different changesets)"); - // return 1; - // } - - // // 1 modified Node, indirectly modify other existing Ways and 1 Relation - // processFile("raw-case-5.osc", db); - // if ( getWKTFromDB("relations", 211766, db).compare(expectedGeometries[4]) == 0) { - // runtest.pass("1 modified Node, indirectly modify other existing Ways and 1 Relation (different changesets)"); - // } else { - // runtest.fail("1 modified Node, indirectly modify other existing Ways and 1 Relation (different changesets)"); - // return 1; - // } - - // // 4 created Nodes, 2 created Ways, 1 created Relation with type=multilinestring - // processFile("raw-case-6.osc", db); - // if ( getWKTFromDB("relations", 211776, db).compare(expectedGeometries[5]) == 0) { - // runtest.pass("4 created Nodes, 2 created Ways, 1 created Relation with type=multilinestring (same changeset)"); - // } else { - // runtest.fail("4 created Nodes, 2 created Ways, 1 created Relation with type=multilinestring (same changeset)"); - // return 1; - // } - - // // Complex, 1 polygon relation made of multiple ways - // processFile("raw-case-7.osc", db); - // if ( getWKTFromDB("relations", 17331328, db).compare(expectedGeometries[6]) == 0) { - // runtest.pass("Complex, 1 polygon relation made of multiple ways (same changeset)"); - // } else { - // runtest.fail("Complex, 1 polygon relation made of multiple ways (same changeset)"); - // return 1; - // } - - // - // processFile("raw-case-8.osc", db); - // if ( getWKTFromDB("relations", 16191459, db).compare(expectedGeometries[7]) == 0) { - // runtest.pass("Complex, 1 polygon relation made of multiple ways (same changeset)"); - // } else { - // runtest.fail("Complex, 1 polygon relation made of multiple ways (same changeset)"); - // return 1; - // } + processFile("raw-case-1.osc", db); + processFile("raw-case-2.osc", db); + + std::string waysIds = "101874,101875"; + queryraw->getWaysByIds(waysIds, waycache); + + // 4 created Nodes, 1 created Way (same changeset) + if ( getWKT(waycache.at(101874)->polygon).compare(expectedGeometries[0]) == 0) { + runtest.pass("4 created Nodes, 1 created Ways (same changeset)"); + } else { + runtest.fail("4 created Nodes, 1 created Ways (same changeset)"); + return 1; + } + + // 1 created Way, 4 existing Nodes (different changeset) + if ( getWKT(waycache.at(101875)->polygon).compare(expectedGeometries[1]) == 0) { + runtest.pass("1 created Way, 4 existing Nodes (different changesets)"); + } else { + runtest.fail("1 created Way, 4 existing Nodes (different changesets)"); + return 1; + } + + // 1 modified node, indirectly modify other existing ways + processFile("raw-case-3.osc", db); + waycache.erase(101875); + queryraw->getWaysByIds(waysIds, waycache); + if ( getWKT(waycache.at(101875)->polygon).compare(expectedGeometries[2]) == 0) { + runtest.pass("1 modified Node, indirectly modify other existing Ways (different changesets)"); + } else { + runtest.fail("1 modified Node, indirectly modify other existing Ways (different changesets) - - raw-case-3.osc"); + return 1; + } + + // 1 created Relation referencing 1 created Way and 1 existing Way + processFile("raw-case-4.osc", db); + if ( getWKTFromDB("relations", 211766, db).compare(expectedGeometries[3]) == 0) { + runtest.pass("1 created Relation referencing 1 created Way and 1 existing Way (different changesets)"); + } else { + runtest.fail("1 created Relation referencing 1 created Way and 1 existing Way (different changesets) - - raw-case-4.osc"); + return 1; + } + + // 1 modified Node, indirectly modify other existing Ways and 1 Relation + processFile("raw-case-5.osc", db); + if ( getWKTFromDB("relations", 211766, db).compare(expectedGeometries[4]) == 0) { + runtest.pass("1 modified Node, indirectly modify other existing Ways and 1 Relation (different changesets)"); + } else { + runtest.fail("1 modified Node, indirectly modify other existing Ways and 1 Relation (different changesets) - raw-case-5.osc"); + return 1; + } + + // 4 created Nodes, 2 created Ways, 1 created Relation with type=multilinestring + processFile("raw-case-6.osc", db); + if ( getWKTFromDB("relations", 211776, db).compare(expectedGeometries[5]) == 0) { + runtest.pass("4 created Nodes, 2 created Ways, 1 created Relation with type=multilinestring (same changeset)"); + } else { + runtest.fail("4 created Nodes, 2 created Ways, 1 created Relation with type=multilinestring (same changeset) - raw-case-6.osc"); + return 1; + } + + // Complex, 1 polygon relation made of multiple ways + processFile("raw-case-7.osc", db); + if ( getWKTFromDB("relations", 17331328, db).compare(expectedGeometries[6]) == 0) { + runtest.pass("Complex, 1 polygon relation made of multiple ways (same changeset) - raw-case-7.osc"); + } else { + runtest.fail("Complex, 1 polygon relation made of multiple ways (same changeset) - raw-case-7.osc"); + return 1; + } + + + processFile("raw-case-8.osc", db); + if ( getWKTFromDB("relations", 16191459, db).compare(expectedGeometries[7]) == 0) { + runtest.pass("Complex, 1 polygon relation made of multiple ways (same changeset) - raw-case-8.osc"); + } else { + runtest.fail("Complex, 1 polygon relation made of multiple ways (same changeset) - raw-case-8.osc"); + return 1; + } processFile("raw-case-9.osc", db); if ( getWKTFromDB("relations", 16193116, db).compare(expectedGeometries[8]) == 0) { - runtest.pass("Complex, 2 polygon relation made of multiple ways (same changeset)"); + runtest.pass("Complex, 2 polygon relation made of multiple ways (same changeset) - raw-case-9.osc"); } else { - runtest.fail("Complex, 2 polygon relation made of multiple ways (same changeset)"); + runtest.fail("Complex, 2 polygon relation made of multiple ways (same changeset) - raw-case-9.osc"); return 1; } + processFile("raw-case-10.osc", db); + if ( getWKTFromDB("relations", 16770204, db).compare(expectedGeometries[9]) == 0) { + runtest.pass("MultiLinestring made of multiple ways (same changeset) - raw-case-10.osc"); + } else { + runtest.fail("MultiLinestring made of multiple ways (same changeset) - raw-case-10.osc"); + return 1; + } + + processFile("raw-case-11.osc", db); + if ( getWKTFromDB("relations", 16768791, db).compare(expectedGeometries[10]) == 0) { + runtest.pass("MultiLinestring made of multiple ways (same changeset) - raw-case-11.osc"); + } else { + runtest.fail("MultiLinestring made of multiple ways (same changeset) - raw-case-11.osc"); + return 1; + } + + processFile("raw-case-12.osc", db); + if ( getWKTFromDB("relations", 16766330, db).compare(expectedGeometries[11]) == 0) { + runtest.pass("MultiPolygon made of multiple ways (same changeset) - raw-case-12.osc"); + } else { + runtest.fail("MultiPolygon made of multiple ways (same changeset) - raw-case-12.osc"); + return 1; + } + + + processFile("raw-case-13.osc", db); + if ( getWKTFromDB("relations", 17060812, db).compare(expectedGeometries[12]) == 0) { + runtest.pass("MultiPolygon made of multiple ways (same changeset) - raw-case-13.osc"); + } else { + runtest.fail("MultiPolygon made of multiple ways (same changeset) - raw-case-13.osc"); + return 1; + } + + processFile("raw-case-14.osc", db); + if ( getWKTFromDB("relations", 357043, db).compare(expectedGeometries[13]) == 0) { + runtest.pass("MultiLinestring made of multiple ways (same changeset) - raw-case-14.osc"); + } else { + runtest.fail("MultiLinestring made of multiple ways (same changeset) - raw-case-14.osc"); + return 1; + } + + processFile("raw-case-15.osc", db); + if ( getWKTFromDB("relations", 13341377, db).compare(expectedGeometries[14]) == 0) { + runtest.pass("MultiLinestring made of multiple ways (same changeset) - raw-case-15.osc"); + } else { + runtest.fail("MultiLinestring made of multiple ways (same changeset) - raw-case-15.osc"); + return 1; + } + } else { + std::cout << "ERROR: can't connect to the test DB (" << dbconn << " dbname=underpass_test" << ")" << std::endl; } diff --git a/src/testsuite/libunderpass.all/replication-test.cc b/src/testsuite/libunderpass.all/replication-test.cc index 5fc73c02e..b228b28ad 100644 --- a/src/testsuite/libunderpass.all/replication-test.cc +++ b/src/testsuite/libunderpass.all/replication-test.cc @@ -1,5 +1,5 @@ // -// Copyright (c) 2020, 2021, 2022, 2023 Humanitarian OpenStreetMap Team +// Copyright (c) 2020, 2021, 2022, 2023, 2024 Humanitarian OpenStreetMap Team // // This file is part of Underpass. // diff --git a/src/testsuite/libunderpass.all/stats-test.cc b/src/testsuite/libunderpass.all/stats-test.cc index 79cda99ae..076c76d6f 100644 --- a/src/testsuite/libunderpass.all/stats-test.cc +++ b/src/testsuite/libunderpass.all/stats-test.cc @@ -1,5 +1,5 @@ // -// Copyright (c) 2020, 2021, 2022, 2023 Humanitarian OpenStreetMap Team +// Copyright (c) 2020, 2021, 2022, 2023, 2024 Humanitarian OpenStreetMap Team // // This file is part of Underpass. // @@ -161,8 +161,8 @@ class TestStats { } return config; } - void - testStat(std::shared_ptr changestats, std::map validation, std::string tag) { + void + testStat(std::shared_ptr changestats, std::map &validation, const std::string &tag, const std::string &filename) { logger::LogFile &dbglogfile = logger::LogFile::getDefaultInstance(); dbglogfile.setWriteDisk(true); @@ -188,7 +188,7 @@ class TestStats { runtest.pass("Calculating added " + tag); } else{ TestState runtest; - runtest.fail("Calculating added " + tag); + runtest.fail("Calculating added " + tag + " file " + filename); } } else if (this->verbose) { std::cout << "[Underpass] added_" + tag + ": 0" << std::endl; @@ -211,7 +211,7 @@ class TestStats { runtest.pass("Calculating modified " + tag); } else { TestState runtest; - runtest.fail("Calculating modified " + tag); + runtest.fail("Calculating modified " + tag + " file " + filename); } } else if (this->verbose) { std::cout << "[Underpass] modified_" + tag + ": 0" << std::endl; @@ -237,13 +237,13 @@ class TestStats { if (this->verbose) { std::cout << "changeset: " << changestats->changeset << std::endl; } - testStat(changestats, validation, "highway"); - testStat(changestats, validation, "building"); - testStat(changestats, validation, "humanitarian_building"); - testStat(changestats, validation, "police"); - testStat(changestats, validation, "fire_station"); - testStat(changestats, validation, "hospital"); - testStat(changestats, validation, "waterway"); + testStat(changestats, validation, "highway", statsFile); + testStat(changestats, validation, "building", statsFile); + testStat(changestats, validation, "humanitarian_building" ,statsFile); + testStat(changestats, validation, "police" ,statsFile); + testStat(changestats, validation, "fire_station" ,statsFile); + testStat(changestats, validation, "hospital" ,statsFile); + testStat(changestats, validation, "waterway" ,statsFile); } } } diff --git a/src/testsuite/libunderpass.all/statsconfig-test.cc b/src/testsuite/libunderpass.all/statsconfig-test.cc index 56c1a2304..dd0fc7b22 100644 --- a/src/testsuite/libunderpass.all/statsconfig-test.cc +++ b/src/testsuite/libunderpass.all/statsconfig-test.cc @@ -1,5 +1,5 @@ // -// Copyright (c) 2020, 2021, 2022, 2023 Humanitarian OpenStreetMap Team +// Copyright (c) 2020, 2021, 2022, 2023, 2024 Humanitarian OpenStreetMap Team // // This file is part of Underpass. // diff --git a/src/testsuite/libunderpass.all/test-playground.cc b/src/testsuite/libunderpass.all/test-playground.cc index 15f51951b..db05201c3 100644 --- a/src/testsuite/libunderpass.all/test-playground.cc +++ b/src/testsuite/libunderpass.all/test-playground.cc @@ -1,5 +1,5 @@ // -// Copyright (c) 2020, 2021, 2022, 2023 Humanitarian OpenStreetMap Team +// Copyright (c) 2020, 2021, 2022, 2023, 2024 Humanitarian OpenStreetMap Team // // This file is part of Underpass. // diff --git a/src/testsuite/libunderpass.all/under-test.cc b/src/testsuite/libunderpass.all/under-test.cc index 84ab26d24..27e3914be 100644 --- a/src/testsuite/libunderpass.all/under-test.cc +++ b/src/testsuite/libunderpass.all/under-test.cc @@ -1,5 +1,5 @@ // -// Copyright (c) 2020, 2021, 2022, 2023 Humanitarian OpenStreetMap Team +// Copyright (c) 2020, 2021, 2022, 2023, 2024 Humanitarian OpenStreetMap Team // // This file is part of Underpass. // diff --git a/src/testsuite/libunderpass.all/val-test.cc b/src/testsuite/libunderpass.all/val-test.cc index 9c1c72075..870abdbfe 100644 --- a/src/testsuite/libunderpass.all/val-test.cc +++ b/src/testsuite/libunderpass.all/val-test.cc @@ -1,5 +1,5 @@ // -// Copyright (c) 2020, 2021, 2022, 2023 Humanitarian OpenStreetMap Team +// Copyright (c) 2020, 2021, 2022, 2023, 2024 Humanitarian OpenStreetMap Team // // This file is part of Underpass. // diff --git a/src/testsuite/libunderpass.all/val-unsquared-test.cc b/src/testsuite/libunderpass.all/val-unsquared-test.cc index 4b857eb5f..7c75997fe 100644 --- a/src/testsuite/libunderpass.all/val-unsquared-test.cc +++ b/src/testsuite/libunderpass.all/val-unsquared-test.cc @@ -1,5 +1,5 @@ // -// Copyright (c) 2023 Humanitarian OpenStreetMap Team +// Copyright (c) 2023, 2024 Humanitarian OpenStreetMap Team // // This file is part of Underpass. // diff --git a/src/testsuite/libunderpass.all/yaml-test.cc b/src/testsuite/libunderpass.all/yaml-test.cc index 8eab7fd29..0a218f465 100644 --- a/src/testsuite/libunderpass.all/yaml-test.cc +++ b/src/testsuite/libunderpass.all/yaml-test.cc @@ -1,5 +1,5 @@ // -// Copyright (c) 2020, 2021, 2022, 2023 Humanitarian OpenStreetMap Team +// Copyright (c) 2020, 2021, 2022, 2023, 2024 Humanitarian OpenStreetMap Team // // This file is part of Underpass. // diff --git a/src/testsuite/testdata/raw/raw-case-1.osc b/src/testsuite/testdata/raw/raw-case-1.osc index d8adf1813..387435d2c 100644 --- a/src/testsuite/testdata/raw/raw-case-1.osc +++ b/src/testsuite/testdata/raw/raw-case-1.osc @@ -1,10 +1,10 @@ - - - - + + + + diff --git a/src/testsuite/testdata/raw/raw-case-10.osc b/src/testsuite/testdata/raw/raw-case-10.osc new file mode 100644 index 000000000..d97734b60 --- /dev/null +++ b/src/testsuite/testdata/raw/raw-case-10.osc @@ -0,0 +1,49 @@ + + + + + + + + + + + \n \n + \n \n + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/src/testsuite/testdata/raw/raw-case-11.osc b/src/testsuite/testdata/raw/raw-case-11.osc new file mode 100644 index 000000000..032c5e00b --- /dev/null +++ b/src/testsuite/testdata/raw/raw-case-11.osc @@ -0,0 +1,123 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + \n \n + + + + + + \n \n \n \n \n + + \n \n \n \n \n + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/src/testsuite/testdata/raw/raw-case-12.osc b/src/testsuite/testdata/raw/raw-case-12.osc new file mode 100644 index 000000000..b0a114f3b --- /dev/null +++ b/src/testsuite/testdata/raw/raw-case-12.osc @@ -0,0 +1,190 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/src/testsuite/testdata/raw/raw-case-13.osc b/src/testsuite/testdata/raw/raw-case-13.osc new file mode 100644 index 000000000..a66af5dc7 --- /dev/null +++ b/src/testsuite/testdata/raw/raw-case-13.osc @@ -0,0 +1,68 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/src/testsuite/testdata/raw/raw-case-14.osc b/src/testsuite/testdata/raw/raw-case-14.osc new file mode 100644 index 000000000..1ccc07c6f --- /dev/null +++ b/src/testsuite/testdata/raw/raw-case-14.osc @@ -0,0 +1,80 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/src/testsuite/testdata/raw/raw-case-15.osc b/src/testsuite/testdata/raw/raw-case-15.osc new file mode 100644 index 000000000..fb168542e --- /dev/null +++ b/src/testsuite/testdata/raw/raw-case-15.osc @@ -0,0 +1,420 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/src/testsuite/testdata/raw/raw-case-16.osc b/src/testsuite/testdata/raw/raw-case-16.osc new file mode 100644 index 000000000..c724f8fca --- /dev/null +++ b/src/testsuite/testdata/raw/raw-case-16.osc @@ -0,0 +1,1133 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/src/testsuite/testdata/raw/raw-case-3.osc b/src/testsuite/testdata/raw/raw-case-3.osc index 2d411bfb9..2c261a330 100644 --- a/src/testsuite/testdata/raw/raw-case-3.osc +++ b/src/testsuite/testdata/raw/raw-case-3.osc @@ -1,6 +1,6 @@ - + \ No newline at end of file diff --git a/src/testsuite/testdata/raw/raw-case-4.osc b/src/testsuite/testdata/raw/raw-case-4.osc index 1c744f9e8..03d4dcfb2 100644 --- a/src/testsuite/testdata/raw/raw-case-4.osc +++ b/src/testsuite/testdata/raw/raw-case-4.osc @@ -1,10 +1,10 @@ - - - - + + + + diff --git a/src/testsuite/testdata/raw/raw-case-5.osc b/src/testsuite/testdata/raw/raw-case-5.osc index f6d041a69..f4a709a3d 100644 --- a/src/testsuite/testdata/raw/raw-case-5.osc +++ b/src/testsuite/testdata/raw/raw-case-5.osc @@ -1,7 +1,7 @@ - + diff --git a/src/testsuite/testdata/raw/raw-case-6.osc b/src/testsuite/testdata/raw/raw-case-6.osc index e7baaa44a..c6add96c0 100644 --- a/src/testsuite/testdata/raw/raw-case-6.osc +++ b/src/testsuite/testdata/raw/raw-case-6.osc @@ -1,10 +1,10 @@ - - - - + + + + diff --git a/src/testsuite/testdata/stats/test_stats.yaml b/src/testsuite/testdata/stats/test_stats.yaml index b150dc057..6623d3033 100644 --- a/src/testsuite/testdata/stats/test_stats.yaml +++ b/src/testsuite/testdata/stats/test_stats.yaml @@ -7,7 +7,7 @@ - modified_building: - 1 - added_building: - - 1 + - 2 - modified_waterway: - 1 - added_waterway: diff --git a/src/unconfig.h b/src/unconfig.h deleted file mode 100644 index 0f40e20b7..000000000 --- a/src/unconfig.h +++ /dev/null @@ -1,156 +0,0 @@ -/* unconfig.h. Generated from unconfig.h.in by configure. */ -/* unconfig.h.in. Generated from configure.ac by autoheader. */ - -/* Define to 1 if translation of program messages to the user's native - language is requested. */ -/* #undef ENABLE_NLS */ - -/* define if the Boost library is available */ -#define HAVE_BOOST /**/ - -/* define if the Boost::Date_Time library is available */ -#define HAVE_BOOST_DATE_TIME /**/ - -/* define if the Boost::Filesystem library is available */ -#define HAVE_BOOST_FILESYSTEM /**/ - -/* define if the Boost::IOStreams library is available */ -#define HAVE_BOOST_IOSTREAMS /**/ - -/* define if the Boost::Locale library is available */ -#define HAVE_BOOST_LOCALE /**/ - -/* define if the Boost::Log library is available */ -#define HAVE_BOOST_LOG /**/ - -/* define if the Boost::PROGRAM_OPTIONS library is available */ -#define HAVE_BOOST_PROGRAM_OPTIONS /**/ - -/* define if the Boost::Python library is available */ -#define HAVE_BOOST_PYTHON /**/ - -/* define if the Boost::Serialization library is available */ -#define HAVE_BOOST_SERIALIZATION /**/ - -/* define if the Boost::System library is available */ -#define HAVE_BOOST_SYSTEM /**/ - -/* define if the Boost::Thread library is available */ -#define HAVE_BOOST_THREAD /**/ - -/* define if the Boost::Timer library is available */ -#define HAVE_BOOST_TIMER /**/ - -/* Define to 1 if you have the Mac OS X function - CFLocaleCopyPreferredLanguages in the CoreFoundation framework. */ -/* #undef HAVE_CFLOCALECOPYPREFERREDLANGUAGES */ - -/* Define to 1 if you have the Mac OS X function CFPreferencesCopyAppValue in - the CoreFoundation framework. */ -/* #undef HAVE_CFPREFERENCESCOPYAPPVALUE */ - -/* Define if the GNU dcgettext() function is already present or preinstalled. - */ -/* #undef HAVE_DCGETTEXT */ - -/* Define to 1 if you have the header file. */ -#define HAVE_DLFCN_H 1 - -/* Define if the GNU gettext() function is already present or preinstalled. */ -/* #undef HAVE_GETTEXT */ - -/* Define if you have the iconv() function and it works. */ -/* #undef HAVE_ICONV */ - -/* Define to 1 if you have the header file. */ -#define HAVE_INTTYPES_H 1 - -/* Define to 1 if you have the header file. */ -#define HAVE_MEMORY_H 1 - -/* Define to 1 if you have the header file. */ -#define HAVE_OSMIUM_OSM_NODE_HPP 1 - -/* If available, contains the Python version number currently in use. */ -#define HAVE_PYTHON "3.8" - -/* Define to 1 if you have the header file. */ -#define HAVE_STDINT_H 1 - -/* Define to 1 if you have the header file. */ -#define HAVE_STDLIB_H 1 - -/* Define to 1 if you have the header file. */ -#define HAVE_STRINGS_H 1 - -/* Define to 1 if you have the header file. */ -#define HAVE_STRING_H 1 - -/* Define to 1 if you have the header file. */ -#define HAVE_SYS_STAT_H 1 - -/* Define to 1 if you have the header file. */ -#define HAVE_SYS_TYPES_H 1 - -/* Define to 1 if you have the header file. */ -#define HAVE_UNISTD_H 1 - -/* Define to 1 if the system has the type `_Bool'. */ -#define HAVE__BOOL 1 - -/* Use libxml++ library */ -#define LIBXML 1 - -/* Define to the sub-directory where libtool stores uninstalled libraries. */ -#define LT_OBJDIR ".libs/" - -/* Enable memory debugging */ -/* #undef MEMORY_DEBUG */ - -/* Name of package */ -#define PACKAGE "underpass" - -/* Define to the address where bug reports for this package should be sent. */ -#define PACKAGE_BUGREPORT "" - -/* Define to the full name of this package. */ -#define PACKAGE_NAME "underpass" - -/* Define to the full name and version of this package. */ -#define PACKAGE_STRING "underpass 0.3_dev" - -/* Define to the one symbol short name of this package. */ -#define PACKAGE_TARNAME "underpass" - -/* Define to the home page for this package. */ -#define PACKAGE_URL "" - -/* Define to the version of this package. */ -#define PACKAGE_VERSION "0.3_dev" - -/* Use rapidxml library in boost */ -/* #undef RAPIDXML */ - -/* Define to 1 if you have the ANSI C header files. */ -#define STDC_HEADERS 1 - -/* Store downloaded files to disk */ -#define USE_CACHE 1 - -/* Do additional conflation calculations */ -/* #undef USE_CONFLATION */ - -/* Use multiple threaded downloader */ -/* #undef USE_MULTI_LOADER */ - -/* Enable Python binding */ -#define USE_PYTHON 1 - -/* Don't use multiple threaded file downloader */ -#define USE_SINGLE_LOADER 1 - -/* Use tmp file for downloaded files */ -/* #undef USE_TMPFILE */ - -/* Version number of package */ -#define VERSION "0.3_dev" diff --git a/src/underpass.cc b/src/underpass.cc index 689bd91f0..34575ecb4 100644 --- a/src/underpass.cc +++ b/src/underpass.cc @@ -1,5 +1,5 @@ // -// Copyright (c) 2020, 2021, 2022, 2023 Humanitarian OpenStreetMap Team +// Copyright (c) 2020, 2021, 2022, 2023, 2024 Humanitarian OpenStreetMap Team // // This file is part of Underpass. // @@ -123,7 +123,8 @@ main(int argc, char *argv[]) ("disable-raw", "Disable raw OSM data") ("norefs", "Disable refs (useful for non OSM data)") ("bootstrap", "Bootstrap data tables") - ("silent", "Silent"); + ("silent", "Silent") + ("rawdb", opts::value(), "Database URI for raw OSM data"); // clang-format on opts::store(opts::command_line_parser(argc, argv).options(desc).positional(p).run(), vm); @@ -156,6 +157,10 @@ main(int argc, char *argv[]) } // Database + if (vm.count("rawdb")) { + config.underpass_osm_db_url = vm["rawdb"].as(); + } + if (vm.count("server")) { config.underpass_db_url = vm["server"].as(); } @@ -291,7 +296,8 @@ main(int argc, char *argv[]) // OsmChanges std::thread osmChangeThread; - if (!vm.count("changesets")) { + if ((!vm.count("changesets") && !vm.count("changeseturl")) || + (vm.count("changeseturl") && (vm.count("timestamp") || vm.count("url")))) { multipolygon_t * osmboundary = &poly; if (!vm.count("osmnoboundary")) { osmboundary = &geou.boundary; @@ -300,8 +306,13 @@ main(int argc, char *argv[]) if (!config.silent) { osmchange->dump(); } +#ifdef SINGLE_THREAD // debugging hack + replicatorthreads::startMonitorChanges(std::ref(osmchange), + std::ref(*osmboundary), config); +#else osmChangeThread = std::thread(replicatorthreads::startMonitorChanges, std::ref(osmchange), std::ref(*osmboundary), config); +#endif } // Changesets @@ -320,18 +331,23 @@ main(int argc, char *argv[]) if (!config.silent) { changeset->dump(); } +#ifdef SINGLE_THREAD // debugging hack + replicatorthreads::startMonitorChangesets(std::ref(changeset), std::ref(*oscboundary), config); +#else changesetThread = std::thread(replicatorthreads::startMonitorChangesets, std::ref(changeset), std::ref(*oscboundary), config); +#endif } // Start processing - +#ifndef SINGLE_THREAD if (changesetThread.joinable()) { changesetThread.join(); } if (osmChangeThread.joinable()) { osmChangeThread.join(); } +#endif exit(0); diff --git a/src/underpassconfig.hh b/src/underpassconfig.hh index 7ab92ce9d..d3acf66aa 100644 --- a/src/underpassconfig.hh +++ b/src/underpassconfig.hh @@ -98,17 +98,19 @@ struct UnderpassConfig { /// /// \brief underpassconfig constructor: will try to initialize from uppercased same-name - /// environment variables prefixed by REPLICATOR_ (e.g. REPLICATOR_underpass_db_URL) + /// environment variables prefixed by REPLICATOR_ (e.g. REPLICATOR_underpass_db_RL) /// UnderpassConfig() { - std::string filespec = ETCDIR; - filespec += "default.yaml"; + filespec += "/default.yaml"; if (std::filesystem::exists(filespec)) { yaml::Yaml yaml; yaml.read(filespec); auto yamlConfig = yaml.get("config"); + if (yaml.contains_key("underpass_osm_db_url")) { + underpass_osm_db_url = yamlConfig.get_value("underpass_osm_db_url"); + } if (yaml.contains_key("underpass_db_url")) { underpass_db_url = yamlConfig.get_value("underpass_db_url"); } @@ -128,6 +130,9 @@ struct UnderpassConfig { } } + if (getenv("REPLICATOR_OSM_DB_URL")) { + underpass_osm_db_url = getenv("REPLICATOR_OSM_DB_URL"); + } if (getenv("REPLICATOR_UNDERPASS_DB_URL")) { underpass_db_url = getenv("REPLICATOR_UNDERPASS_DB_URL"); } @@ -154,6 +159,7 @@ struct UnderpassConfig { } }; + std::string underpass_osm_db_url = "localhost/underpass"; std::string underpass_db_url = "localhost/underpass"; std::string destdir_base; std::string planet_server; diff --git a/src/utils/geo.cc b/src/utils/geo.cc index 3ab0a1e7a..e884d2580 100644 --- a/src/utils/geo.cc +++ b/src/utils/geo.cc @@ -1,5 +1,5 @@ // -// Copyright (c) 2023 Humanitarian OpenStreetMap Team +// Copyright (c) 2023, 2024 Humanitarian OpenStreetMap Team // // This file is part of Underpass. // diff --git a/src/utils/log.cc b/src/utils/log.cc index 213975a65..584ea44fc 100644 --- a/src/utils/log.cc +++ b/src/utils/log.cc @@ -1,5 +1,5 @@ // -// Copyright (c) 2020, 2021, 2022, 2023 Humanitarian OpenStreetMap Team +// Copyright (c) 2020, 2021, 2022, 2023, 2024 Humanitarian OpenStreetMap Team // // This file is part of Underpass. // diff --git a/src/utils/yaml.cc b/src/utils/yaml.cc index c6c3e20ec..48cd9dd4d 100644 --- a/src/utils/yaml.cc +++ b/src/utils/yaml.cc @@ -1,5 +1,5 @@ // -// Copyright (c) 2020, 2021, 2022, 2023 Humanitarian OpenStreetMap Team +// Copyright (c) 2020, 2021, 2022, 2023, 2024 Humanitarian OpenStreetMap Team // // This file is part of Underpass. // diff --git a/src/utils/yaml.hh b/src/utils/yaml.hh index 2ad7a8242..335905e30 100644 --- a/src/utils/yaml.hh +++ b/src/utils/yaml.hh @@ -48,7 +48,7 @@ class Node { /// /// \brief children the Node can contain several children /// - std::vector children; + std::vector children; /// /// \brief get returns a Node identified by a key /// \param key is the value that must to match with the Node diff --git a/src/validate/Makefile.am b/src/validate/Makefile.am index c593637cf..a89218e92 100644 --- a/src/validate/Makefile.am +++ b/src/validate/Makefile.am @@ -55,4 +55,4 @@ AM_CPPFLAGS = \ # $(MKDIR_P) $(DESTDIR)/$(pkglibdir) # cp -vp .libs/libunderpass.so $(DESTDIR)/$(pkglibdir) # $(MKDIR_P) $(DESTDIR)/$(pkglibdir)/config/validate -# cp -rvp $(top_srcdir)/config/validate/*.yaml $(DESTDIR)/$(pkglibdir)/config/validate +# cp -rvp $(top_srcdir)/config/validate/*.yaml $(DESTDIR)/$(pkglibdir)/config/validate \ No newline at end of file diff --git a/src/validate/defaultvalidation.cc b/src/validate/defaultvalidation.cc index b4520ad4c..9a30ce9ba 100644 --- a/src/validate/defaultvalidation.cc +++ b/src/validate/defaultvalidation.cc @@ -1,5 +1,5 @@ // -// Copyright (c) 2020, 2021, 2022, 2023 Humanitarian OpenStreetMap Team +// Copyright (c) 2020, 2021, 2022, 2023, 2024 Humanitarian OpenStreetMap Team // // This file is part of Underpass. // diff --git a/src/validate/geospatial.cc b/src/validate/geospatial.cc index 81a2f9925..6f19a9047 100644 --- a/src/validate/geospatial.cc +++ b/src/validate/geospatial.cc @@ -1,5 +1,5 @@ // -// Copyright (c) 2020, 2021, 2022, 2023 Humanitarian OpenStreetMap Team +// Copyright (c) 2020, 2021, 2022, 2023, 2024 Humanitarian OpenStreetMap Team // // This file is part of Underpass. // diff --git a/src/validate/queryvalidate.cc b/src/validate/queryvalidate.cc index afdd11db6..b12fac85e 100644 --- a/src/validate/queryvalidate.cc +++ b/src/validate/queryvalidate.cc @@ -1,5 +1,5 @@ // -// Copyright (c) 2020, 2021, 2022, 2023 Humanitarian OpenStreetMap Team +// Copyright (c) 2020, 2021, 2022, 2023, 2024 Humanitarian OpenStreetMap Team // // This file is part of Underpass. // @@ -87,63 +87,67 @@ QueryValidate::QueryValidate(std::shared_ptr db) { dbconn = db; } -std::string +std::shared_ptr QueryValidate::updateValidation(std::shared_ptr> removals) { #ifdef TIMING_DEBUG_X boost::timer::auto_cpu_timer timer("updateValidation: took %w seconds\n"); #endif - std::string query = ""; + auto query = std::make_shared(); if (removals->size() > 0) { - query = "DELETE FROM validation WHERE osm_id IN("; + *query = "DELETE FROM validation WHERE osm_id IN("; for (const auto &osm_id : *removals) { - query += std::to_string(osm_id) + ","; + query->append(std::to_string(osm_id) + ","); }; - query.erase(query.size() - 1); - query += ");"; + query->erase(query->size() - 1); + query->append(");"); } return query; } -std::string -QueryValidate::updateValidation(long osm_id, const valerror_t &status, const std::string &source) const +std::shared_ptr +QueryValidate::updateValidation(long osm_id, + const valerror_t &status, + const std::string &source) const { + auto query = std::make_shared(); std::string format = "DELETE FROM validation WHERE osm_id = %d and source = '%s' and status = '%s';"; boost::format fmt(format); fmt % osm_id; fmt % source; fmt % status_list[status]; - std::string query = fmt.str(); + *query = fmt.str(); return query; } -std::string +std::shared_ptr QueryValidate::updateValidation(long osm_id, const valerror_t &status) const { + auto query = std::make_shared(); std::string format = "DELETE FROM validation WHERE osm_id = %d and status = '%s';"; boost::format fmt(format); fmt % osm_id; fmt % status_list[status]; - std::string query = fmt.str(); + *query = fmt.str(); return query; } -std::string +std::shared_ptr QueryValidate::applyChange(const ValidateStatus &validation, const valerror_t &status) const { #ifdef TIMING_DEBUG_X boost::timer::auto_cpu_timer timer("applyChange(validation): took %w seconds\n"); #endif - log_debug("Applying Validation data"); + // log_debug("Applying Validation data"); std::string format; - std::string query; + auto query = std::make_shared(); if (validation.values.size() > 0) { - query = "INSERT INTO validation as v (osm_id, changeset, uid, type, status, values, timestamp, location, source, version) VALUES("; + *query = "INSERT INTO validation as v (osm_id, changeset, uid, type, status, values, timestamp, location, source, version) VALUES("; format = "%d, %d, %g, \'%s\', \'%s\', ARRAY[%s], \'%s\', ST_GeomFromText(\'%s\', 4326), \'%s\', %s) "; } else { - query = "INSERT INTO validation as v (osm_id, changeset, uid, type, status, timestamp, location, source, version) VALUES("; + *query = "INSERT INTO validation as v (osm_id, changeset, uid, type, status, timestamp, location, source, version) VALUES("; format = "%d, %d, %g, \'%s\', \'%s\', \'%s\', ST_GeomFromText(\'%s\', 4326), \'%s\', %s) "; } format += "ON CONFLICT (osm_id, status, source) DO UPDATE SET version = %d, timestamp = \'%s\' WHERE v.version < %d;"; @@ -177,87 +181,86 @@ QueryValidate::applyChange(const ValidateStatus &validation, const valerror_t &s fmt % validation.version; fmt % to_simple_string(validation.timestamp); fmt % validation.version; - query += fmt.str(); + query->append(fmt.str()); return query; } - -void -QueryValidate::ways( - std::shared_ptr>> wayval, - std::string &task_query -) { +std::shared_ptr> +QueryValidate::ways(std::shared_ptr>> wayval) { + auto query = std::make_shared>(); for (auto it = wayval->begin(); it != wayval->end(); ++it) { if (it->get()->status.size() > 0) { for (auto status_it = it->get()->status.begin(); status_it != it->get()->status.end(); ++status_it) { - task_query += applyChange(*it->get(), *status_it); + query->push_back(*applyChange(*it->get(), *status_it)); } } } + return query; } -void +std::shared_ptr> QueryValidate::ways( std::shared_ptr>> wayval, - std::string &task_query, std::shared_ptr> validation_removals ) { + auto query = std::make_shared>(); for (auto it = wayval->begin(); it != wayval->end(); ++it) { if (it->get()->status.size() > 0) { for (auto status_it = it->get()->status.begin(); status_it != it->get()->status.end(); ++status_it) { - task_query += applyChange(*it->get(), *status_it); + query->push_back(*applyChange(*it->get(), *status_it)); } if (!it->get()->hasStatus(overlapping)) { - task_query += updateValidation(it->get()->osm_id, overlapping, "building"); + query->push_back(*updateValidation(it->get()->osm_id, overlapping, "building")); } if (!it->get()->hasStatus(duplicate)) { - task_query += updateValidation(it->get()->osm_id, duplicate, "building"); + query->push_back(*updateValidation(it->get()->osm_id, duplicate, "building")); } if (!it->get()->hasStatus(badgeom)) { - task_query += updateValidation(it->get()->osm_id, badgeom, "building"); + query->push_back(*updateValidation(it->get()->osm_id, badgeom, "building")); } if (!it->get()->hasStatus(badvalue)) { - task_query += updateValidation(it->get()->osm_id, badvalue); + query->push_back(*updateValidation(it->get()->osm_id, badvalue)); } } else { validation_removals->push_back(it->get()->osm_id); } } + return query; } -void -QueryValidate::nodes( - std::shared_ptr>> nodeval, - std::string &task_query -) { +std::shared_ptr> +QueryValidate::nodes(std::shared_ptr>> nodeval) { + auto query = std::make_shared>(); for (auto it = nodeval->begin(); it != nodeval->end(); ++it) { if (it->get()->status.size() > 0) { for (auto status_it = it->get()->status.begin(); status_it != it->get()->status.end(); ++status_it) { - task_query += applyChange(*it->get(), *status_it); + query->push_back(*applyChange(*it->get(), *status_it)); } } } + return query; } -void +std::shared_ptr> QueryValidate::nodes( std::shared_ptr>> nodeval, - std::string &task_query, std::shared_ptr> validation_removals ) { + auto query = std::make_shared>(); for (auto it = nodeval->begin(); it != nodeval->end(); ++it) { if (it->get()->status.size() > 0) { for (auto status_it = it->get()->status.begin(); status_it != it->get()->status.end(); ++status_it) { - task_query += applyChange(*it->get(), *status_it); + query->push_back(*applyChange(*it->get(), *status_it)); } if (!it->get()->hasStatus(badvalue)) { - task_query += updateValidation(it->get()->osm_id, badvalue); + query->push_back(*updateValidation(it->get()->osm_id, badvalue)); } } else { validation_removals->push_back(it->get()->osm_id); } } + return query; } } // namespace queryvalidate diff --git a/src/validate/queryvalidate.hh b/src/validate/queryvalidate.hh index f3c3536f6..01b81acf8 100644 --- a/src/validate/queryvalidate.hh +++ b/src/validate/queryvalidate.hh @@ -74,17 +74,30 @@ class QueryValidate { QueryValidate(std::shared_ptr db); /// Apply data validation to the database - std::string applyChange(const ValidateStatus &validation, const valerror_t &status) const; + std::shared_ptr applyChange(const ValidateStatus &validation, + const valerror_t &status) const; /// Update the validation table, delete any feature that has been fixed. - std::string updateValidation(std::shared_ptr> removals); - std::string updateValidation(long osm_id, const valerror_t &status, const std::string &source) const; - std::string updateValidation(long osm_id, const valerror_t &status) const; - void ways(std::shared_ptr>> wayval, std::string &task_query); - void nodes(std::shared_ptr>> nodeval, std::string &task_query); - void rels(std::shared_ptr>> relval, std::string &task_query); - void ways(std::shared_ptr>> wayval, std::string &task_query, std::shared_ptr> validation_removals); - void nodes(std::shared_ptr>> nodeval, std::string &task_query, std::shared_ptr> validation_removals); - void rels(std::shared_ptr>> relval, std::string &task_query, std::shared_ptr> validation_removals); + std::shared_ptr updateValidation( + std::shared_ptr> removals); + std::shared_ptr updateValidation( + long osm_id, const valerror_t &status, const std::string &source) const; + std::shared_ptr updateValidation( + long osm_id, const valerror_t &status) const; + std::shared_ptr> ways( + std::shared_ptr>> wayval); + std::shared_ptr> ways( + std::shared_ptr>> wayval, + std::shared_ptr> validation_removals); + std::shared_ptr> nodes( + std::shared_ptr>> nodeval); + std::shared_ptr> nodes( + std::shared_ptr>> nodeval, + std::shared_ptr> validation_removals); + std::shared_ptr> rels( + std::shared_ptr>> relval); + std::shared_ptr> rels( + std::shared_ptr>> relval, + std::shared_ptr> validation_removals); // Database connection, used for escape strings std::shared_ptr dbconn; }; diff --git a/src/validate/semantic.cc b/src/validate/semantic.cc index 11bb05dca..ad066d489 100644 --- a/src/validate/semantic.cc +++ b/src/validate/semantic.cc @@ -1,5 +1,5 @@ // -// Copyright (c) 2020, 2021, 2022, 2023 Humanitarian OpenStreetMap Team +// Copyright (c) 2020, 2021, 2022, 2023, 2024 Humanitarian OpenStreetMap Team // // This file is part of Underpass. // diff --git a/src/wrappers/python.cc b/src/wrappers/python.cc index eff00f571..9f392a4bd 100644 --- a/src/wrappers/python.cc +++ b/src/wrappers/python.cc @@ -1,5 +1,5 @@ // -// Copyright (c) 2020, 2021, 2022, 2023 Humanitarian OpenStreetMap Team +// Copyright (c) 2020, 2021, 2022, 2023, 2024 Humanitarian OpenStreetMap Team // // This file is part of Underpass. //