Improvements for Python DB API query performance.

hotosm · Jan 11, 2024 · 6b0f2d5 · 6b0f2d5
1 parent 6fbfb2a
commit 6b0f2d5
Show file tree

Hide file tree

Showing 3 changed files with 36 additions and 33 deletions.
diff --git a/python/dbapi/api/filters.py b/python/dbapi/api/filters.py
@@ -18,4 +18,4 @@ def tagsQueryFilter(tagsQuery, table):
     return query
 
 def hashtagQueryFilter(hashtag, table):
-    return table + ".changeset IN (SELECT c.id FROM changesets c where jsonb_path_exists(to_jsonb(hashtags), '$[*] ? (@ like_regex \"^{0}\")') GROUP BY C.id)".format(hashtag)
+    return "'{0}' = ANY (hashtags)".format(hashtag)
diff --git a/python/dbapi/api/raw.py b/python/dbapi/api/raw.py
@@ -68,6 +68,8 @@ def listAllFeaturesQuery(
         status,
         orderBy,
         page,
+        dateFrom,
+        dateTo,
         table,
     ):
 
@@ -82,10 +84,12 @@ def listAllFeaturesQuery(
             SELECT '" + osmType + "' as type, '" + geoType + "' as geotype, " + table + ".osm_id as id, ST_X(ST_Centroid(geom)) as lat, ST_Y(ST_Centroid(geom)) as lon, " + table + ".timestamp, tags, " + table + ".changeset, c.created_at, v.status FROM " + table + " \
             LEFT JOIN changesets c ON c.id = " + table + ".changeset \
             LEFT JOIN validation v ON v.osm_id = " + table + ".osm_id \
-            WHERE {0} {1} {2} {3} {4} \
+            WHERE {0} {1} {2} {3} {4} {5} {6} \
             )\
         ".format(
-            "status = '{0}'".format(status) if (status) else "1=1",
+            "created_at >= '{0}'".format(dateFrom) if (dateFrom) else "1=1",
+            "AND created_at <= '{0}'".format(dateTo) if (dateTo) else "",
+            "AND status = '{0}'".format(status) if (status) else "",
             "AND " + hashtagQueryFilter(hashtag, table) if hashtag else "",
             "AND ST_Intersects(\"geom\", ST_GeomFromText('MULTIPOLYGON((({0})))', 4326) )".format(area) if area else "",
             "AND (" + tagsQueryFilter(tags, table) + ")" if tags else "",
@@ -98,13 +102,14 @@ def queryToJSONAllFeatures(query, dateFrom, dateTo, orderBy):
     query = "with predata AS (" + query + ") , \
            data as ( \
                 select predata.type, geotype, predata.id, predata.timestamp, tags, status, predata.changeset, predata.created_at as created_at, lat, lon from predata \
-                WHERE {0} {1} \
+                WHERE {0} {1} {2} \
             ),\
             t_features AS ( \
             SELECT to_jsonb(data) as feature from data \
         ) SELECT jsonb_agg(t_features.feature) as result FROM t_features;" \
         .format(
-            "created_at >= '{0}' AND created_at <= '{1}'".format(dateFrom, dateTo) if (dateFrom and dateTo) else "1=1",
+            "created_at >= '{0}'".format(dateFrom) if (dateFrom) else "1=1",
+            "AND created_at <= '{0}'".format(dateTo) if (dateTo) else "",
             "AND {0}{1} IS NOT NULL ORDER BY {0}{1} DESC".format("predata.",orderBy) if orderBy != "osm_id" else "ORDER BY id DESC",
         ).replace("WHERE 1=1 AND", "WHERE")
     return query
@@ -359,6 +364,8 @@ def getPolygonsList(
             status,
             orderBy or "ways_poly.osm_id",
             page or 0,
+            dateFrom,
+            dateTo,
             "ways_poly")
 
         query = queryToJSONAllFeatures(
@@ -388,6 +395,8 @@ def getLinesList(
             status,
             orderBy or "ways_line.osm_id",
             page or 0,
+            dateFrom,
+            dateTo,
             "ways_line")
 
         query = queryToJSONAllFeatures(
@@ -417,6 +426,8 @@ def getNodesList(
             status,
             orderBy or "nodes.osm_id",
             page or 0,
+            dateFrom,
+            dateTo,
             "nodes")
 
         query = queryToJSONAllFeatures(
@@ -446,6 +457,8 @@ def getAllList(
         status,
         orderBy or "ways_poly.osm_id",
         page or 0,
+        dateFrom,
+        dateTo,
         "ways_poly")
 
         queryLines = listAllFeaturesQuery(
@@ -455,6 +468,8 @@ def getAllList(
         status,
         orderBy or "ways_line.osm_id",
         page or 0,
+        dateFrom,
+        dateTo,
         "ways_line")
 
         queryNodes = listAllFeaturesQuery(
@@ -464,6 +479,8 @@ def getAllList(
         status,
         orderBy or "nodes.osm_id",
         page or 0,
+        dateFrom,
+        dateTo,
         "nodes")
 
         query = queryToJSONAllFeatures(

diff --git a/python/dbapi/api/stats.py b/python/dbapi/api/stats.py
@@ -40,34 +40,20 @@ def getCount(
         else:
             table = "ways_poly"
 
-        if status:
-            query = "with t1 as ( \
-                    select count(validation.osm_id) from validation \
-                    left join " + table + " on validation.osm_id = " + table + ".osm_id \
-                    where {0} {1} {2} {3} {4}".format(
-                    "ST_Intersects(\"geom\", ST_GeomFromText('MULTIPOLYGON((({0})))', 4326) )".format(area) if area else "1=1 ",
-                    "AND (" + tagsQueryFilter(tags, table) + ")" if tags else "",
-                    "AND " + hashtagQueryFilter(hashtag, table) if hashtag else "",
-                    "AND created at >= {0} AND created_at <= {1}".format(dateFrom, dateTo) if dateFrom and dateTo else "",
-                    "AND status = '" + status + "'")
-            query += "), t2 as ( \
-                    select count(" + table + ".osm_id) as total from " + table + " \
-                    where {0} {1} {2} {3} \
-                    ) select t1.count, t2.total from t1,t2".format(
-                    "ST_Intersects(\"geom\", ST_GeomFromText('MULTIPOLYGON((({0})))', 4326) )".format(area) if area else "1=1 ",
-                    "AND (" + tagsQueryFilter(tags, table) + ")" if tags else "",
-                    "AND " + hashtagQueryFilter(hashtag, table) if hashtag else "",
-                    "AND created at >= {0} AND created_at <= {1}".format(dateFrom, dateTo) if dateFrom and dateTo else "")
-        else:
-            query = "select count(" + table + ".osm_id) from " + table \
-                + " where \
-                {0} {1} {2} {3} {4}".format(
-                    "ST_Intersects(\"geom\", ST_GeomFromText('MULTIPOLYGON((({0})))', 4326) )".format(area) if area else "1=1 ",
-                    "AND (" + tagsQueryFilter(tags, table) + ")" if tags else "",
-                    "AND " + hashtagQueryFilter(hashtag, table) if hashtag else "",
-                    "AND created at >= {0} AND created_at <= {1}".format(dateFrom, dateTo) if dateFrom and dateTo else "",
-                    "AND status = '{0}'".format(status) if (status) else "",
-                )
+        query = "with features as ( \
+            select {0}.osm_id, ways_poly.changeset, tags, hashtags, status from {0} \
+            left join changesets c on changeset = c.id \
+            left join validation v on {0}.osm_id = v.osm_id \
+            where {1} {2} {3} {4} {5} \
+            ) select count(distinct(osm_id)) from features;".format(
+                table,
+                "created_at >= '{0}'".format(dateFrom) if (dateFrom) else "1=1",
+                "AND created_at <= '{0}'".format(dateTo) if (dateTo) else "",
+                "AND ST_Intersects(\"geom\", ST_GeomFromText('MULTIPOLYGON((({0})))', 4326) )".format(area) if area else "",
+                "AND (" + tagsQueryFilter(tags, table) + ")" if tags else "",
+                "AND " + hashtagQueryFilter(hashtag, table) if hashtag else "",
+                "AND status = '" + status + "'" if status else "",
+            )
         return(self.underpassDB.run(query, True))