Fix for underpass command with no args. Add support for bootstrap wit…

…h no refs
hotosm · Oct 23, 2023 · 35958c2 · 35958c2
1 parent 92e8fe9
commit 35958c2
Show file tree

Hide file tree

Showing 8 changed files with 93 additions and 39 deletions.
diff --git a/config/replicator/planetreplicator.yaml b/config/replicator/planetreplicator.yaml
@@ -22,6 +22,8 @@ minute:
     2021-04-15 10:01:23
   5000000:
     2022-04-03 16:25:35
+  5500000:
+    2023-03-22 21:57:39
 changeset:
   0:
     2012-10-28 19:36:01
@@ -42,6 +44,6 @@ changeset:
   4000000:
     2020-06-28 23:26:01
   4500000:
-    2021-06-17 03:53
-  5000000:
-    2022-05-30 17:50:02
+    2021-06-17 03:53:00
+  5500000:
+    2023-05-19 06:23:54
diff --git a/src/bootstrap/bootstrap.cc b/src/bootstrap/bootstrap.cc
@@ -42,12 +42,14 @@ namespace bootstrap {
 
 void startProcessingWays(const underpassconfig::UnderpassConfig &config) {
 
+    std::cout << "Connecting to the database (" << config.underpass_db_url << ") ..." << std::endl;
     auto db = std::make_shared<Pq>();
     if (!db->connect(config.underpass_db_url)) {
         log_error("Could not connect to Underpass DB, aborting monitoring thread!");
         return;
     }
 
+    std::cout << "Loading plugins ... " << std::endl;
     std::string plugins;
     if (boost::filesystem::exists("src/validate/.libs")) {
         plugins = "src/validate/.libs";
@@ -73,12 +75,15 @@ void startProcessingWays(const underpassconfig::UnderpassConfig &config) {
     };
 
     for (auto table_it = tables.begin(); table_it != tables.end(); ++table_it) {
+        std::cout << "Counting geometries ... " << std::endl;
         int total = queryraw->getWaysCount(*table_it);
+        std::cout << "Total ways:" << total << std::endl;
         if (total > 0) {
             int count = 0;
             long lastid = 0;
             while (count < total) {
                 int percentage = (count * 100) / total;
+                std::cout << "\r" << "Processing " << *table_it << ": " << count << "/" << total << " (" << percentage << "%)";
                 auto task = std::make_shared<BootstrapTask>();
                 WayTask wayTask;
                 wayTask.plugin = validator;
@@ -87,11 +92,10 @@ void startProcessingWays(const underpassconfig::UnderpassConfig &config) {
                 wayTask.task = task;
                 wayTask.lastid = lastid;
 
-                processWays(wayTask, *table_it);
+                processWays(wayTask, *table_it, config);
                 db->query(task->query);
                 lastid = wayTask.lastid;
                 count += wayTask.processed;
-                std::cout << "\r" << "Processing " << *table_it << ": " << count << "/" << total << " (" << percentage << "%)";
             }
         }
     }
@@ -100,7 +104,7 @@ void startProcessingWays(const underpassconfig::UnderpassConfig &config) {
 
 // This thread get started for every page of way
 void
-processWays(WayTask &wayTask, const std::string &tableName)
+processWays(WayTask &wayTask, const std::string &tableName, const underpassconfig::UnderpassConfig &config)
 {
 #ifdef TIMING_DEBUG
     boost::timer::auto_cpu_timer timer("bootstrap::processWays(wayTask): took %w seconds\n");
@@ -112,14 +116,18 @@ processWays(WayTask &wayTask, const std::string &tableName)
     auto queryraw = wayTask.queryraw;
     auto lastid = wayTask.lastid;
 
-    auto ways = queryraw->getWaysFromDB(lastid, tableName);
+    auto ways = std::make_shared<std::vector<OsmWay>>();
+    if (!config.norefs) {
+        ways = queryraw->getWaysFromDB(lastid, tableName);
+    } else {
+        ways = queryraw->getWaysFromDBWithoutRefs(lastid, tableName);
+    }
     wayTask.processed = ways->size();
     if (wayTask.processed > 0) {
         // Proccesing ways
         for (auto way = ways->begin(); way != ways->end(); ++way) {
-
             // If it's closed polygon
-            if (way->refs.front() == way->refs.back()) {
+            if (way->isClosed()) {
                 log_debug("Way Id: %1%", way->id);
 
                 // Bad geometry
@@ -135,8 +143,10 @@ processWays(WayTask &wayTask, const std::string &tableName)
             }
 
             // Fill the way_refs table
-            for (auto ref = way->refs.begin(); ref != way->refs.end(); ++ref) {
-                task->query += "INSERT INTO way_refs (way_id, node_id) VALUES (" + std::to_string(way->id) + "," + std::to_string(*ref) + "); ";
+            if (!config.norefs) {
+                for (auto ref = way->refs.begin(); ref != way->refs.end(); ++ref) {
+                    task->query += "INSERT INTO way_refs (way_id, node_id) VALUES (" + std::to_string(way->id) + "," + std::to_string(*ref) + "); ";
+                }
             }
         }
         wayTask.lastid = ways->back().id;

diff --git a/src/bootstrap/bootstrap.hh b/src/bootstrap/bootstrap.hh
@@ -47,6 +47,6 @@ struct WayTask {
 void startProcessingWays(const underpassconfig::UnderpassConfig &config);
 
 // This thread get started for every page of way
-void processWays(WayTask &wayTask, const std::string &tableName);
+void processWays(WayTask &wayTask, const std::string &tableName, const underpassconfig::UnderpassConfig &config);
 
 }
diff --git a/src/osm/osmobjects.hh b/src/osm/osmobjects.hh
@@ -169,12 +169,7 @@ class OsmWay : public OsmObject {
     /// is a linestring
     bool isClosed(void)
     {
-        if (refs.size() > 0) {
-            if (refs[0] == refs[refs.size() - 1]) {
-                return true;
-            }
-        }
-        return false;
+        return !linestring.empty() && boost::geometry::equals(linestring.back(), linestring.front());
     };
     /// Return the number of nodes in this way
     int numPoints(void) { return boost::geometry::num_points(linestring); };

diff --git a/src/raw/queryraw.cc b/src/raw/queryraw.cc
@@ -436,7 +436,7 @@ QueryRaw::getWaysFromDB(int lastid, const std::string &tableName) {
         waysQuery = "SELECT osm_id, refs, ST_AsText(geom, 4326)";
     }
     waysQuery += ", version, tags FROM " + tableName + " where osm_id > " + std::to_string(lastid) + " order by osm_id asc limit 500;";
-    
+
     auto ways_result = dbconn->query(waysQuery);
     // Fill vector of OsmWay objects
     auto ways = std::make_shared<std::vector<OsmWay>>();
@@ -469,6 +469,45 @@ QueryRaw::getWaysFromDB(int lastid, const std::string &tableName) {
     return ways;
 }
 
+std::shared_ptr<std::vector<OsmWay>>
+QueryRaw::getWaysFromDBWithoutRefs(int lastid, const std::string &tableName) {
+    std::string waysQuery;
+    if (tableName == QueryRaw::polyTable) {
+        waysQuery = "SELECT osm_id, ST_AsText(ST_ExteriorRing(geom), 4326)";
+    } else {
+        waysQuery = "SELECT osm_id, ST_AsText(geom, 4326)";
+    }
+    waysQuery += ", version, tags FROM " + tableName + " where osm_id > " + std::to_string(lastid) + " order by osm_id asc limit 500;";
+
+    auto ways_result = dbconn->query(waysQuery);
+    // Fill vector of OsmWay objects
+    auto ways = std::make_shared<std::vector<OsmWay>>();
+    for (auto way_it = ways_result.begin(); way_it != ways_result.end(); ++way_it) {
+        OsmWay way;
+        way.id = (*way_it)[0].as<long>();
+
+        std::string poly = (*way_it)[1].as<std::string>();
+        boost::geometry::read_wkt(poly, way.linestring);
+
+        if (tableName == QueryRaw::polyTable) {
+            way.polygon = { {std::begin(way.linestring), std::end(way.linestring)} };
+        }
+        way.version = (*way_it)[2].as<long>();
+        auto tags = (*way_it)[3];
+        if (!tags.is_null()) {
+            auto tags = parseTagsString((*way_it)[3].as<std::string>());
+            for (auto const& [key, val] : tags)
+            {
+                way.addTag(key, val);
+            }
+        }
+        ways->push_back(way);
+
+    }
+
+    return ways;
+}
+
 } // namespace queryraw
 
 // local Variables:

diff --git a/src/raw/queryraw.hh b/src/raw/queryraw.hh
@@ -77,6 +77,7 @@ class QueryRaw {
     int getWaysCount(const std::string &tableName);
     // Get ways by page
     std::shared_ptr<std::vector<OsmWay>> getWaysFromDB(int page, const std::string &tableName);
+    std::shared_ptr<std::vector<OsmWay>> getWaysFromDBWithoutRefs(int lastid, const std::string &tableName);
 
 };
 

diff --git a/src/underpass.cc b/src/underpass.cc
@@ -37,7 +37,6 @@
 #include <tuple>
 #include <vector>
 
-#include "underpassconfig.hh"
 #include "replicator/planetreplicator.hh"
 #include "boost/date_time/posix_time/posix_time.hpp"
 #include <boost/date_time.hpp>
@@ -96,9 +95,9 @@ main(int argc, char *argv[])
 
     opts::positional_options_description p;
     opts::variables_map vm;
+    opts::options_description desc("Allowed options");
 
     try {
-        opts::options_description desc("Allowed options");
         // clang-format off
         desc.add_options()
             ("help,h", "display help")
@@ -124,33 +123,22 @@ main(int argc, char *argv[])
             ("disable-stats", "Disable statistics")
             ("disable-validation", "Disable validation")
             ("disable-raw", "Disable raw OSM data")
+            ("norefs", "Disable refs (useful for non OSM data)")
             ("bootstrap", "Bootstrap data tables");
         // clang-format on
 
         opts::store(opts::command_line_parser(argc, argv).options(desc).positional(p).run(), vm);
         opts::notify(vm);
 
-        if (vm.count("help")) {
-
-            std::cout << "Usage: options_description [options]" << std::endl;
-            std::cout << desc << std::endl;
-            std::cout << "A few configuration options can be set through the "
-                         "environment,"
-                      << std::endl;
-            std::cout << "this is the current status of the configuration "
-                         "options with"
-                      << std::endl;
-            std::cout << "the environment variable names and their current "
-                         "values (possibly defaults)."
-                      << std::endl;
-            std::cout << config.dbConfigHelp() << std::endl;
-            return 0;
-        }
     } catch (std::exception &e) {
         std::cout << e.what() << std::endl;
         return 1;
     }
 
+    if (vm.count("norefs")) {
+        config.norefs = true;
+    }
+
     // Logging
     logger::LogFile &dbglogfile = logger::LogFile::getDefaultInstance();
     if (vm.count("verbose")) {
@@ -185,7 +173,7 @@ main(int argc, char *argv[])
         config.concurrency = std::thread::hardware_concurrency();
     }
 
-    if (!vm.count("bootstrap")) {
+    if (vm.count("timestamp") || vm.count("url")) {
         // Planet server
         if (vm.count("planet")) {
             config.planet_server = vm["planet"].as<std::string>();
@@ -330,15 +318,33 @@ main(int argc, char *argv[])
 
         exit(0);
 
-    } else {
+    }
+
+    if (vm.count("bootstrap")){
         std::thread bootstrapThread;
+        std::cout << "Starting bootstrapping proccess ..." << std::endl;
         bootstrapThread = std::thread(bootstrap::startProcessingWays, std::ref(config));
         log_info("Waiting...");
         if (bootstrapThread.joinable()) {
             bootstrapThread.join();
         }
+        exit(0);
     }
 
+    std::cout << "Usage: options_description [options]" << std::endl;
+    std::cout << desc << std::endl;
+    std::cout << "A few configuration options can be set through the "
+                    "environment,"
+                << std::endl;
+    std::cout << "this is the current status of the configuration "
+                    "options with"
+                << std::endl;
+    std::cout << "the environment variable names and their current "
+                    "values (possibly defaults)."
+                << std::endl;
+    // std::cout << config.dbConfigHelp() << std::endl;
+    exit(0);
+
 }
 
 // Local Variables:

diff --git a/src/underpassconfig.hh b/src/underpassconfig.hh
@@ -157,6 +157,7 @@ struct UnderpassConfig {
     bool disable_validation = false;
     bool disable_stats = false;
     bool disable_raw = false;
+    bool norefs = false;
 
     ///
     /// \brief getPlanetServer returns either the command line supplied planet server