GlobalDescriptor (PyDescriptor / netvlad) (#1163)

* First commit for pydescriptor

* Fixed Python refactoring errors

* GUI: added PyDescriptor parameters

* reordered python3 includes

* updated rtabmap_netvlad.py test main

* fixed build from last merge

* integrated #1255

* rescaled dot product result

Closing #1105

corelib/include/rtabmap/core/GlobalDescriptor.h

@@ -1,5 +1,5 @@
 /*
-Copyright (c) 2010-2020, Mathieu Labbe - IntRoLab - Universite de Sherbrooke
+Copyright (c) 2010-2024, Mathieu Labbe - IntRoLab - Universite de Sherbrooke
 All rights reserved.
 
 Redistribution and use in source and binary forms, with or without

corelib/include/rtabmap/core/GlobalDescriptorExtractor.h

@@ -0,0 +1,73 @@
+/*
+Copyright (c) 2010-2024, Mathieu Labbe - IntRoLab - Universite de Sherbrooke
+All rights reserved.
+
+Redistribution and use in source and binary forms, with or without
+modification, are permitted provided that the following conditions are met:
+ * Redistributions of source code must retain the above copyright
+ notice, this list of conditions and the following disclaimer.
+ * Redistributions in binary form must reproduce the above copyright
+ notice, this list of conditions and the following disclaimer in the
+ documentation and/or other materials provided with the distribution.
+ * Neither the name of the Universite de Sherbrooke nor the
+ names of its contributors may be used to endorse or promote products
+ derived from this software without specific prior written permission.
+
+THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND
+ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
+WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
+DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY
+DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
+(INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
+LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND
+ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
+SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+*/
+
+#ifndef GLOBAL_DESCRIPTOR_EXTRACTOR_H_
+#define GLOBAL_DESCRIPTOR_EXTRACTOR_H_
+
+#include "rtabmap/core/rtabmap_core_export.h" // DLL export/import defines
+
+#include "rtabmap/core/Parameters.h"
+#include "rtabmap/core/SensorData.h"
+
+
+namespace rtabmap {
+
+// Feature2D
+class RTABMAP_CORE_EXPORT GlobalDescriptorExtractor {
+public:
+ enum Type {
+ kUndef=0,
+ kPyDescriptor=1};
+
+ static std::string typeName(Type type)
+ {
+ switch(type){
+ case kPyDescriptor:
+ return "PyDescriptor";
+ default:
+ return "Unknown";
+ }
+ }
+
+ static GlobalDescriptorExtractor * create(const ParametersMap & parameters = ParametersMap());
+ static GlobalDescriptorExtractor * create(GlobalDescriptorExtractor::Type type, const ParametersMap & parameters = ParametersMap()); // for convenience
+
+public:
+ virtual ~GlobalDescriptorExtractor();
+
+ virtual GlobalDescriptor extract(const SensorData & data) const = 0;
+
+ virtual void parseParameters(const ParametersMap & parameters) {}
+ virtual GlobalDescriptorExtractor::Type getType() const = 0;
+
+protected:
+ GlobalDescriptorExtractor(const ParametersMap & parameters = ParametersMap());
+};
+
+}
+
+#endif /* GLOBAL_DESCRIPTOR_EXTRACTOR_H_ */

corelib/include/rtabmap/core/Memory.h

@@ -59,6 +59,7 @@ class RegistrationVis;
 class Stereo;
 class LocalGridMaker;
 class MarkerDetector;
+class GlobalDescriptorExtractor;
 
 class RTABMAP_CORE_EXPORT Memory
 {
@@ -375,6 +376,8 @@ class RTABMAP_CORE_EXPORT Memory
  LocalGridMaker * _localMapMaker;
 
  MarkerDetector * _markerDetector;
+
+ GlobalDescriptorExtractor * _globalDescriptorExtractor;
 };
 
 } // namespace rtabmap

corelib/include/rtabmap/core/Parameters.h

@@ -231,6 +231,7 @@ class RTABMAP_CORE_EXPORT Parameters
  RTABMAP_PARAM(Mem, UseOdomFeatures, bool, true, "Use odometry features instead of regenerating them.");
  RTABMAP_PARAM(Mem, UseOdomGravity, bool, false, uFormat("Use odometry instead of IMU orientation to add gravity links to new nodes created. We assume that odometry is already aligned with gravity (e.g., we are using a VIO approach). Gravity constraints are used by graph optimization only if \"%s\" is not zero.", kOptimizerGravitySigma().c_str()));
  RTABMAP_PARAM(Mem, CovOffDiagIgnored, bool, true, "Ignore off diagonal values of the covariance matrix.");
+ RTABMAP_PARAM(Mem, GlobalDescriptorStrategy, int, 0, "Extract global descriptor from sensor data. 0=disabled, 1=PyDescriptor");
  RTABMAP_PARAM(Mem, RotateImagesUpsideUp, bool, false, "Rotate images so that upside is up if they are not already. This can be useful in case the robots don't have all same camera orientation but are using the same map, so that not rotation-invariant visual features can still be used across the fleet.");
 
  // KeypointMemory (Keypoint-based)
@@ -725,6 +726,10 @@ class RTABMAP_CORE_EXPORT Parameters
  RTABMAP_PARAM(GMS, WithScale, bool, false, "Take scale transformation into account.");
  RTABMAP_PARAM(GMS, ThresholdFactor, double, 6.0, "The higher, the less matches.");
 
+ // Global descriptor approaches
+ RTABMAP_PARAM_STR(PyDescriptor, Path, "", "Path to python script file (see available ones in rtabmap/corelib/src/pydescriptor/*). See the header to see where the script should be used.");
+ RTABMAP_PARAM(PyDescriptor, Dim, int, 4096, "Descriptor dimension.");
+
  // ICP registration parameters
 #ifdef RTABMAP_POINTMATCHER
  RTABMAP_PARAM(Icp, Strategy, int, 1, "ICP implementation: 0=Point Cloud Library, 1=libpointmatcher, 2=CCCoreLib (CloudCompare).");

corelib/src/CMakeLists.txt

@@ -115,6 +115,8 @@ SET(SRC_FILES
 
  MarkerDetector.cpp
 
+ GlobalDescriptorExtractor.cpp
+
  GainCompensator.cpp
 
  rtflann/ext/lz4.c
@@ -224,6 +226,7 @@ IF(WITH_PYTHON AND Python3_FOUND)
  python/PythonInterface.cpp
  python/PyMatcher.cpp
  python/PyDetector.cpp
+ python/PyDescriptor.cpp
  )
  SET(INCLUDE_DIRS
  ${TORCH_INCLUDE_DIRS}

corelib/src/GlobalDescriptorExtractor.cpp

@@ -0,0 +1,76 @@
+/*
+Copyright (c) 2010-2024, Mathieu Labbe - IntRoLab - Universite de Sherbrooke
+All rights reserved.
+
+Redistribution and use in source and binary forms, with or without
+modification, are permitted provided that the following conditions are met:
+ * Redistributions of source code must retain the above copyright
+ notice, this list of conditions and the following disclaimer.
+ * Redistributions in binary form must reproduce the above copyright
+ notice, this list of conditions and the following disclaimer in the
+ documentation and/or other materials provided with the distribution.
+ * Neither the name of the Universite de Sherbrooke nor the
+ names of its contributors may be used to endorse or promote products
+ derived from this software without specific prior written permission.
+
+THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND
+ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
+WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
+DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY
+DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
+(INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
+LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND
+ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
+SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+*/
+
+
+#include "rtabmap/core/GlobalDescriptorExtractor.h"
+
+#ifdef RTABMAP_PYTHON
+#include "python/PyDescriptor.h"
+#endif
+
+namespace rtabmap {
+
+GlobalDescriptorExtractor::GlobalDescriptorExtractor(const ParametersMap & parameters)
+{
+}
+GlobalDescriptorExtractor::~GlobalDescriptorExtractor()
+{
+}
+GlobalDescriptorExtractor * GlobalDescriptorExtractor::create(const ParametersMap & parameters)
+{
+ int type = Parameters::defaultMemGlobalDescriptorStrategy();
+ Parameters::parse(parameters, Parameters::kMemGlobalDescriptorStrategy(), type);
+ return create((GlobalDescriptorExtractor::Type)type, parameters);
+}
+GlobalDescriptorExtractor * GlobalDescriptorExtractor::create(GlobalDescriptorExtractor::Type type, const ParametersMap & parameters)
+{
+ UDEBUG("Creating global descriptor of type %d", (int)type);
+#ifndef RTABMAP_PYTHON
+ if(type == GlobalDescriptorExtractor::kPyDescriptor)
+ {
+ UWARN("PyDescriptor cannot be used as rtabmap is not built with Python3 support.");
+ type = GlobalDescriptorExtractor::kUndef;
+ }
+#endif
+
+ GlobalDescriptorExtractor * GlobalDescriptorExtractor = 0;
+ switch(type)
+ {
+#ifdef RTABMAP_PYTHON
+ case GlobalDescriptorExtractor::kPyDescriptor:
+ GlobalDescriptorExtractor = new PyDescriptor(parameters);
+ break;
+#endif
+ default:
+ type = GlobalDescriptorExtractor::kUndef;
+ break;
+ }
+ return GlobalDescriptorExtractor;
+}
+
+}
+

corelib/src/Memory.cpp

@@ -40,6 +40,7 @@ SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 #include <rtabmap/core/EpipolarGeometry.h>
 #include "rtabmap/core/VisualWord.h"
 #include "rtabmap/core/Features2d.h"
+#include "rtabmap/core/GlobalDescriptorExtractor.h"
 #include "rtabmap/core/RegistrationIcp.h"
 #include "rtabmap/core/Registration.h"
 #include "rtabmap/core/RegistrationVis.h"
@@ -132,6 +133,7 @@ Memory::Memory(const ParametersMap & parameters) :
  _feature2D = Feature2D::create(parameters);
  _vwd = new VWDictionary(parameters);
  _registrationPipeline = Registration::create(parameters);
+ _globalDescriptorExtractor = GlobalDescriptorExtractor::create(parameters);
  if(!_registrationPipeline->isImageRequired())
  {
  // make sure feature matching is used instead of optical flow to compute the guess
@@ -761,6 +763,22 @@ void Memory::parseParameters(const ParametersMap & parameters)
  _markerDetector->parseParameters(params);
  }
 
+ int globalDescriptorStrategy = -1;
+ Parameters::parse(params, Parameters::kMemGlobalDescriptorStrategy(), globalDescriptorStrategy);
+ if(globalDescriptorStrategy != -1 &&
+ (_globalDescriptorExtractor==0 || (int)_globalDescriptorExtractor->getType() != globalDescriptorStrategy))
+ {
+ if(_globalDescriptorExtractor)
+ {
+ delete _globalDescriptorExtractor;
+ }
+ _globalDescriptorExtractor = GlobalDescriptorExtractor::create(parameters_);
+ }
+ else if(_globalDescriptorExtractor)
+ {
+ _globalDescriptorExtractor->parseParameters(params);
+ }
+
  // do this after all params are parsed
  // SLAM mode vs Localization mode
  iter = params.find(Parameters::kMemIncrementalMemory());
@@ -5934,7 +5952,17 @@ Signature * Memory::createSignature(const SensorData & inputData, const Transfor
  s->sensorData().setGroundTruth(data.groundTruth());
  s->sensorData().setGPS(data.gps());
  s->sensorData().setEnvSensors(data.envSensors());
- s->sensorData().setGlobalDescriptors(data.globalDescriptors());
+
+ std::vector<GlobalDescriptor> globalDescriptors = data.globalDescriptors();
+ if(_globalDescriptorExtractor)
+ {
+ GlobalDescriptor gdescriptor = _globalDescriptorExtractor->extract(inputData);
+ if(!gdescriptor.data().empty())
+ {
+ globalDescriptors.push_back(gdescriptor);
+ }
+ }
+ s->sensorData().setGlobalDescriptors(globalDescriptors);
 
  t = timer.ticks();
  if(stats) stats->addStatistic(Statistics::kTimingMemCompressing_data(), t*1000.0f);

corelib/src/Signature.cpp

@@ -249,17 +249,40 @@ void Signature::removeLandmark(int landmarkId)
 
 float Signature::compareTo(const Signature & s) const
 {
+ UASSERT(this->sensorData().globalDescriptors().size() == s.sensorData().globalDescriptors().size());
+
  float similarity = 0.0f;
- const std::multimap<int, int> & words = s.getWords();
+ int totalDescs = 0;
+
+ for(size_t i=0; i<this->sensorData().globalDescriptors().size(); ++i)
+ {
+ if(this->sensorData().globalDescriptors()[i].type()==1 && s.sensorData().globalDescriptors()[i].type()==1)
+ {
+ // rescale dot product from -1<->1 to 0<->1 (we assume normalized vectors!)
+ float dotProd = (this->sensorData().globalDescriptors()[i].data().dot(s.sensorData().globalDescriptors()[i].data()) + 1.0f) / 2.0f;
+ UASSERT_MSG(dotProd>=0, "Global descriptors should be normalized!");
+ similarity += dotProd;
+ totalDescs += 1;
+ }
+ }
 
- if(!s.isBadSignature() && !this->isBadSignature())
+ if(totalDescs)
  {
- std::list<std::pair<int, std::pair<int, int> > > pairs;
- int totalWords = ((int)_words.size()-_invalidWordsCount)>((int)words.size()-s.getInvalidWordsCount())?((int)_words.size()-_invalidWordsCount):((int)words.size()-s.getInvalidWordsCount());
- UASSERT(totalWords > 0);
- EpipolarGeometry::findPairs(words, _words, pairs);
+ similarity /= totalDescs;
+ }
+ else
+ {
+ const std::multimap<int, int> & words = s.getWords();
 
- similarity = float(pairs.size()) / float(totalWords);
+ if(!s.isBadSignature() && !this->isBadSignature())
+ {
+ std::list<std::pair<int, std::pair<int, int> > > pairs;
+ int totalWords = ((int)_words.size()-_invalidWordsCount)>((int)words.size()-s.getInvalidWordsCount())?((int)_words.size()-_invalidWordsCount):((int)words.size()-s.getInvalidWordsCount());
+ UASSERT(totalWords > 0);
+ EpipolarGeometry::findPairs(words, _words, pairs);
+
+ similarity = float(pairs.size()) / float(totalWords);
+ }
  }
  return similarity;
 }