From 83254ab16da18137b8af0e254ab919dd77d181fc Mon Sep 17 00:00:00 2001 From: Khai Yi Chin Date: Fri, 1 Nov 2024 01:27:17 -0400 Subject: [PATCH] Completed documentation. 1. Cleaned up versioning and copyright tags. 2. Added homepage for API documentation. 3. Added documentation for Optimizer classes. --- CMakeLists.txt | 3 + README.md | 5 +- doc/Doxyfile.in | 9 +-- doc/instructions.md | 2 +- include/Optimizer | 10 ++++ include/SVGDCpp/Core.hpp | 4 +- include/SVGDCpp/Exceptions.hpp | 16 ++++-- include/SVGDCpp/Kernel/GaussianRBFKernel.hpp | 4 +- include/SVGDCpp/Kernel/Kernel.hpp | 8 +-- include/SVGDCpp/Model/Model.hpp | 6 +- include/SVGDCpp/Model/MultivariateNormal.hpp | 4 +- include/SVGDCpp/Optimizer/AdaGrad.hpp | 43 +++++++++++++-- include/SVGDCpp/Optimizer/Adam.hpp | 58 +++++++++++++++++--- include/SVGDCpp/Optimizer/Optimizer.hpp | 32 ++++++++++- include/SVGDCpp/Optimizer/RMSProp.hpp | 44 +++++++++++++-- include/SVGDCpp/SVGD.hpp | 10 ++-- 16 files changed, 205 insertions(+), 53 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index 9d159b3..596f3d6 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -5,6 +5,9 @@ add_compile_options(-Wall -Wextra -Wpedantic) set(CMAKE_CXX_STANDARD 17) +# Define current version +set(SVGDCPP_VERSION "1.0.0") + # Define variables set(SVGDCPP_CMAKE_INSTALL_DIR "${CMAKE_INSTALL_PREFIX}/share/${PROJECT_NAME}/cmake") set(SVGDCPP_INCLUDE_INSTALL_DIR "${CMAKE_INSTALL_PREFIX}/include/${PROJECT_NAME}") # assign include destination to variable diff --git a/README.md b/README.md index 3e83311..8c22689 100644 --- a/README.md +++ b/README.md @@ -10,7 +10,8 @@ This library provides the [Stein Variational Gradient Descent](https://arxiv.org - `-D include_eigen=true` - `-D CMAKE_BUILD_TYPE=Release` - [OpenMP v4.5+ (201511)](https://www.openmp.org/) - _should be shipped with your compiler_ -- [Doxygen](https://www.doxygen.nl/) - _required only if documentation is desired_ +- [GraphViz](https://graphviz.org/) - _required only if documentation is desired; can be installed using `apt install graphviz`_ +- [Doxygen](https://www.doxygen.nl/) - _required only if documentation is desired; can be installed using `apt install doxygen`_ ## Installation 1. Clone this repository and enter it. @@ -32,7 +33,7 @@ This library provides the [Stein Variational Gradient Descent](https://arxiv.org ``` ## Getting Started -See the [examples directory](examples/) for tutorials on how to use them and see [here](doc/instructions.md) for detailed instructions. +See the [examples directory](./examples/) for tutorials on how to use them and see [here](./doc/instructions.md) for detailed instructions. ## Tests Unit tests have been provided to aid source code development. Besides identifying the kinds of testing imposed on the source code, looking into the test files can help you understand how the algorithm works. All you need to do is build the code with `-D CMAKE_BUILD_TYPE=Debug` and then run the tests either using `CTest`: diff --git a/doc/Doxyfile.in b/doc/Doxyfile.in index 716cad7..f121782 100644 --- a/doc/Doxyfile.in +++ b/doc/Doxyfile.in @@ -38,7 +38,7 @@ PROJECT_NAME = "SVGDCpp" # could be handy for archiving the generated documentation or if some version # control system is used. -PROJECT_NUMBER = "0.1" +PROJECT_NUMBER = @SVGDCPP_VERSION@ # Using the PROJECT_BRIEF tag one can provide an optional one line description # for a project that appears at the top of each page and should give viewer a @@ -830,7 +830,8 @@ WARN_LOGFILE = # spaces. See also FILE_PATTERNS and EXTENSION_MAPPING # Note: If this tag is empty the current directory is searched. -INPUT = @CMAKE_SOURCE_DIR@/include/ +INPUT = @CMAKE_SOURCE_DIR@/include/ \ + ../README.md # This tag can be used to specify the character encoding of the source files # that doxygen parses. Internally doxygen uses the UTF-8 encoding. Doxygen uses @@ -982,7 +983,7 @@ FILTER_SOURCE_PATTERNS = # (index.html). This can be useful if you have a project on for instance GitHub # and want to reuse the introduction page also for the doxygen output. -USE_MDFILE_AS_MAINPAGE = +USE_MDFILE_AS_MAINPAGE = ../README.md #--------------------------------------------------------------------------- # Configuration options related to source browsing @@ -1251,7 +1252,7 @@ HTML_COLORSTYLE_GAMMA = 80 # The default value is: NO. # This tag requires that the tag GENERATE_HTML is set to YES. -HTML_TIMESTAMP = NO +HTML_TIMESTAMP = YES # If the HTML_DYNAMIC_MENUS tag is set to YES then the generated HTML # documentation will contain a main index with vertical navigation menus that diff --git a/doc/instructions.md b/doc/instructions.md index baf9728..ef5e2b7 100644 --- a/doc/instructions.md +++ b/doc/instructions.md @@ -30,7 +30,7 @@ Mainly, there are 5 elements required to run a minimally working SVGD algorithm: 3. `Optimizer` object; 4. `SVGD` object. -The first four are required by the fifth as shared pointer arguments. +The first four are required by the fifth as shared pointer arguments. The `Model` and `Kernel` classes are used to compute $\hat{\phi}^*$ in Equation (8) of the [SVGD paper](https://arxiv.org/abs/1608.04471), and the step size $\epsilon$ is provided with the `Optimizer` class. ### Basic usage This example uses the provided classes `MultivariateNormal`, `GaussianRBF`, and `Adam` optimizer to estimate a multivariate normal distribution. diff --git a/include/Optimizer b/include/Optimizer index 96dbb17..50b6c8f 100644 --- a/include/Optimizer +++ b/include/Optimizer @@ -1,3 +1,13 @@ +/** + * @defgroup Optimizer_Module Optimizer Module + * @brief Provides functionality to define optimizer functions. + * @details This module provides the base and derived @ref Optimizer classes. + * + * @code{.cpp} + * #include + * @endcode + */ + #ifndef SVGDCPP_OPTIMIZER_MODULE_HPP #define SVGDCPP_OPTIMIZER_MODULE_HPP diff --git a/include/SVGDCpp/Core.hpp b/include/SVGDCpp/Core.hpp index f343a17..cc20a3d 100644 --- a/include/SVGDCpp/Core.hpp +++ b/include/SVGDCpp/Core.hpp @@ -2,10 +2,8 @@ * @file Core.hpp * @author Khai Yi Chin (khaiyichin@gmail.com) * @brief Core header to provide common types and functions. - * @version 0.1 - * @date 2024-03-23 * - * @copyright Copyright (c) 2024 + * @copyright Copyright (c) 2024 Khai Yi Chin * */ diff --git a/include/SVGDCpp/Exceptions.hpp b/include/SVGDCpp/Exceptions.hpp index 552cff7..d86f050 100644 --- a/include/SVGDCpp/Exceptions.hpp +++ b/include/SVGDCpp/Exceptions.hpp @@ -2,10 +2,8 @@ * @file Exceptions.hpp * @author Khai Yi Chin (khaiyichin@gmail.com) * @brief Exceptions header to provide some commonly used custom exceptions. - * @version 0.1 - * @date 2024-10-16 * - * @copyright Copyright (c) 2024 + * @copyright Copyright (c) 2024 Khai Yi Chin * */ @@ -15,8 +13,13 @@ #include #include -#define SVGDCPP_LOG_PREFIX std::string("SVGDCpp: ") +#define SVGDCPP_LOG_PREFIX std::string("SVGDCpp: ") ///< Convenience macro to prefix output logs. +/** + * @class DimensionMismatchException + * @brief Exception for dimension mismatch type errors. + * @ingroup Core_Module + */ class DimensionMismatchException : public std::exception { public: @@ -32,6 +35,11 @@ class DimensionMismatchException : public std::exception std::string message_; }; +/** + * @class UnsetException + * @brief Exception for unset value type errors. + * @ingroup Core_Module + */ class UnsetException : public std::exception { public: diff --git a/include/SVGDCpp/Kernel/GaussianRBFKernel.hpp b/include/SVGDCpp/Kernel/GaussianRBFKernel.hpp index 81036c9..da68718 100644 --- a/include/SVGDCpp/Kernel/GaussianRBFKernel.hpp +++ b/include/SVGDCpp/Kernel/GaussianRBFKernel.hpp @@ -2,10 +2,8 @@ * @file GaussianRBFKernel.hpp * @author Khai Yi Chin (khaiyichin@gmail.com) * @brief Gaussian RBF kernel class header. - * @version 0.1 - * @date 2024-03-23 * - * @copyright Copyright (c) 2024 + * @copyright Copyright (c) 2024 Khai Yi Chin * */ diff --git a/include/SVGDCpp/Kernel/Kernel.hpp b/include/SVGDCpp/Kernel/Kernel.hpp index 1095ec3..353236f 100644 --- a/include/SVGDCpp/Kernel/Kernel.hpp +++ b/include/SVGDCpp/Kernel/Kernel.hpp @@ -1,11 +1,9 @@ /** * @file Kernel.hpp * @author Khai Yi Chin (khaiyichin@gmail.com) - * @brief Kernel class header - * @version 0.1 - * @date 2024-03-23 + * @brief Kernel class header. * - * @copyright Copyright (c) 2024 + * @copyright Copyright (c) 2024 Khai Yi Chin * */ #ifndef SVGDCPP_KERNEL_HPP @@ -350,7 +348,7 @@ class Kernel } /** - * @brief Execute methods required for each step. + * @brief Execute methods required at each step. * @details Override this method to include methods that you need to have run every step of the iteration * @a e.g., computing the scale parameter of the kernel function. * diff --git a/include/SVGDCpp/Model/Model.hpp b/include/SVGDCpp/Model/Model.hpp index 4b479ea..dd41dbb 100644 --- a/include/SVGDCpp/Model/Model.hpp +++ b/include/SVGDCpp/Model/Model.hpp @@ -2,10 +2,8 @@ * @file Model.hpp * @author Khai Yi Chin (khaiyichin@gmail.com) * @brief Model class header - * @version 0.1 - * @date 2024-03-22 * - * @copyright Copyright (c) 2024 + * @copyright Copyright (c) 2024 Khai Yi Chin * */ @@ -408,7 +406,7 @@ class Model } /** - * @brief Execute methods required for each step. + * @brief Execute methods required at each step. * @details Override this method to include methods that you need to have run every step of the iteration. * */ diff --git a/include/SVGDCpp/Model/MultivariateNormal.hpp b/include/SVGDCpp/Model/MultivariateNormal.hpp index 03e445a..682f563 100644 --- a/include/SVGDCpp/Model/MultivariateNormal.hpp +++ b/include/SVGDCpp/Model/MultivariateNormal.hpp @@ -2,10 +2,8 @@ * @file MultivariateNormal.hpp * @author Khai Yi Chin (khaiyichin@gmail.com) * @brief Multivariate normal model class header. - * @version 0.1 - * @date 2024-03-22 * - * @copyright Copyright (c) 2024 + * @copyright Copyright (c) 2024 Khai Yi Chin * */ diff --git a/include/SVGDCpp/Optimizer/AdaGrad.hpp b/include/SVGDCpp/Optimizer/AdaGrad.hpp index b087305..13fa05d 100644 --- a/include/SVGDCpp/Optimizer/AdaGrad.hpp +++ b/include/SVGDCpp/Optimizer/AdaGrad.hpp @@ -1,12 +1,33 @@ +/** + * @file AdaGrad.hpp + * @author Khai Yi Chin (khaiyichin@gmail.com) + * @brief AdaGrad optimizer class header. + * + * @copyright Copyright (c) 2024 Khai Yi Chin + * + */ #ifndef SVGDCPP_ADAGRAD_HPP #define SVGDCPP_ADAGRAD_HPP #include "../Core.hpp" #include "Optimizer.hpp" +/** + * @class AdaGrad + * @brief This class provides the AdaGrad optimizer. + * @ingroup Optimizer_Module + */ class AdaGrad : public Optimizer { public: + /** + * @brief Default constructor. + * + * @param dimension Dimension of the problem. + * @param num_particles Number of particles used in the problem. + * @param lr Learning rate. + * @param epsilon Numerical stabilizer. + */ AdaGrad(const size_t &dimension, const size_t &num_particles, const double &lr, @@ -15,13 +36,27 @@ class AdaGrad : public Optimizer dimension_(dimension), num_particles_(num_particles) {} + /** + * @brief Default destructor. + * + */ virtual ~AdaGrad() {} + /** + * @brief Initialize the optimizer. + * + */ virtual void Initialize() override { sum_of_sq_grad_ = Eigen::MatrixXd::Zero(dimension_, num_particles_).array(); } + /** + * @brief Execute the optimization of the gradient at each step. + * + * @param grad_matrix Gradient matrix for each particle coordinate. + * @return Optimized gradient matrix. + */ virtual Eigen::MatrixXd Step(const Eigen::MatrixXd &grad_matrix) { sum_of_sq_grad_ += grad_matrix.array().square(); @@ -30,13 +65,13 @@ class AdaGrad : public Optimizer } protected: - size_t dimension_; + size_t dimension_; ///< Dimension of the problem. - size_t num_particles_; + size_t num_particles_; ///< Number of particles used in the problem. - double decay_rate_; + double decay_rate_; ///< Decay rate (unused). - Eigen::ArrayXXd sum_of_sq_grad_; + Eigen::ArrayXXd sum_of_sq_grad_; ///< Sum of squares gradient matrix. }; #endif \ No newline at end of file diff --git a/include/SVGDCpp/Optimizer/Adam.hpp b/include/SVGDCpp/Optimizer/Adam.hpp index dab7c46..5506c60 100644 --- a/include/SVGDCpp/Optimizer/Adam.hpp +++ b/include/SVGDCpp/Optimizer/Adam.hpp @@ -1,12 +1,35 @@ +/** + * @file Adam.hpp + * @author Khai Yi Chin (khaiyichin@gmail.com) + * @brief Adam optimizer class header. + * + * @copyright Copyright (c) 2024 Khai Yi Chin + * + */ #ifndef SVGDCPP_ADAM_HPP #define SVGDCPP_ADAM_HPP #include "../Core.hpp" #include "Optimizer.hpp" +/** + * @class Adam + * @brief This class provides the Adam optimizer. + * @ingroup Optimizer_Module + */ class Adam : public Optimizer { public: + /** + * @brief Default constructor. + * + * @param dimension Dimension of the problem. + * @param num_particles Number of particles used in the problem. + * @param lr Learning rate. + * @param beta1 Exponential decay rate for the 1st moment estimates. + * @param beta2 Exponential decay rate for the 2nd moment estimates. + * @param epsilon Numerical stabilizer. + */ Adam(const size_t &dimension, const size_t &num_particles, const double &lr, @@ -25,8 +48,16 @@ class Adam : public Optimizer } } + /** + * @brief Default destructor. + * + */ virtual ~Adam() {} + /** + * @brief Initialize the optimizer. + * + */ virtual void Initialize() override { sum_of_sq_grad_ = Eigen::MatrixXd::Zero(dimension_, num_particles_).array(); @@ -35,6 +66,12 @@ class Adam : public Optimizer counter_ = 0; } + /** + * @brief Execute the optimization of the gradient at each step. + * + * @param grad_matrix Gradient matrix for each particle coordiniate. + * @return Optimized gradient matrix. + */ virtual Eigen::MatrixXd Step(const Eigen::MatrixXd &grad_matrix) { sum_of_grad_ = decay_rate_1_ * sum_of_grad_ + (1 - decay_rate_1_) * grad_matrix.array(); // Momentum part @@ -46,24 +83,31 @@ class Adam : public Optimizer } protected: + /** + * @brief Correct moment estimate for bias. + * + * @param arr Biased gradient estimate array. + * @param decay Decay rate. + * @return Bias-corrected moment estimate. + */ Eigen::ArrayXXd CorrectForBias(const Eigen::ArrayXXd arr, const double &decay) { return arr / (1.0 - std::pow(decay, counter_)); } - size_t counter_ = 0; + size_t counter_ = 0; ///< Time step counter. - size_t dimension_; + size_t dimension_; ///< Dimension of the problem. - size_t num_particles_; + size_t num_particles_; ///< Number of particles used in the problem. - double decay_rate_1_; + double decay_rate_1_; ///< Exponential decay rate for the 1st moment estimate. - double decay_rate_2_; + double decay_rate_2_; ///< Exponential decay rate for the 2nd moment estimate. - Eigen::ArrayXXd sum_of_sq_grad_; + Eigen::ArrayXXd sum_of_sq_grad_; ///< 2nd biased moment estimate. - Eigen::ArrayXXd sum_of_grad_; + Eigen::ArrayXXd sum_of_grad_; ///< 1st biased moment estimate }; #endif \ No newline at end of file diff --git a/include/SVGDCpp/Optimizer/Optimizer.hpp b/include/SVGDCpp/Optimizer/Optimizer.hpp index 2b2f6b6..258e8da 100644 --- a/include/SVGDCpp/Optimizer/Optimizer.hpp +++ b/include/SVGDCpp/Optimizer/Optimizer.hpp @@ -1,24 +1,50 @@ +/** + * @file Optimizer.hpp + * @author Khai Yi Chin (khaiyichin@gmail.com) + * @brief Optimizer class header + * + * @copyright Copyright (c) 2024 Khai Yi Chin + * + */ #ifndef SVGDCPP_OPTIMIZER_HPP #define SVGDCPP_OPTIMIZER_HPP #include "../Core.hpp" +/** + * @class Optimizer + * @brief This is used as a template to derive optimizer classes. Specifically, the `Step` function needs to be defined. + * @ingroup Optimizer_Module + */ class Optimizer { public: + /** + * @brief Default constructor. + * + * @param lr Learning rate. + * @param epsilon Numerical stabilizer. + */ Optimizer(const double &lr, const double &epsilon = 1.0e-8) : learning_rate_(lr), stabilizer_(epsilon) {} + /** + * @brief Default destructor. + * + */ virtual ~Optimizer() {} + /** + * @brief Initialize the optimizer. + * + */ virtual void Initialize() = 0; virtual Eigen::MatrixXd Step(const Eigen::MatrixXd &grad_matrix) = 0; protected: + double learning_rate_; ///< Learning rate. - double learning_rate_; - - double stabilizer_; + double stabilizer_; /// Numerical stabilizer. }; #endif \ No newline at end of file diff --git a/include/SVGDCpp/Optimizer/RMSProp.hpp b/include/SVGDCpp/Optimizer/RMSProp.hpp index a9704a2..a37169b 100644 --- a/include/SVGDCpp/Optimizer/RMSProp.hpp +++ b/include/SVGDCpp/Optimizer/RMSProp.hpp @@ -1,12 +1,34 @@ +/** + * @file RMSProp.hpp + * @author Khai Yi Chin (khaiyichin@gmail.com) + * @brief RMSProp optimizer class header. + * + * @copyright Copyright (c) 2024 Khai Yi Chin + * + */ #ifndef SVGDCPP_RMSPROP_HPP #define SVGDCPP_RMSPROP_HPP #include "../Core.hpp" #include "Optimizer.hpp" +/** + * @class RMSProp + * @brief This class provides the RMSProp optimizer. + * @ingroup Optimizer_Module + */ class RMSProp : public Optimizer { public: + /** + * @brief Default constructor. + * + * @param dimension Dimension of the problem. + * @param num_particles Number of particles used in the problem. + * @param lr Learning rate. + * @param beta Weight decay rate. + * @param epsilon Numerical stabilizer. + */ RMSProp(const size_t &dimension, const size_t &num_particles, const double &lr, @@ -23,13 +45,27 @@ class RMSProp : public Optimizer } } + /** + * @brief Default destructor. + * + */ virtual ~RMSProp() {} + /** + * @brief Initialize the optimizer. + * + */ void Initialize() override { sum_of_sq_grad_ = Eigen::MatrixXd::Zero(dimension_, num_particles_).array(); } + /** + * @brief Execute the optimization of the gradient at each step. + * + * @param grad_matrix Gradient matrix for each particle coordinate. + * @return Optimized gradient matrix. + */ virtual Eigen::MatrixXd Step(const Eigen::MatrixXd &grad_matrix) { sum_of_sq_grad_ = decay_rate_ * sum_of_sq_grad_ + (1 - decay_rate_) * grad_matrix.array().square(); @@ -38,13 +74,13 @@ class RMSProp : public Optimizer } protected: - size_t dimension_; + size_t dimension_; ///< Dimension of the problem. - size_t num_particles_; + size_t num_particles_; ///< Number of particles used in the problem. - double decay_rate_; + double decay_rate_; ///< Weight decay rate. - Eigen::ArrayXXd sum_of_sq_grad_; + Eigen::ArrayXXd sum_of_sq_grad_; ///< Sum of squares gradient matrix. }; #endif \ No newline at end of file diff --git a/include/SVGDCpp/SVGD.hpp b/include/SVGDCpp/SVGD.hpp index 441430f..90f2e0f 100644 --- a/include/SVGDCpp/SVGD.hpp +++ b/include/SVGDCpp/SVGD.hpp @@ -1,11 +1,9 @@ /** * @file SVGD.hpp * @author Khai Yi Chin (khaiyichin@gmail.com) - * @brief SVGD class header - * @version 0.1 - * @date 2024-03-22 + * @brief SVGD class header. * - * @copyright Copyright (c) 2024 + * @copyright Copyright (c) 2024 Khai Yi Chin * */ @@ -244,7 +242,7 @@ class SVGD // Parallel application inspired from https://github.com/coin-or/CppAD/issues/197#issuecomment-1983462984 // Create copies of the original kernel for n particles - for (size_t i = 0; i < coord_matrix_ptr_->cols(); ++i) + for (int i = 0; i < coord_matrix_ptr_->cols(); ++i) { kernel_ptr_vector_.push_back(kernel_ptr_->CloneUniquePointer()); } @@ -380,7 +378,7 @@ class SVGD if (parallel_) { #pragma omp parallel for - for (size_t i = 0; i < coord_matrix_ptr_->cols(); ++i) + for (int i = 0; i < coord_matrix_ptr_->cols(); ++i) { kernel_ptr_vector_[i]->Step(); }