Reworked storage configuration section - added storage URI (#43)

Changed the way how storage backend is configured: instead of 'type' and 'path'
parameters we now have single 'uri', which incorporates both and is open for
future extensions
Currently, two schemas are supported:
* 'file://<path>' for local filesystem;
* 's3://<access_key_id>:<secret_access_key>@<bucket_name>/<path>' for AWS S3.
Please note that for 's3' credentials can be omitted, in other words
's3://<bucket_name>/<path>' is also a valid URI provided that this <bucket_name>
bucket is publicly accessible.

's3_storage_backend' class (still a stub) extended with missing AWS S3
configuration parameters:
* 'access_key_id';
* 'secret_access_key';
* 'bucket'.

Reworked constructors of both 's3_storage_backend' and
'filesystem_storage_backend': they now accept single
'boost::urls::url_view_base' parameter. These constructors extended with basic
URI validation (for schema, userinfo, host, port, path, query, and fragment).

'do_get_description()' method of both 's3_storage_backend' and
'filesystem_storage_backend' extended with including more configuration
parameters:
* 'path' for local filesystem;
* 'bucket' and 'path' for AWS S3 (credentials are omitted deliberately).

Reworked 'storage_backend_factory::create()' method: it now determines which
concrete implementation of the 'basic_storage_backend' interface it should
construct based on the schema part of the 'uri' parameter from the 'storage'
configuration section.

Update "help" message for the main application to reflect new 'storage.uri'
parameter.

Updated sample configuration JSON file.

CMakeLists.txt

@@ -72,12 +72,12 @@ if(WITH_ASAN)
   target_link_options(binlog_server_compiler_flags INTERFACE "-fsanitize=address")
 endif()
 
-find_package(Boost 1.84.0 EXACT REQUIRED)
+find_package(Boost 1.84.0 EXACT REQUIRED COMPONENTS url)
 
 find_package(MySQL REQUIRED)
 
 find_package(ZLIB REQUIRED)
-find_package(AWSSDK REQUIRED COMPONENTS s3-crt)
+find_package(AWSSDK 1.11.286 EXACT REQUIRED COMPONENTS s3-crt)
 
 set(source_files
   # main application files
@@ -260,7 +260,8 @@ target_link_libraries(binlog_server
   PUBLIC
     binlog_server_compiler_flags
   PRIVATE
-    Boost::headers MySQL::client
+    Boost::headers Boost::url
+    MySQL::client
     aws-cpp-sdk-s3-crt
 )
 # for some reason it is not possible to propagate CXX_EXTENSIONS and

main_config.json

@@ -10,7 +10,6 @@
     "password": ""
   },
   "storage": {
-    "type": "fs",
-    "path": "./storage"
+    "uri": "file:///home/user/vault"
   }
 }

mtr/binlog_streaming/r/binsrv.result

@@ -21,6 +21,7 @@ INSERT INTO t1 VALUES(DEFAULT);
 *** Generating a configuration file in JSON format for the Binlog
 *** Server utility.
 SET @storage_path = '<BINSRV_STORAGE_PATH>';
+SET @storage_uri = CONCAT('file://', @storage_path);
 SET @log_path = '<BINSRV_LOG_PATH>';
 SET @delimiter_pos = INSTR(USER(), '@');
 SET @connection_user = SUBSTRING(USER(), 1, @delimiter_pos - 1);
@@ -38,8 +39,7 @@ SET @binsrv_config_json = JSON_OBJECT(
 'password', ''
   ),
 'storage', JSON_OBJECT(
-'type', 'fs',
-'path', @storage_path
+'uri', @storage_uri
 )
 );
 

mtr/binlog_streaming/t/binsrv.test

@@ -42,6 +42,7 @@ INSERT INTO t1 VALUES(DEFAULT);
 --let $binsrv_storage_path = $MYSQL_TMP_DIR/storage
 --replace_result $binsrv_storage_path <BINSRV_STORAGE_PATH>
 eval SET @storage_path = '$binsrv_storage_path';
+SET @storage_uri = CONCAT('file://', @storage_path);
 
 --let $binsrv_log_path = $MYSQL_TMP_DIR/binsrv_utility.log
 --replace_result $binsrv_log_path <BINSRV_LOG_PATH>
@@ -64,8 +65,7 @@ eval SET @binsrv_config_json = JSON_OBJECT(
      'password', ''
   ),
   'storage', JSON_OBJECT(
-     'type', 'fs',
-     'path', @storage_path
+     'uri', @storage_uri
   )
 );
 

src/app.cpp

@@ -161,7 +161,7 @@ int main(int argc, char *argv[]) {
     std::cerr << "usage: " << executable_name
               << " <logger.level> <logger.file> <connection.host>"
                  " <connection.port> <connection.user> <connection.password>"
-                 " <storage.type> <storage.path>\n"
+                 " <storage.uri>\n"
               << "       " << executable_name << " <json_config_file>\n";
     return exit_code;
   }
@@ -218,11 +218,6 @@ int main(int argc, char *argv[]) {
     auto storage_backend{
         binsrv::storage_backend_factory::create(storage_config)};
     logger->log(binsrv::log_severity::info, "created binlog storage backend");
-    msg = "type: ";
-    msg += storage_config.get<"type">();
-    msg += ", path: ";
-    msg += storage_config.get<"path">();
-    logger->log(binsrv::log_severity::info, msg);
     msg = "description: ";
     msg += storage_backend->get_description();
     logger->log(binsrv::log_severity::info, msg);

src/binsrv/filesystem_storage_backend.cpp

@@ -24,6 +24,10 @@
 #include <string>
 #include <string_view>
 
+#include <boost/url/host_type.hpp>
+#include <boost/url/scheme.hpp>
+#include <boost/url/url_view_base.hpp>
+
 // TODO: remove this include when switching to clang-18 where transitive
 //       IWYU 'export' pragmas are handled properly
 #include "binsrv/basic_storage_backend_fwd.hpp"
@@ -34,9 +38,39 @@
 namespace binsrv {
 
 filesystem_storage_backend::filesystem_storage_backend(
-    std::string_view root_path)
-    : root_path_{root_path}, ofs_{} {
+    const boost::urls::url_view_base &uri)
+    : root_path_{}, ofs_{} {
   // TODO: switch to utf8 file names
+
+  // "file://<path>" for local filesystem
+  if (uri.scheme_id() != boost::urls::scheme::file ||
+      uri.scheme() != uri_schema) {
+    util::exception_location().raise<std::invalid_argument>(
+        "URI of invalid scheme provided");
+  }
+  if (uri.host_type() != boost::urls::host_type::name || !uri.host().empty()) {
+    util::exception_location().raise<std::invalid_argument>(
+        "file URI must not have host");
+  }
+  if (uri.has_port()) {
+    util::exception_location().raise<std::invalid_argument>(
+        "file URI must not have port");
+  }
+  if (uri.has_userinfo()) {
+    util::exception_location().raise<std::invalid_argument>(
+        "file URI must not have userinfo");
+  }
+  if (uri.has_query()) {
+    util::exception_location().raise<std::invalid_argument>(
+        "file URI must not have query");
+  }
+  if (uri.has_fragment()) {
+    util::exception_location().raise<std::invalid_argument>(
+        "file URI must not have fragment");
+  }
+
+  root_path_ = uri.path();
+
   if (!std::filesystem::exists(root_path_)) {
     util::exception_location().raise<std::invalid_argument>(
         "root path does not exist");
@@ -134,7 +168,7 @@ void filesystem_storage_backend::do_close_stream() { ofs_.close(); }
 
 [[nodiscard]] std::string
 filesystem_storage_backend::do_get_description() const {
-  return "local filesystem";
+  return "local filesystem - " + root_path_.generic_string();
 }
 
 [[nodiscard]] std::filesystem::path

src/binsrv/filesystem_storage_backend.hpp

@@ -20,13 +20,17 @@
 #include <fstream>
 #include <string_view>
 
+#include <boost/url/url_view_base.hpp>
+
 #include "binsrv/basic_storage_backend.hpp" // IWYU pragma: export
 
 namespace binsrv {
 
 class [[nodiscard]] filesystem_storage_backend : public basic_storage_backend {
 public:
-  explicit filesystem_storage_backend(std::string_view root_path);
+  static constexpr std::string_view uri_schema{"file"};
+
+  explicit filesystem_storage_backend(const boost::urls::url_view_base &uri);
 
   [[nodiscard]] const std::filesystem::path &get_root_path() const noexcept {
     return root_path_;

src/binsrv/s3_storage_backend.cpp

@@ -18,16 +18,22 @@
 #include <cstddef>
 #include <filesystem>
 #include <memory>
+#include <stdexcept>
 #include <string>
 #include <string_view>
 
+#include <boost/url/host_type.hpp>
+#include <boost/url/scheme.hpp>
+#include <boost/url/url_view_base.hpp>
+
 #include <aws/core/Aws.h>
 
 // TODO: remove this include when switching to clang-18 where transitive
 //       IWYU 'export' pragmas are handled properly
 #include "binsrv/basic_storage_backend_fwd.hpp"
 
 #include "util/byte_span.hpp"
+#include "util/exception_location_helpers.hpp"
 #include "util/impl_helpers.hpp"
 
 namespace binsrv {
@@ -43,9 +49,50 @@ void s3_storage_backend::options_deleter::operator()(void *ptr) const noexcept {
   delete_helper{}(casted_ptr);
 }
 
-s3_storage_backend::s3_storage_backend(std::string_view root_path)
-    : root_path_{root_path}, options_{new Aws::SDKOptions} {
+s3_storage_backend::s3_storage_backend(const boost::urls::url_view_base &uri)
+    : access_key_id_{}, secret_access_key_{}, root_path_{},
+      options_{new Aws::SDKOptions} {
   Aws::InitAPI(*options_deimpl::get(options_));
+
+  // "s3://<access_key_id>:<secret_access_key>@<bucket_name>/<path>" for AWS S3
+  if (uri.scheme_id() != boost::urls::scheme::unknown ||
+      uri.scheme() != uri_schema) {
+    util::exception_location().raise<std::invalid_argument>(
+        "URI of invalid scheme provided");
+  }
+  if (uri.host_type() != boost::urls::host_type::name || uri.host().empty()) {
+    util::exception_location().raise<std::invalid_argument>(
+        "s3 URI must have host");
+  }
+  if (uri.host().find('.') != std::string::npos) {
+    util::exception_location().raise<std::invalid_argument>(
+        "s3 URI host must be a single bucket name");
+  }
+  bucket_ = uri.host();
+
+  if (uri.has_port()) {
+    util::exception_location().raise<std::invalid_argument>(
+        "s3 URI must not have port");
+  }
+
+  if (uri.has_userinfo()) {
+    if (!uri.has_password()) {
+      util::exception_location().raise<std::invalid_argument>(
+          "s3 URI must have either both user and password or none");
+    }
+    access_key_id_ = uri.user();
+    secret_access_key_ = uri.password();
+  }
+  if (uri.has_query()) {
+    util::exception_location().raise<std::invalid_argument>(
+        "s3 URI must not have query");
+  }
+  if (uri.has_fragment()) {
+    util::exception_location().raise<std::invalid_argument>(
+        "s3 URI must not have fragment");
+  }
+
+  root_path_ = uri.path();
 }
 
 [[nodiscard]] storage_object_name_container
@@ -79,7 +126,15 @@ void s3_storage_backend::do_close_stream() {}
   res += std::to_string(casted_options.sdkVersion.minor);
   res += '.';
   res += std::to_string(casted_options.sdkVersion.patch);
-  res += ')';
+  res += ") - ";
+
+  res += "bucket: ";
+  res += bucket_;
+  res += ", path: ";
+  res += root_path_.generic_string();
+  res += ", credentials: ";
+  res += (has_credentials() ? "***hidden***" : "none");
+
   return res;
 }
 } // namespace binsrv

src/binsrv/s3_storage_backend.hpp

@@ -19,19 +19,40 @@
 #include <filesystem>
 #include <string_view>
 
+#include <boost/url/url_view_base.hpp>
+
 #include "binsrv/basic_storage_backend.hpp" // IWYU pragma: export
 
 namespace binsrv {
 
 class [[nodiscard]] s3_storage_backend : public basic_storage_backend {
 public:
-  explicit s3_storage_backend(std::string_view root_path);
+  static constexpr std::string_view uri_schema{"s3"};
+
+  explicit s3_storage_backend(const boost::urls::url_view_base &uri);
+
+  [[nodiscard]] const std::string &get_bucket() const noexcept {
+    return bucket_;
+  }
+  [[nodiscard]] const std::string &get_access_key_id() const noexcept {
+    return access_key_id_;
+  }
+  [[nodiscard]] const std::string &get_secret_access_key() const noexcept {
+    return secret_access_key_;
+  }
+  [[nodiscard]] bool has_credentials() const noexcept {
+    return !access_key_id_.empty();
+  }
 
   [[nodiscard]] const std::filesystem::path &get_root_path() const noexcept {
     return root_path_;
   }
 
 private:
+  std::string access_key_id_;
+  // TODO: do not store secret_access_key in plain
+  std::string secret_access_key_;
+  std::string bucket_;
   std::filesystem::path root_path_;
   struct options_deleter {
     void operator()(void *ptr) const noexcept;

src/binsrv/storage_backend_factory.cpp

@@ -18,6 +18,9 @@
 #include <memory>
 #include <stdexcept>
 
+#include <boost/url/parse.hpp>
+#include <boost/url/url_view.hpp>
+
 #include "binsrv/basic_storage_backend_fwd.hpp"
 #include "binsrv/filesystem_storage_backend.hpp"
 #include "binsrv/s3_storage_backend.hpp"
@@ -29,12 +32,22 @@ namespace binsrv {
 
 basic_storage_backend_ptr
 storage_backend_factory::create(const storage_config &config) {
-  const auto &storage_backend_type = config.get<"type">();
-  if (storage_backend_type == "fs") {
-    return std::make_unique<filesystem_storage_backend>(config.get<"path">());
+  const auto &storage_backend_uri = config.get<"uri">();
+
+  const auto uri_parse_result{
+      boost::urls::parse_absolute_uri(storage_backend_uri)};
+  if (!uri_parse_result) {
+    util::exception_location().raise<std::invalid_argument>(
+        "invalid storage backend URI");
+  }
+
+  const auto storage_backend_type{uri_parse_result->scheme()};
+
+  if (storage_backend_type == filesystem_storage_backend::uri_schema) {
+    return std::make_unique<filesystem_storage_backend>(*uri_parse_result);
   }
-  if (storage_backend_type == "s3") {
-    return std::make_unique<s3_storage_backend>(config.get<"path">());
+  if (storage_backend_type == s3_storage_backend::uri_schema) {
+    return std::make_unique<s3_storage_backend>(*uri_parse_result);
   }
   util::exception_location().raise<std::invalid_argument>(
       "unknown storage backend type");

src/binsrv/storage_config.hpp

@@ -27,8 +27,7 @@ namespace binsrv {
 // clang-format off
 struct [[nodiscard]] storage_config
     : util::nv_tuple<
-          util::nv<"type", std::string>,
-          util::nv<"path", std::string>
+          util::nv<"uri", std::string>
       > {};
 // clang-format on