Skip to content

Commit

Permalink
Added s3_storage backend (WIP)
Browse files Browse the repository at this point in the history
  • Loading branch information
percona-ysorokin committed Mar 20, 2024
1 parent 3e53133 commit 4e804b4
Show file tree
Hide file tree
Showing 10 changed files with 241 additions and 30 deletions.
122 changes: 96 additions & 26 deletions .github/workflows/cmake.yml
Original file line number Diff line number Diff line change
Expand Up @@ -7,6 +7,9 @@ on:
branches: [ "main" ]

env:
AWS_SDK_CPP_MAJOR: 1
AWS_SDK_CPP_MINOR: 11
AWS_SDK_CPP_PATCH: 286
BOOST_MAJOR: 1
BOOST_MINOR: 83
BOOST_PATCH: 0
Expand All @@ -17,7 +20,8 @@ jobs:
runs-on: ubuntu-22.04
name: Formatting checks
steps:
- uses: actions/checkout@v4
- name: Cheking out code
uses: actions/checkout@v4
with:
fetch-depth: 2

Expand Down Expand Up @@ -52,15 +56,15 @@ jobs:
build_type: "Debug",
cc: "gcc-13",
cxx: "g++-13",
label: "debug_gcc13",
label: "Debug-gcc13",
run_mtr: true
}
- {
name: "GCC 13 RelWithDebInfo",
build_type: "RelWithDebInfo",
cc: "gcc-13",
cxx: "g++-13",
label: "relwithdebinfo_gcc13",
label: "RelWithDebInfo-gcc13",
run_mtr: true
}
- {
Expand All @@ -69,7 +73,8 @@ jobs:
cc: "gcc-13",
cxx: "g++-13",
sanitizer_cmake_flags: "-DWITH_ASAN=ON",
label: "asan_gcc13",
aws_sanitizer_cmake_flags: "-DENABLE_ADDRESS_SANITIZER=ON",
label: "ASan-gcc13",
run_mtr: true,
mtr_options: "--sanitize"
}
Expand All @@ -79,7 +84,8 @@ jobs:
cc: "clang-17",
cxx: "clang++-17",
libcxx_cmake_flags: "-DWITH_STDLIB_LIBCXX=ON",
label: "debug_clang17",
aws_libcxx_cmake_flags: "-DCMAKE_CXX_FLAGS_INIT=-stdlib=libc++",
label: "Debug-clang17",
run_clang_tidy: true,
}
- {
Expand All @@ -88,7 +94,8 @@ jobs:
cc: "clang-17",
cxx: "clang++-17",
libcxx_cmake_flags: "-DWITH_STDLIB_LIBCXX=ON",
label: "relwithdebinfo_clang17",
aws_libcxx_cmake_flags: "-DCMAKE_CXX_FLAGS_INIT=-stdlib=libc++",
label: "RelWithDebInfo-clang17",
run_clang_tidy: true,
}
- {
Expand All @@ -97,8 +104,10 @@ jobs:
cc: "clang-17",
cxx: "clang++-17",
libcxx_cmake_flags: "-DWITH_STDLIB_LIBCXX=ON",
aws_libcxx_cmake_flags: "-DCMAKE_CXX_FLAGS_INIT=-stdlib=libc++",
sanitizer_cmake_flags: "-DWITH_ASAN=ON",
label: "asan_clang17",
aws_sanitizer_cmake_flags: "-DENABLE_ADDRESS_SANITIZER=ON",
label: "ASan-clang17",
run_mtr: true,
mtr_options: "--sanitize"
}
Expand All @@ -117,10 +126,15 @@ jobs:
- name: Info df
run: df -h

- name: Install MySQL client libraries
- name: Info GitHub directories
run: |
echo github.workspace: ${{github.workspace}}
echo runner.temp : ${{runner.temp}}
- name: Install MySQL client libraries and CURL Development libraries
run: |
sudo apt-get update
sudo apt-get install libmysqlclient-dev
sudo apt-get install libmysqlclient-dev libcurl4-openssl-dev
- name: Install MySQL server and MTR
if: matrix.config.run_mtr
Expand Down Expand Up @@ -151,44 +165,100 @@ jobs:
- name: Creating deps directory
run: mkdir -p ${{runner.temp}}/deps

- name: Cache boost tarball
id: cache-boost-tarball
- name: Cache boost libraries
id: cache-boost-libraries
uses: actions/cache@v4
with:
path: ${{runner.temp}}/deps/${{format('boost_{0}_{1}_{2}{3}', env.BOOST_MAJOR, env.BOOST_MINOR, env.BOOST_PATCH, env.BOOST_EXT)}}
key: boost-tarball
path: ${{runner.temp}}/deps/${{format('boost_{0}_{1}_{2}', env.BOOST_MAJOR, env.BOOST_MINOR, env.BOOST_PATCH)}}
key: ${{format('boost-libraries-{0}-{1}-{2}', env.BOOST_MAJOR, env.BOOST_MINOR, env.BOOST_PATCH)}}

- name: Download boost libraries
if: steps.cache-boost-tarball.outputs.cache-hit != 'true'
- name: Download and unpack boost libraries
if: steps.cache-boost-libraries.outputs.cache-hit != 'true'
working-directory: ${{runner.temp}}/deps
run: wget --quiet ${{format('https://boostorg.jfrog.io/artifactory/main/release/{0}.{1}.{2}/source/boost_{0}_{1}_{2}{3}', env.BOOST_MAJOR, env.BOOST_MINOR, env.BOOST_PATCH, env.BOOST_EXT)}}
run: |
wget --quiet ${{format('https://boostorg.jfrog.io/artifactory/main/release/{0}.{1}.{2}/source/boost_{0}_{1}_{2}{3}', env.BOOST_MAJOR, env.BOOST_MINOR, env.BOOST_PATCH, env.BOOST_EXT)}}
tar xf ${{format('boost_{0}_{1}_{2}{3}', env.BOOST_MAJOR, env.BOOST_MINOR, env.BOOST_PATCH, env.BOOST_EXT)}}
rm -f ${{format('boost_{0}_{1}_{2}{3}', env.BOOST_MAJOR, env.BOOST_MINOR, env.BOOST_PATCH, env.BOOST_EXT)}}
- name: Unpack boost libraries
working-directory: ${{runner.temp}}/deps
run: tar xf ${{format('boost_{0}_{1}_{2}{3}', env.BOOST_MAJOR, env.BOOST_MINOR, env.BOOST_PATCH, env.BOOST_EXT)}}
- name: Cache AWS SDK C++ libraries
id: cache-aws-sdk-cpp-libraries
uses: actions/cache@v4
with:
path: ${{runner.temp}}/deps/aws-sdk-cpp-install-${{matrix.config.label}}
key: ${{format('aws-cpp-sdk-libraries-{0}-{1}-{2}-{3}', env.AWS_SDK_CPP_MAJOR, env.AWS_SDK_CPP_MINOR, env.AWS_SDK_CPP_PATCH, matrix.config.label)}}

- uses: actions/checkout@v4
- name: Checking out AWS SDK C++ source tree
if: steps.cache-aws-sdk-cpp-libraries.outputs.cache-hit != 'true'
uses: actions/checkout@v4
with:
repository: aws/aws-sdk-cpp
ref: ${{format('{0}.{1}.{2}', env.AWS_SDK_CPP_MAJOR, env.AWS_SDK_CPP_MINOR, env.AWS_SDK_CPP_PATCH)}}
path: aws-sdk-cpp
submodules: recursive

- name: Configure CMake for AWS SDK C++
if: steps.cache-aws-sdk-cpp-libraries.outputs.cache-hit != 'true'
run: |
cmake \
-B ${{github.workspace}}/../aws-sdk-cpp-build-${{matrix.config.label}} \
-S ${{github.workspace}}/aws-sdk-cpp \
-DCMAKE_INSTALL_PREFIX=${{runner.temp}}/deps/aws-sdk-cpp-install-${{matrix.config.label}} \
-DCMAKE_BUILD_TYPE=${{matrix.config.build_type}} \
-DCMAKE_C_COMPILER=${{matrix.config.cc}} \
-DCMAKE_CXX_COMPILER=${{matrix.config.cxx}} \
${{matrix.config.aws_libcxx_cmake_flags}} \
${{matrix.config.aws_sanitizer_cmake_flags}} \
-DCPP_STANDARD=20 \
-DENABLE_UNITY_BUILD=ON \
-DBUILD_SHARED_LIBS=OFF \
-DFORCE_SHARED_CRT=OFF \
-DENABLE_TESTING=OFF \
-DAUTORUN_UNIT_TESTS=OFF \
-DBUILD_ONLY=s3-crt
- name: CMake info for AWS SDK C++
if: steps.cache-aws-sdk-cpp-libraries.outputs.cache-hit != 'true'
run: cmake -L ${{github.workspace}}/../aws-sdk-cpp-build-${{matrix.config.label}}

- name: Build for AWS SDK C++
if: steps.cache-aws-sdk-cpp-libraries.outputs.cache-hit != 'true'
run: cmake --build ${{github.workspace}}/../aws-sdk-cpp-build-${{matrix.config.label}} --config ${{matrix.config.build_type}} --parallel

- name: Install for AWS SDK C++
if: steps.cache-aws-sdk-cpp-libraries.outputs.cache-hit != 'true'
run: cmake --install ${{github.workspace}}/../aws-sdk-cpp-build-${{matrix.config.label}} --config ${{matrix.config.build_type}}

- name: Checking out source tree
uses: actions/checkout@v4
with:
fetch-depth: 2

- name: Configure CMake
# Configure CMake in a 'build' subdirectory. `CMAKE_BUILD_TYPE` is only required if you are using a single-configuration generator such as make.
# See https://cmake.org/cmake/help/latest/variable/CMAKE_BUILD_TYPE.html?highlight=cmake_build_type
run: |
cmake -Wdev -Werror=dev -Wdeprecated -Werror=deprecated \
-B ${{github.workspace}}/../build-${{matrix.config.label}} \
-B ${{github.workspace}}/../percona-binlog-server-build-${{matrix.config.label}} \
-S ${{github.workspace}} \
-DCMAKE_BUILD_TYPE=${{matrix.config.build_type}} \
-DCMAKE_C_COMPILER=${{matrix.config.cc}} \
-DCMAKE_CXX_COMPILER=${{matrix.config.cxx}} \
${{matrix.config.libcxx_cmake_flags}} \
${{matrix.config.sanitizer_cmake_flags}} \
-DCMAKE_PREFIX_PATH=${{runner.temp}}/deps/aws-sdk-cpp-install-${{matrix.config.label}} \
-DBoost_ROOT=${{runner.temp}}/deps/${{format('boost_{0}_{1}_{2}', env.BOOST_MAJOR, env.BOOST_MINOR, env.BOOST_PATCH)}} \
-DCMAKE_EXPORT_COMPILE_COMMANDS=ON
- name: CMake info
run: cmake -L ${{github.workspace}}/..//build-${{matrix.config.label}}
run: cmake -L ${{github.workspace}}/../percona-binlog-server-build-${{matrix.config.label}}

- name: Build
# Build your program with the given configuration
run: cmake --build ${{github.workspace}}/../build-${{matrix.config.label}} --config ${{matrix.config.build_type}} --parallel
run: cmake --build ${{github.workspace}}/../percona-binlog-server-build-${{matrix.config.label}} --config ${{matrix.config.build_type}} --parallel

- name: Info Build artefacts
run: |
ls -la ${{github.workspace}}/..
- name: Info Clang Tidy
if: matrix.config.run_clang_tidy
Expand All @@ -197,7 +267,7 @@ jobs:
- name: Clang Tidy
if: matrix.config.run_clang_tidy
# Run Clang Tidy
run: run-clang-tidy-17 -header-filter=.* -j=${{steps.cpu-cores.outputs.count}} -use-color -p=${{github.workspace}}/../build-${{matrix.config.label}}
run: run-clang-tidy-17 -header-filter=.* -j=${{steps.cpu-cores.outputs.count}} -use-color -p=${{github.workspace}}/../percona-binlog-server-build-${{matrix.config.label}}

- name: MTR tests
if: matrix.config.run_mtr
Expand All @@ -208,13 +278,13 @@ jobs:
# Linking the "binlog_streaming" from the source tree into the MTR suits directory on the system
sudo ln -s ${{github.workspace}}/mtr/binlog_streaming /usr/lib/mysql-test/suite/binlog_streaming
# Running MTR from the system package
BINSRV=${{github.workspace}}/../build-${{matrix.config.label}}/binlog_server ./mtr \
BINSRV=${{github.workspace}}/../percona-binlog-server-build-${{matrix.config.label}}/binlog_server ./mtr \
--client-bindir=/usr/lib/mysql-test/bin --vardir=${{github.workspace}}/../mtrvardir \
--force --max-test-fail=0 --retry=0 --nounit-tests --big-test --repeat=2 --parallel=${{steps.cpu-cores.outputs.count}} \
--suite=binlog_streaming ${{matrix.config.mtr_options}}
- name: CTest
working-directory: ${{github.workspace}}/../build-${{matrix.config.label}}
working-directory: ${{github.workspace}}/../percona-binlog-server-build-${{matrix.config.label}}
# Execute tests defined by the CMake configuration.
# See https://cmake.org/cmake/help/latest/manual/ctest.1.html for more detail
run: ctest -C ${{matrix.config.build_type}} --parallel
7 changes: 7 additions & 0 deletions CMakeLists.txt
Original file line number Diff line number Diff line change
Expand Up @@ -61,6 +61,9 @@ find_package(Boost 1.83.0 EXACT REQUIRED)

find_package(MySQL REQUIRED)

find_package(ZLIB REQUIRED)
find_package(AWSSDK REQUIRED COMPONENTS s3-crt)

set(source_files
# main application files
src/app.cpp
Expand Down Expand Up @@ -167,6 +170,9 @@ set(source_files
src/binsrv/main_config.hpp
src/binsrv/main_config.cpp

src/binsrv/s3_storage_backend.hpp
src/binsrv/s3_storage_backend.cpp

src/binsrv/storage_fwd.hpp
src/binsrv/storage.hpp
src/binsrv/storage.cpp
Expand Down Expand Up @@ -240,6 +246,7 @@ target_link_libraries(binlog_server
binlog_server_compiler_flags
PRIVATE
Boost::headers MySQL::client
aws-cpp-sdk-s3-crt
)
# for some reason it is not possible to propagate CXX_EXTENSIONS and
# CXX_STANDARD_REQUIRED via interface library (binlog_server_compiler_flags)
Expand Down
3 changes: 3 additions & 0 deletions src/app.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -208,6 +208,9 @@ int main(int argc, char *argv[]) {
msg += ", path: ";
msg += storage_config.get<"path">();
logger->log(binsrv::log_severity::info, msg);
msg = "description: ";
msg += storage_backend->get_description();
logger->log(binsrv::log_severity::info, msg);

binsrv::storage storage{std::move(storage_backend)};
logger->log(binsrv::log_severity::info,
Expand Down
4 changes: 4 additions & 0 deletions src/binsrv/basic_storage_backend.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -51,4 +51,8 @@ void basic_storage_backend::close_stream() {
stream_opened_ = false;
}

[[nodiscard]] std::string basic_storage_backend::get_description() const {
return do_get_description();
}

} // namespace binsrv
4 changes: 4 additions & 0 deletions src/binsrv/basic_storage_backend.hpp
Original file line number Diff line number Diff line change
Expand Up @@ -28,6 +28,8 @@ class basic_storage_backend {
void write_data_to_stream(util::const_byte_span data);
void close_stream();

[[nodiscard]] std::string get_description() const;

private:
bool stream_opened_{false};

Expand All @@ -39,6 +41,8 @@ class basic_storage_backend {
virtual void do_open_stream(std::string_view name) = 0;
virtual void do_write_data_to_stream(util::const_byte_span data) = 0;
virtual void do_close_stream() = 0;

[[nodiscard]] virtual std::string do_get_description() const = 0;
};

} // namespace binsrv
Expand Down
5 changes: 5 additions & 0 deletions src/binsrv/filesystem_storage_backend.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -117,6 +117,11 @@ void filesystem_storage_backend::do_write_data_to_stream(

void filesystem_storage_backend::do_close_stream() { ofs_.close(); }

[[nodiscard]] std::string
filesystem_storage_backend::do_get_description() const {
return "local filesystem";
}

[[nodiscard]] std::filesystem::path
filesystem_storage_backend::get_object_path(std::string_view name) const {
auto result{root_path_};
Expand Down
2 changes: 2 additions & 0 deletions src/binsrv/filesystem_storage_backend.hpp
Original file line number Diff line number Diff line change
Expand Up @@ -31,6 +31,8 @@ class [[nodiscard]] filesystem_storage_backend : public basic_storage_backend {
void do_write_data_to_stream(util::const_byte_span data) override;
void do_close_stream() override;

[[nodiscard]] std::string do_get_description() const override;

[[nodiscard]] std::filesystem::path
get_object_path(std::string_view name) const;
};
Expand Down
70 changes: 70 additions & 0 deletions src/binsrv/s3_storage_backend.cpp
Original file line number Diff line number Diff line change
@@ -0,0 +1,70 @@
#include "binsrv/s3_storage_backend.hpp"

#include <cstddef>
#include <filesystem>
#include <memory>
#include <string>
#include <string_view>

#include <aws/core/Aws.h>

// TODO: remove this include when switching to clang-18 where transitive
// IWYU 'export' pragmas are handled properly
#include "binsrv/basic_storage_backend_fwd.hpp"

#include "util/byte_span.hpp"
#include "util/impl_helpers.hpp"

namespace binsrv {

using options_deimpl = util::deimpl<Aws::SDKOptions>;

void s3_storage_backend::options_deleter::operator()(void *ptr) const noexcept {
auto *casted_ptr{static_cast<Aws::SDKOptions *>(ptr)};
Aws::ShutdownAPI(*casted_ptr);
// deleting via std::default_delete to avoid
// cppcoreguidelines-owning-memory warnings
using delete_helper = std::default_delete<Aws::SDKOptions>;
delete_helper{}(casted_ptr);
}

s3_storage_backend::s3_storage_backend(std::string_view root_path)
: root_path_{root_path}, options_{new Aws::SDKOptions} {
Aws::InitAPI(*options_deimpl::get(options_));
}

[[nodiscard]] storage_object_name_container
s3_storage_backend::do_list_objects() {
storage_object_name_container result;
return result;
}

[[nodiscard]] std::string
s3_storage_backend::do_get_object(std::string_view /*name*/) {
static constexpr std::size_t file_size{8};
std::string file_content(file_size, 'x');
return file_content;
}

void s3_storage_backend::do_put_object(std::string_view /*name*/,
util::const_byte_span /*content*/) {}

void s3_storage_backend::do_open_stream(std::string_view /*name*/) {}

void s3_storage_backend::do_write_data_to_stream(
util::const_byte_span /*data*/) {}

void s3_storage_backend::do_close_stream() {}

[[nodiscard]] std::string s3_storage_backend::do_get_description() const {
std::string res{"AWS S3 (SDK "};
const auto &casted_options = *options_deimpl::get(options_);
res += std::to_string(casted_options.sdkVersion.major);
res += '.';
res += std::to_string(casted_options.sdkVersion.minor);
res += '.';
res += std::to_string(casted_options.sdkVersion.patch);
res += ')';
return res;
}
} // namespace binsrv
Loading

0 comments on commit 4e804b4

Please sign in to comment.