diff --git a/CMakeLists.txt b/CMakeLists.txt index 4fac66b..b8b6980 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -49,8 +49,68 @@ find_package(Boost 1.83.0 EXACT REQUIRED) find_package(MySQL REQUIRED) set(source_files + # main application files src/app.cpp + # binlog event data structure files + src/binsrv/event/checksum_algorithm_type_fwd.hpp + src/binsrv/event/checksum_algorithm_type.hpp + + src/binsrv/event/code_type_fwd.hpp + src/binsrv/event/code_type.hpp + + src/binsrv/event/common_header_fwd.hpp + src/binsrv/event/common_header.hpp + src/binsrv/event/common_header.cpp + + src/binsrv/event/empty_body_fwd.hpp + src/binsrv/event/empty_body.hpp + src/binsrv/event/empty_body.cpp + + src/binsrv/event/empty_post_header_fwd.hpp + src/binsrv/event/empty_post_header.hpp + src/binsrv/event/empty_post_header.cpp + + src/binsrv/event/event_fwd.hpp + src/binsrv/event/event.hpp + src/binsrv/event/event.cpp + + src/binsrv/event/flag_type_fwd.hpp + src/binsrv/event/flag_type.hpp + + src/binsrv/event/footer_fwd.hpp + src/binsrv/event/footer.hpp + src/binsrv/event/footer.cpp + + src/binsrv/event/format_description_body_impl_fwd.hpp + src/binsrv/event/format_description_body_impl.hpp + src/binsrv/event/format_description_body_impl.cpp + + src/binsrv/event/format_description_post_header_impl_fwd.hpp + src/binsrv/event/format_description_post_header_impl.hpp + src/binsrv/event/format_description_post_header_impl.cpp + + src/binsrv/event/generic_body_fwd.hpp + src/binsrv/event/generic_body.hpp + + src/binsrv/event/generic_post_header_fwd.hpp + src/binsrv/event/generic_post_header.hpp + + src/binsrv/event/rotate_body_impl_fwd.hpp + src/binsrv/event/rotate_body_impl.hpp + src/binsrv/event/rotate_body_impl.cpp + + src/binsrv/event/rotate_post_header_impl_fwd.hpp + src/binsrv/event/rotate_post_header_impl.hpp + src/binsrv/event/rotate_post_header_impl.cpp + + src/binsrv/event/unknown_body_fwd.hpp + src/binsrv/event/unknown_body.hpp + + src/binsrv/event/unknown_post_header_fwd.hpp + src/binsrv/event/unknown_post_header.hpp + + # billog files src/binsrv/basic_logger_fwd.hpp src/binsrv/basic_logger.hpp src/binsrv/basic_logger.cpp @@ -61,16 +121,6 @@ set(source_files src/binsrv/exception_handling_helpers.hpp src/binsrv/exception_handling_helpers.cpp - src/binsrv/event_flag_fwd.hpp - src/binsrv/event_flag.hpp - - src/binsrv/event_header_fwd.hpp - src/binsrv/event_header.hpp - src/binsrv/event_header.cpp - - src/binsrv/event_type_fwd.hpp - src/binsrv/event_type.hpp - src/binsrv/file_logger.hpp src/binsrv/file_logger.cpp @@ -87,6 +137,11 @@ set(source_files src/binsrv/master_config.hpp src/binsrv/master_config.cpp + # various utility files + src/util/byte_span_fwd.hpp + src/util/byte_span.hpp + src/util/byte_span_extractors.hpp + src/util/command_line_helpers_fwd.hpp src/util/command_line_helpers.hpp src/util/command_line_helpers.cpp @@ -112,6 +167,9 @@ set(source_files src/util/nv_tuple_from_command_line.hpp src/util/nv_tuple_from_json.hpp + src/util/redirectable.hpp + + # mysql wrapper library files src/easymysql/core_error_helpers_private.hpp src/easymysql/core_error_helpers_private.cpp src/easymysql/core_error.hpp diff --git a/src/app.cpp b/src/app.cpp index ee29ff6..f46c7cd 100644 --- a/src/app.cpp +++ b/src/app.cpp @@ -5,6 +5,7 @@ #include #include #include +#include #include #include #include @@ -12,49 +13,95 @@ #include #include "binsrv/basic_logger.hpp" -#include "binsrv/event_header.hpp" #include "binsrv/exception_handling_helpers.hpp" #include "binsrv/log_severity.hpp" #include "binsrv/logger_factory.hpp" #include "binsrv/master_config.hpp" +#include "binsrv/event/code_type.hpp" +#include "binsrv/event/event.hpp" + #include "easymysql/binlog.hpp" #include "easymysql/connection.hpp" #include "easymysql/connection_config.hpp" #include "easymysql/library.hpp" +#include "util/byte_span_fwd.hpp" #include "util/command_line_helpers.hpp" #include "util/exception_location_helpers.hpp" #include "util/nv_tuple.hpp" void log_span_dump(binsrv::basic_logger &logger, - easymysql::binlog_stream_span portion) { - static constexpr std::size_t bytes_per_dump_line = 16; - std::size_t offset = 0; + util::const_byte_span portion) { + static constexpr std::size_t bytes_per_dump_line{16U}; + std::size_t offset{0U}; while (offset < std::size(portion)) { std::ostringstream oss; oss << '['; oss << std::setfill('0') << std::hex; auto sub = portion.subspan( offset, std::min(bytes_per_dump_line, std::size(portion) - offset)); - for (const std::uint8_t current_byte : sub) { - oss << ' ' << std::setw(2) << static_cast(current_byte); + for (auto current_byte : sub) { + oss << ' ' << std::setw(2) + << std::to_integer(current_byte); + } + const std::size_t filler_length = + (bytes_per_dump_line - std::size(sub)) * 3U; + oss << std::setfill(' ') << std::setw(static_cast(filler_length)) + << ""; + oss << " ] "; + const auto &ctype_facet{ + std::use_facet>(std::locale::classic())}; + + for (auto current_byte : sub) { + auto current_char{std::to_integer(current_byte)}; + if (!ctype_facet.is(std::ctype_base::print, current_char)) { + current_char = '.'; + } + oss.put(current_char); } - oss << " ]"; logger.log(binsrv::log_severity::trace, oss.str()); offset += bytes_per_dump_line; } } -void log_generic_event(binsrv::basic_logger &logger, - const binsrv::event_header &generic_event) { +void log_event_common_header( + binsrv::basic_logger &logger, + const binsrv::event::common_header &common_header) { std::ostringstream oss; - oss << "ts: " << generic_event.get_readable_timestamp() - << ", type:" << generic_event.get_readable_type_code() - << ", server_id:" << generic_event.get_server_id() - << ", event size:" << generic_event.get_event_size() - << ", next event position:" << generic_event.get_next_event_position() - << ", flags: (" << generic_event.get_readable_flags() << ')'; + oss << "ts: " << common_header.get_readable_timestamp() + << ", type:" << common_header.get_readable_type_code() + << ", server id:" << common_header.get_server_id_raw() + << ", event size:" << common_header.get_event_size_raw() + << ", next event position:" << common_header.get_next_event_position_raw() + << ", flags: (" << common_header.get_readable_flags() << ')'; + + logger.log(binsrv::log_severity::debug, oss.str()); +} + +void log_format_description_event( + binsrv::basic_logger &logger, + const binsrv::event::generic_post_header< + binsrv::event::code_type::format_description> &post_header, + const binsrv::event::generic_body< + binsrv::event::code_type::format_description> &body) { + std::ostringstream oss; + oss << '\n' + << " binlog version : " << post_header.get_binlog_version_raw() + << '\n' + << " server version : " << post_header.get_server_version() << '\n' + << " create timestamp : " << post_header.get_readable_create_timestamp() + << '\n' + << " header length : " << post_header.get_common_header_length() + << '\n' + << " checksum algorithm: " << body.get_readable_checksum_algorithm() + << '\n' + << " post-header length for ROTATE: " + << post_header.get_post_header_length(binsrv::event::code_type::rotate) + << '\n' + << " post-header length for FDE : " + << post_header.get_post_header_length( + binsrv::event::code_type::format_description); logger.log(binsrv::log_severity::debug, oss.str()); } @@ -70,9 +117,10 @@ int main(int argc, char *argv[]) { if (number_of_cmd_args != binsrv::master_config::flattened_size + 1 && number_of_cmd_args != 2) { - std::cerr << "usage: " << executable_name - << " \n" - << " " << executable_name << " \n"; + std::cerr + << "usage: " << executable_name + << " \n" + << " " << executable_name << " \n"; return exit_code; } binsrv::basic_logger_ptr logger; @@ -92,7 +140,7 @@ int main(int argc, char *argv[]) { util::get_readable_command_line_arguments(cmd_args)); binsrv::master_config_ptr config; - if (number_of_cmd_args == 2) { + if (number_of_cmd_args == 2U) { logger->log(binsrv::log_severity::delimiter, "Reading connection configuration from the JSON file."); config = std::make_shared(cmd_args[1]); @@ -155,16 +203,28 @@ int main(int argc, char *argv[]) { msg += connection.get_character_set_name(); logger->log(binsrv::log_severity::info, msg); - static constexpr std::uint32_t default_server_id = 0; + static constexpr std::uint32_t default_server_id{0U}; auto binlog = connection.create_binlog(default_server_id); logger->log(binsrv::log_severity::info, "opened binary log connection"); - easymysql::binlog_stream_span portion; + // TODO: make sure we write 'Binlog File Header' [ 0xFE 'bin’]` to the + // beginning of the binlog file + // TODO: The first event is either a START_EVENT_V3 or a + // FORMAT_DESCRIPTION_EVENT while the last event is either a + // STOP_EVENT or ROTATE_EVENT. For Binlog Version 4 (current one) + // only FORMAT_DESCRIPTION_EVENT / ROTATE_EVENT pair should be + // acceptable. + + // Network streams are requested with COM_BINLOG_DUMP and + // each Binlog Event response is prepended with 00 OK-byte. + static constexpr std::byte expected_event_packet_prefix{'\0'}; + + util::const_byte_span portion; + binsrv::event::optional_format_description_post_header fde_post_header{}; + binsrv::event::optional_format_description_body fde_body{}; + while (!(portion = binlog.fetch()).empty()) { - // Network streams are requested with COM_BINLOG_DUMP and - // prepend each Binlog Event with 00 OK-byte. - static constexpr unsigned char expected_event_prefix = '\0'; - if (portion[0] != expected_event_prefix) { + if (portion[0] != expected_event_packet_prefix) { util::exception_location().raise( "unexpected event prefix"); } @@ -173,8 +233,25 @@ int main(int argc, char *argv[]) { "fetched " + std::to_string(std::size(portion)) + "-byte(s) event from binlog"); - const binsrv::event_header generic_event{portion}; - log_generic_event(*logger, generic_event); + const binsrv::event::event generic_event{portion, fde_post_header, + fde_body}; + + log_event_common_header(*logger, generic_event.get_common_header()); + if (generic_event.get_common_header().get_type_code() == + binsrv::event::code_type::format_description) { + const auto &local_fde_post_header = + std::get>( + generic_event.get_post_header()); + const auto &local_fde_body = std::get>( + generic_event.get_body()); + + log_format_description_event(*logger, local_fde_post_header, + local_fde_body); + fde_post_header = local_fde_post_header; + fde_body = local_fde_body; + } log_span_dump(*logger, portion); } diff --git a/src/binsrv/basic_logger.cpp b/src/binsrv/basic_logger.cpp index e623245..521aa8c 100644 --- a/src/binsrv/basic_logger.cpp +++ b/src/binsrv/basic_logger.cpp @@ -16,8 +16,8 @@ basic_logger::basic_logger(log_severity min_level) noexcept void basic_logger::log(log_severity level, std::string_view message) { if (level >= min_level_) { - static constexpr auto timestamp_length = - std::size("YYYY-MM-DDTHH:MM:SS.fffffffff") - 1; + static constexpr auto timestamp_length{ + std::size("YYYY-MM-DDTHH:MM:SS.fffffffff") - 1U}; const auto timestamp = boost::posix_time::microsec_clock::universal_time(); ; const auto level_label = to_string_view(level); diff --git a/src/binsrv/event/checksum_algorithm_type.hpp b/src/binsrv/event/checksum_algorithm_type.hpp new file mode 100644 index 0000000..4f70e46 --- /dev/null +++ b/src/binsrv/event/checksum_algorithm_type.hpp @@ -0,0 +1,48 @@ +#ifndef BINSRV_EVENT_CHECKSUM_ALGORITHM_TYPE_HPP +#define BINSRV_EVENT_CHECKSUM_ALGORITHM_TYPE_HPP + +#include "binsrv/event/checksum_algorithm_type_fwd.hpp" // IWYU pragma: export + +#include +#include +#include +#include + +namespace binsrv::event { + +// NOLINTBEGIN(cppcoreguidelines-macro-usage) +// Checksum algorithm type codes copied from +// https://github.com/mysql/mysql-server/blob/mysql-8.0.35/libbinlogevents/include/binlog_event.h#L425 +// clang-format off +#define BINSRV_CHECKSUM_ALGORITHM_TYPE_XY_SEQUENCE() \ + BINSRV_CHECKSUM_ALGORITHM_TYPE_XY_MACRO(off , 0), \ + BINSRV_CHECKSUM_ALGORITHM_TYPE_XY_MACRO(crc32, 1) +// clang-format on + +#define BINSRV_CHECKSUM_ALGORITHM_TYPE_XY_MACRO(X, Y) X = Y +enum class checksum_algorithm_type : std::uint8_t { + BINSRV_CHECKSUM_ALGORITHM_TYPE_XY_SEQUENCE(), + delimiter +}; +#undef BINSRV_CHECKSUM_ALGORITHM_TYPE_XY_MACRO + +inline std::string_view to_string_view(checksum_algorithm_type code) noexcept { + using namespace std::string_view_literals; + using nv_pair = std::pair; +#define BINSRV_CHECKSUM_ALGORITHM_TYPE_XY_MACRO(X, Y) \ + nv_pair { checksum_algorithm_type::X, #X##sv } + static constexpr std::array labels{ + BINSRV_CHECKSUM_ALGORITHM_TYPE_XY_SEQUENCE(), + nv_pair{checksum_algorithm_type::delimiter, ""sv}}; +#undef BINSRV_CHECKSUM_ALGORITHM_TYPE_XY_MACRO + return std::ranges::find(labels, + std::min(checksum_algorithm_type::delimiter, code), + &nv_pair::first) + ->second; +} +#undef BINSRV_CHECKSUM_ALGORITHM_TYPE_XY_SEQUENCE +// NOLINTEND(cppcoreguidelines-macro-usage) + +} // namespace binsrv::event + +#endif // BINSRV_EVENT_CHECKSUM_ALGORITHM_TYPE_HPP diff --git a/src/binsrv/event/checksum_algorithm_type_fwd.hpp b/src/binsrv/event/checksum_algorithm_type_fwd.hpp new file mode 100644 index 0000000..4287928 --- /dev/null +++ b/src/binsrv/event/checksum_algorithm_type_fwd.hpp @@ -0,0 +1,12 @@ +#ifndef BINSRV_EVENT_CHECKSUM_ALGORITHM_TYPE_FWD_HPP +#define BINSRV_EVENT_CHECKSUM_ALGORITHM_TYPE_FWD_HPP + +#include + +namespace binsrv::event { + +enum class checksum_algorithm_type : std::uint8_t; + +} // namespace binsrv::event + +#endif // BINSRV_EVENT_CHECKSUM_ALGORITHM_TYPE_FWD_HPP diff --git a/src/binsrv/event/code_type.hpp b/src/binsrv/event/code_type.hpp new file mode 100644 index 0000000..bc56a3a --- /dev/null +++ b/src/binsrv/event/code_type.hpp @@ -0,0 +1,79 @@ +#ifndef BINSRV_EVENT_CODE_TYPE_HPP +#define BINSRV_EVENT_CODE_TYPE_HPP + +#include "binsrv/event/code_type_fwd.hpp" // IWYU pragma: export + +#include +#include +#include +#include + +namespace binsrv::event { + +// NOLINTBEGIN(cppcoreguidelines-macro-usage) +// Event type codes copied from +// https://github.com/mysql/mysql-server/blob/mysql-8.0.35/libbinlogevents/include/binlog_event.h#L274 +// clang-format off +#define BINSRV_EVENT_CODE_TYPE_XY_SEQUENCE() \ + BINSRV_EVENT_CODE_TYPE_XY_MACRO(unknown , 0), \ + BINSRV_EVENT_CODE_TYPE_XY_MACRO(start_v3 , 1), \ + BINSRV_EVENT_CODE_TYPE_XY_MACRO(query , 2), \ + BINSRV_EVENT_CODE_TYPE_XY_MACRO(stop , 3), \ + BINSRV_EVENT_CODE_TYPE_XY_MACRO(rotate , 4), \ + BINSRV_EVENT_CODE_TYPE_XY_MACRO(intvar , 5), \ + BINSRV_EVENT_CODE_TYPE_XY_MACRO(slave , 7), \ + BINSRV_EVENT_CODE_TYPE_XY_MACRO(append_block , 9), \ + BINSRV_EVENT_CODE_TYPE_XY_MACRO(delete_file , 11), \ + BINSRV_EVENT_CODE_TYPE_XY_MACRO(rand , 13), \ + BINSRV_EVENT_CODE_TYPE_XY_MACRO(user_var , 14), \ + BINSRV_EVENT_CODE_TYPE_XY_MACRO(format_description , 15), \ + BINSRV_EVENT_CODE_TYPE_XY_MACRO(xid , 16), \ + BINSRV_EVENT_CODE_TYPE_XY_MACRO(begin_load_query , 17), \ + BINSRV_EVENT_CODE_TYPE_XY_MACRO(execute_load_query , 18), \ + BINSRV_EVENT_CODE_TYPE_XY_MACRO(table_map , 19), \ + BINSRV_EVENT_CODE_TYPE_XY_MACRO(write_rows_v1 , 23), \ + BINSRV_EVENT_CODE_TYPE_XY_MACRO(update_rows_v1 , 24), \ + BINSRV_EVENT_CODE_TYPE_XY_MACRO(delete_rows_v1 , 25), \ + BINSRV_EVENT_CODE_TYPE_XY_MACRO(incident , 26), \ + BINSRV_EVENT_CODE_TYPE_XY_MACRO(heartbeat_log , 27), \ + BINSRV_EVENT_CODE_TYPE_XY_MACRO(ignorable_log , 28), \ + BINSRV_EVENT_CODE_TYPE_XY_MACRO(rows_query_log , 29), \ + BINSRV_EVENT_CODE_TYPE_XY_MACRO(write_rows , 30), \ + BINSRV_EVENT_CODE_TYPE_XY_MACRO(update_rows , 31), \ + BINSRV_EVENT_CODE_TYPE_XY_MACRO(delete_rows , 32), \ + BINSRV_EVENT_CODE_TYPE_XY_MACRO(gtid_log , 33), \ + BINSRV_EVENT_CODE_TYPE_XY_MACRO(anonymous_gtid_log , 34), \ + BINSRV_EVENT_CODE_TYPE_XY_MACRO(previous_gtids_log , 35), \ + BINSRV_EVENT_CODE_TYPE_XY_MACRO(transaction_context, 36), \ + BINSRV_EVENT_CODE_TYPE_XY_MACRO(view_change , 37), \ + BINSRV_EVENT_CODE_TYPE_XY_MACRO(xa_prepare_log , 38), \ + BINSRV_EVENT_CODE_TYPE_XY_MACRO(partial_update_rows, 39), \ + BINSRV_EVENT_CODE_TYPE_XY_MACRO(transaction_payload, 40), \ + BINSRV_EVENT_CODE_TYPE_XY_MACRO(heartbeat_log_v2 , 41) +// clang-format on + +#define BINSRV_EVENT_CODE_TYPE_XY_MACRO(X, Y) X = Y +enum class code_type : std::uint8_t { + BINSRV_EVENT_CODE_TYPE_XY_SEQUENCE(), + delimiter +}; +#undef BINSRV_EVENT_CODE_TYPE_XY_MACRO + +inline std::string_view to_string_view(code_type code) noexcept { + using namespace std::string_view_literals; + using nv_pair = std::pair; +#define BINSRV_EVENT_CODE_TYPE_XY_MACRO(X, Y) \ + nv_pair { code_type::X, #X##sv } + static constexpr std::array labels{BINSRV_EVENT_CODE_TYPE_XY_SEQUENCE(), + nv_pair{code_type::delimiter, ""sv}}; +#undef BINSRV_EVENT_CODE_TYPE_XY_MACRO + return std::ranges::find(labels, std::min(code_type::delimiter, code), + &nv_pair::first) + ->second; +} +#undef BINSRV_EVENT_CODE_TYPE_XY_SEQUENCE +// NOLINTEND(cppcoreguidelines-macro-usage) + +} // namespace binsrv::event + +#endif // BINSRV_EVENT_CODE_TYPE_HPP diff --git a/src/binsrv/event/code_type_fwd.hpp b/src/binsrv/event/code_type_fwd.hpp new file mode 100644 index 0000000..0da92ae --- /dev/null +++ b/src/binsrv/event/code_type_fwd.hpp @@ -0,0 +1,12 @@ +#ifndef BINSRV_EVENT_CODE_TYPE_FWD_HPP +#define BINSRV_EVENT_CODE_TYPE_FWD_HPP + +#include + +namespace binsrv::event { + +enum class code_type : std::uint8_t; + +} // namespace binsrv::event + +#endif // BINSRV_EVENT_CODE_TYPE_FWD_HPP diff --git a/src/binsrv/event/common_header.cpp b/src/binsrv/event/common_header.cpp new file mode 100644 index 0000000..7a19c28 --- /dev/null +++ b/src/binsrv/event/common_header.cpp @@ -0,0 +1,102 @@ +#include "binsrv/event/common_header.hpp" + +#include +#include +#include + +#include + +#include +#include + +#include "binsrv/event/code_type.hpp" +#include "binsrv/event/flag_type.hpp" + +#include "util/byte_span_extractors.hpp" +#include "util/byte_span_fwd.hpp" +#include "util/conversion_helpers.hpp" +#include "util/exception_location_helpers.hpp" +#include "util/flag_set.hpp" + +namespace binsrv::event { + +common_header::common_header(util::const_byte_span portion) { + // TODO: rework with direct member initialization + + /* + https://github.com/mysql/mysql-server/blob/mysql-8.0.35/libbinlogevents/src/binlog_event.cpp#L197 + + The first 19 bytes in the header is as follows: + +============================================+ + | member_variable offset : len | + +============================================+ + | when.tv_sec 0 : 4 | + +--------------------------------------------+ + | type_code EVENT_TYPE_OFFSET(4) : 1 | + +--------------------------------------------+ + | server_id SERVER_ID_OFFSET(5) : 4 | + +--------------------------------------------+ + | data_written EVENT_LEN_OFFSET(9) : 4 | + +--------------------------------------------+ + | log_pos LOG_POS_OFFSET(13) : 4 | + +--------------------------------------------+ + | flags FLAGS_OFFSET(17) : 2 | + +--------------------------------------------+ + | extra_headers 19 : x-19| + +============================================+ + */ + + // TODO: initialize size_in_bytes directly based on the sum of fields + // widths instead of this static_assert + static_assert(sizeof timestamp_ + sizeof type_code_ + sizeof server_id_ + + sizeof event_size_ + sizeof next_event_position_ + + sizeof flags_ == + size_in_bytes, + "mismatch in common_header::size_in_bytes"); + // make sure we did OK with data members reordering + static_assert(sizeof *this == boost::alignment::align_up( + size_in_bytes, alignof(decltype(*this))), + "inefficient data member reordering in common_header"); + + if (std::size(portion) != size_in_bytes) { + util::exception_location().raise( + "invalid event common header length"); + } + + auto remainder = portion; + util::extract_fixed_int_from_byte_span(remainder, timestamp_); + util::extract_fixed_int_from_byte_span(remainder, type_code_); + util::extract_fixed_int_from_byte_span(remainder, server_id_); + util::extract_fixed_int_from_byte_span(remainder, event_size_); + util::extract_fixed_int_from_byte_span(remainder, next_event_position_); + util::extract_fixed_int_from_byte_span(remainder, flags_); + + if (get_type_code_raw() >= util::enum_to_index(code_type::delimiter) || + to_string_view(get_type_code()).empty()) { + util::exception_location().raise( + "invalid event code in event header"); + } + // TODO: check if flags are valid (all the bits have corresponding enum) + // TODO: check that events with 'artificial' flag set have timestamp == 0 + // and next_event_position == 0 +} + +[[nodiscard]] std::string common_header::get_readable_timestamp() const { + return boost::posix_time::to_simple_string( + boost::posix_time::from_time_t(get_timestamp())); +} + +[[nodiscard]] std::string_view +common_header::get_readable_type_code() const noexcept { + return to_string_view(get_type_code()); +} + +[[nodiscard]] flag_set common_header::get_flags() const noexcept { + return flag_set{get_flags_raw()}; +} + +[[nodiscard]] std::string common_header::get_readable_flags() const { + return to_string(get_flags()); +} + +} // namespace binsrv::event diff --git a/src/binsrv/event/common_header.hpp b/src/binsrv/event/common_header.hpp new file mode 100644 index 0000000..2e9a1d6 --- /dev/null +++ b/src/binsrv/event/common_header.hpp @@ -0,0 +1,68 @@ +#ifndef BINSRV_EVENT_COMMON_HEADER_HPP +#define BINSRV_EVENT_COMMON_HEADER_HPP + +#include "binsrv/event/common_header_fwd.hpp" // IWYU pragma: export + +#include +#include +#include +#include + +#include "binsrv/event/code_type_fwd.hpp" +#include "binsrv/event/flag_type_fwd.hpp" + +#include "util/byte_span_fwd.hpp" + +namespace binsrv::event { + +class [[nodiscard]] common_header { +public: + static constexpr std::size_t size_in_bytes{19U}; + + explicit common_header(util::const_byte_span portion); + + [[nodiscard]] std::uint32_t get_timestamp_raw() const noexcept { + return timestamp_; + } + [[nodiscard]] std::time_t get_timestamp() const noexcept { + return static_cast(get_timestamp_raw()); + } + [[nodiscard]] std::string get_readable_timestamp() const; + + [[nodiscard]] std::uint8_t get_type_code_raw() const noexcept { + return type_code_; + } + [[nodiscard]] code_type get_type_code() const noexcept { + return static_cast(get_type_code_raw()); + } + [[nodiscard]] std::string_view get_readable_type_code() const noexcept; + + [[nodiscard]] std::uint32_t get_server_id_raw() const noexcept { + return server_id_; + } + + [[nodiscard]] std::uint32_t get_event_size_raw() const noexcept { + return event_size_; + } + + [[nodiscard]] std::uint32_t get_next_event_position_raw() const noexcept { + return next_event_position_; + } + + [[nodiscard]] std::uint16_t get_flags_raw() const noexcept { return flags_; } + [[nodiscard]] flag_set get_flags() const noexcept; + [[nodiscard]] std::string get_readable_flags() const; + +private: + // the members are deliberately reordered for better packing + std::uint32_t timestamp_{}; // 0 + std::uint32_t server_id_{}; // 2 + std::uint32_t event_size_{}; // 3 + std::uint32_t next_event_position_{}; // 4 + std::uint16_t flags_{}; // 5 + std::uint8_t type_code_{}; // 1 +}; + +} // namespace binsrv::event + +#endif // BINSRV_EVENT_COMMON_HEADER_HPP diff --git a/src/binsrv/event/common_header_fwd.hpp b/src/binsrv/event/common_header_fwd.hpp new file mode 100644 index 0000000..26759f9 --- /dev/null +++ b/src/binsrv/event/common_header_fwd.hpp @@ -0,0 +1,10 @@ +#ifndef BINSRV_EVENT_COMMON_HEADER_FWD_HPP +#define BINSRV_EVENT_COMMON_HEADER_FWD_HPP + +namespace binsrv::event { + +class common_header; + +} // namespace binsrv::event + +#endif // BINSRV_EVENT_COMMON_HEADER_FWD_HPP diff --git a/src/binsrv/event/empty_body.cpp b/src/binsrv/event/empty_body.cpp new file mode 100644 index 0000000..4627702 --- /dev/null +++ b/src/binsrv/event/empty_body.cpp @@ -0,0 +1,18 @@ +#include "binsrv/event/empty_body.hpp" + +#include +#include + +#include "util/byte_span_fwd.hpp" +#include "util/exception_location_helpers.hpp" + +namespace binsrv::event { + +empty_body::empty_body(util::const_byte_span portion) { + if (std::size(portion) != size_in_bytes) { + util::exception_location().raise( + "invalid event empty body length"); + } +} + +} // namespace binsrv::event diff --git a/src/binsrv/event/empty_body.hpp b/src/binsrv/event/empty_body.hpp new file mode 100644 index 0000000..95ca4a3 --- /dev/null +++ b/src/binsrv/event/empty_body.hpp @@ -0,0 +1,19 @@ +#ifndef BINSRV_EVENT_EMPTY_BODY_HPP +#define BINSRV_EVENT_EMPTY_BODY_HPP + +#include "binsrv/event/empty_post_header_fwd.hpp" // IWYU pragma: export + +#include "util/byte_span_fwd.hpp" + +namespace binsrv::event { + +class [[nodiscard]] empty_body { +public: + static constexpr std::size_t size_in_bytes{0U}; + + explicit empty_body(util::const_byte_span portion); +}; + +} // namespace binsrv::event + +#endif // BINSRV_EVENT_EMPTY_BODY_HPP diff --git a/src/binsrv/event/empty_body_fwd.hpp b/src/binsrv/event/empty_body_fwd.hpp new file mode 100644 index 0000000..da66974 --- /dev/null +++ b/src/binsrv/event/empty_body_fwd.hpp @@ -0,0 +1,10 @@ +#ifndef BINSRV_EVENT_EMPTY_BODY_FWD_HPP +#define BINSRV_EVENT_EMPTY_BODY_FWD_HPP + +namespace binsrv::event { + +class empty_body; + +} // namespace binsrv::event + +#endif // BINSRV_EVENT_EMPTY_BODY_FWD_HPP diff --git a/src/binsrv/event/empty_post_header.cpp b/src/binsrv/event/empty_post_header.cpp new file mode 100644 index 0000000..0f021a6 --- /dev/null +++ b/src/binsrv/event/empty_post_header.cpp @@ -0,0 +1,18 @@ +#include "binsrv/event/empty_post_header.hpp" + +#include +#include + +#include "util/byte_span_fwd.hpp" +#include "util/exception_location_helpers.hpp" + +namespace binsrv::event { + +empty_post_header::empty_post_header(util::const_byte_span portion) { + if (std::size(portion) != size_in_bytes) { + util::exception_location().raise( + "invalid event empty post header length"); + } +} + +} // namespace binsrv::event diff --git a/src/binsrv/event/empty_post_header.hpp b/src/binsrv/event/empty_post_header.hpp new file mode 100644 index 0000000..7a476b1 --- /dev/null +++ b/src/binsrv/event/empty_post_header.hpp @@ -0,0 +1,19 @@ +#ifndef BINSRV_EVENT_EMPTY_POST_HEADER_HPP +#define BINSRV_EVENT_EMPTY_POST_HEADER_HPP + +#include "binsrv/event/empty_post_header_fwd.hpp" // IWYU pragma: export + +#include "util/byte_span_fwd.hpp" + +namespace binsrv::event { + +class [[nodiscard]] empty_post_header { +public: + static constexpr std::size_t size_in_bytes{0U}; + + explicit empty_post_header(util::const_byte_span portion); +}; + +} // namespace binsrv::event + +#endif // BINSRV_EVENT_EMPTY_POST_HEADER_HPP diff --git a/src/binsrv/event/empty_post_header_fwd.hpp b/src/binsrv/event/empty_post_header_fwd.hpp new file mode 100644 index 0000000..11d2bb6 --- /dev/null +++ b/src/binsrv/event/empty_post_header_fwd.hpp @@ -0,0 +1,10 @@ +#ifndef BINSRV_EVENT_EMPTY_POST_HEADER_FWD_HPP +#define BINSRV_EVENT_EMPTY_POST_HEADER_FWD_HPP + +namespace binsrv::event { + +class empty_post_header; + +} // namespace binsrv::event + +#endif // BINSRV_EVENT_EMPTY_POST_HEADER_FWD_HPP diff --git a/src/binsrv/event/event.cpp b/src/binsrv/event/event.cpp new file mode 100644 index 0000000..20e238c --- /dev/null +++ b/src/binsrv/event/event.cpp @@ -0,0 +1,185 @@ +#include "binsrv/event/event.hpp" + +#include +#include +#include +#include +#include +#include +#include + +#include "binsrv/event/checksum_algorithm_type.hpp" +#include "binsrv/event/code_type.hpp" +#include "binsrv/event/flag_type.hpp" +#include "binsrv/event/generic_body.hpp" +#include "binsrv/event/generic_post_header.hpp" + +#include "util/byte_span_fwd.hpp" +#include "util/conversion_helpers.hpp" +#include "util/exception_location_helpers.hpp" + +namespace binsrv::event { + +event::event(util::const_byte_span portion, + const optional_format_description_post_header &fde_post_header, + const optional_format_description_body &fde_body) + : common_header_{ + [](util::const_byte_span event_portion) -> util::const_byte_span { + if (std::size(event_portion) < + decltype(common_header_)::size_in_bytes) { + util::exception_location().raise( + "not enough data for event common header"); + } + return event_portion.subspan( + 0, decltype(common_header_)::size_in_bytes); + }(portion)} { + // TODO: rework with direct member initialization + + const auto type_code = common_header_.get_type_code(); + + std::size_t footer_size{0U}; + if (type_code == code_type::format_description) { + // format_description_events always include event footers with checksums + footer_size = footer::size_in_bytes; + } else { + if (fde_body) { + // if format_description event has already been encountered, we determine + // whether there is a footer in the event from it + footer_size = + (fde_body->get_checksum_algorithm() == checksum_algorithm_type::crc32 + ? footer::size_in_bytes + : 0U); + } else { + // we get in this branch only for the very first artificial rotate event + // and in this case it does not include the footer + footer_size = 0U; + } + } + + const std::size_t event_size = std::size(portion); + if (event_size != common_header_.get_event_size_raw()) { + util::exception_location().raise( + "actual event size does not match the one specified in event common " + "header"); + } + std::size_t post_header_size{0U}; + if (fde_post_header) { + // if format_description event has already been encountered in the stream, + // we take post-header length from it + post_header_size = fde_post_header->get_post_header_length(type_code); + } else { + // we expect that we can receive only 2 events before there is a + // format_description event we can refer to: rotate (with artificial + // flag) and format description event itself + switch (type_code) { + case code_type::rotate: + if (!common_header_.get_flags().has_element(flag_type::artificial)) { + util::exception_location().raise( + "rotate event without preceding format_description event must have " + "'artificial' flag set"); + } + post_header_size = generic_post_header::size_in_bytes; + break; + case code_type::format_description: + post_header_size = + generic_post_header::size_in_bytes; + break; + default: + util::exception_location().raise( + "this type of event must be preceded by a format_description event"); + } + } + + const std::size_t group_size = + common_header::size_in_bytes + post_header_size + footer_size; + if (event_size < group_size) { + util::exception_location().raise( + "not enough data for event post-header + body + footer"); + } + const std::size_t body_size = event_size - group_size; + + const auto post_header_portion = + portion.subspan(common_header::size_in_bytes, post_header_size); + emplace_post_header(type_code, post_header_portion); + + const auto body_portion = portion.subspan( + common_header::size_in_bytes + post_header_size, body_size); + emplace_body(type_code, body_portion); + + if (footer_size != 0U) { + const auto footer_portion = portion.subspan( + common_header::size_in_bytes + post_header_size + body_size, + footer_size); + footer_.emplace(footer_portion); + }; +} + +void event::emplace_post_header(code_type code, util::const_byte_span portion) { + // our goal here is to initialize (emplace) a specific class inside + // 'post_header_' variant (determined via runtime argument 'code') with the + // 'portion' byte range + + // we start with defining an alias for a member function pointer that + // accepts a byte range and performs some modification on an object of this + // class (on 'post_header_' member to be precise) + using emplace_function = void (event::*)(util::const_byte_span); + // then, we define an alias for a container that can store + // '' such member function pointers + using emplace_function_container = + std::array; + // after that we declare a constexpr instance of such array and initialize it + // with immediately invoked lambda + static constexpr emplace_function_container emplace_functions{ + []( + std::index_sequence) -> emplace_function_container { + return {&event::generic_emplace_post_header< + generic_post_header(IndexPack)>>...}; + }(code_index_sequence{})}; + // basically, using 'IndexPack' template parameter expansion this lambda + // will return the following array member function pointers: + // { + // &event::generic_emplace_post_header< + // generic_post_header<0 = unknown>>, + // &event::generic_emplace_post_header< + // generic_post_header<1 = start_v3>>, + // ... + // &event::generic_emplace_post_header< + // generic_post_header> + // } + + // please, notice that we managed to avoid code bloat here by reusing the + // same member function pointer for event codes that have identical + // post header specializations (in this array most of the elements will + // have the '&event::generic_emplace_post_header' + // value) + + // here, we reinterpret the event code as an index in our array + const auto function_index = util::enum_to_index(code); + assert(function_index < util::enum_to_index(code_type::delimiter)); + // all we have to do now is to call a member function pointer determined by + // the 'function_index' offset in the array on 'this' + // this will initialize the 'post_header_' member with expected variant + + // NOLINTNEXTLINE(cppcoreguidelines-pro-bounds-constant-array-index) + (this->*emplace_functions[function_index])(portion); +} + +void event::emplace_body(code_type code, util::const_byte_span portion) { + // here we use the same technique as in 'emplace_post_header()' + using emplace_function = void (event::*)(util::const_byte_span); + using emplace_function_container = + std::array; + static constexpr emplace_function_container emplace_functions{ + []( + std::index_sequence) -> emplace_function_container { + return {&event::generic_emplace_body< + generic_body(IndexPack)>>...}; + }(code_index_sequence{})}; + + const auto function_index = util::enum_to_index(code); + assert(function_index < util::enum_to_index(code_type::delimiter)); + // NOLINTNEXTLINE(cppcoreguidelines-pro-bounds-constant-array-index) + (this->*emplace_functions[function_index])(portion); +} + +} // namespace binsrv::event diff --git a/src/binsrv/event/event.hpp b/src/binsrv/event/event.hpp new file mode 100644 index 0000000..a75332d --- /dev/null +++ b/src/binsrv/event/event.hpp @@ -0,0 +1,124 @@ +#ifndef BINSRV_EVENT_EVENT_HPP +#define BINSRV_EVENT_EVENT_HPP + +#include "binsrv/event/event_fwd.hpp" // IWYU pragma: export + +#include +#include +#include +#include +#include +#include +#include + +#include +#include +#include + +#include "binsrv/event/code_type.hpp" +#include "binsrv/event/common_header.hpp" // IWYU pragma: export +#include "binsrv/event/footer.hpp" // IWYU pragma: export +#include "binsrv/event/format_description_body_impl.hpp" // IWYU pragma: export +#include "binsrv/event/format_description_post_header_impl.hpp" // IWYU pragma: export +#include "binsrv/event/generic_body.hpp" // IWYU pragma: export +#include "binsrv/event/generic_post_header.hpp" // IWYU pragma: export +#include "binsrv/event/rotate_body_impl.hpp" // IWYU pragma: export +#include "binsrv/event/rotate_post_header_impl.hpp" // IWYU pragma: export +#include "binsrv/event/unknown_body.hpp" // IWYU pragma: export +#include "binsrv/event/unknown_post_header.hpp" // IWYU pragma: export + +#include "util/byte_span_fwd.hpp" +#include "util/conversion_helpers.hpp" + +namespace binsrv::event { + +using optional_format_description_post_header = + std::optional>; +using optional_format_description_body = + std::optional>; + +class [[nodiscard]] event { +private: + static constexpr std::size_t number_of_events{ + util::enum_to_index(code_type::delimiter)}; + + // here we create an index sequence (std::index_sequence) specialized + // with the following std::size_t constant pack: 0 .. + using code_index_sequence = std::make_index_sequence; + // almost all boost::mp11 algorithms accept lists of types (rather than + // lists of constants), so we convert std::index_sequence into + // boost::mp11::mp_list of std::integral_constant types + using wrapped_code_index_sequence = + boost::mp11::mp_from_sequence; + + // the following alias template is used later as a boost::mp11 + // metafunction: it transforms an Index (represented via + // std::integral_constant) into one of the post header specializations + template + using index_to_post_header_mf = + generic_post_header(Index::value)>; + // using the list of integral constants and the + // "event code" -> "post header specialization" conversion metafunction, + // we create a type list of all possible post header specializations + // (containing duplicates) + using post_header_type_list = + boost::mp11::mp_transform; + // here we remove all the duplicates from the type list above + using unique_post_header_type_list = + boost::mp11::mp_unique; + // and finally we rename the type list (with unique types) into + // std::variant + using post_header_variant = + boost::mp11::mp_rename; + + // identical techniqure is used to obtain the std::variant of all possible + // unique event bodyies + template + using index_to_body_mf = + generic_body(Index::value)>; + using body_type_list = + boost::mp11::mp_transform; + using unique_body_type_list = boost::mp11::mp_unique; + using body_variant = + boost::mp11::mp_rename; + + using optional_footer = std::optional