PS-9244 fix: binlog_server fails to resume copying binlog file after …

…server restart (#50)

https://perconadev.atlassian.net/browse/PS-9244

Binlog Server Utility now properly handles 'STOP' events which may be
written to the binlog when MySQL server shuts down.

Added specializations of the 'binsrv::event::generic_body_impl<>'  and
'binsrv::event::generic_post_header_impl<>' for 'code_type::stop'
(both implemented via 'redirect_type' to 'empty_body' /
'empty_post_header'). 'STOP' is now a known event type for the
'binsrv::event::event' class and its inner body / post-header variants.

Updated 'binsrv::reader_context'- it is now based on the following state
machine:
(ROTATE(artificial) FORMAT_DESCRIPTION <ANY>* (ROTATE|STOP))*

'process_binlog_event()' in the main application now calls
'close_binlog()' not only on real (non-artificial) 'ROTATE' events but
on 'STOP' events as well.

Introduced new 'binlog_streaming.pull_mode' MTR test case which checks
if the Binlog Server Utility properly handles reconnects and resuming
operation in background ('pull') mode.
Common functionality from the 'binlog_streaming.binsrv' and
'binlog_streaming.pull_mode' extracted into the following include files:
* 'identify_storage_backend.inc' - for identifying whether a local
  filesystem ('file') or a AWS S3 ('s3') storage backend should be used.
* 'set_up_binsrv_environment.inc' - for creating storage directory,
  configuring log file and generating JSON configuration.
* 'tear_down_binsrv_environment.inc' - for cleaning up data directory
  and removing log file along with JSON configuration file.

Temporarily disabled running MTR under "Clang 17 ASan" in GitHub
Workflows because of the "-stdlib=libc++ -fsanitize=address"
alloc-dealloc-mismatch issue
(llvm/llvm-project#59432).
Hopefully, the problem will be fixed in clang-18.

.github/workflows/cmake.yml

@@ -96,7 +96,7 @@ jobs:
             aws_cmake_flags: "-DCMAKE_CXX_FLAGS_INIT=-stdlib=libc++",
             boost_cmake_flags: "-DCMAKE_CXX_FLAGS_INIT=-stdlib=libc++",
             label: "Debug-clang17",
-            run_clang_tidy: true,
+            run_clang_tidy: true
           }
         - {
             name: "Clang 17 RelWithDebInfo",
@@ -107,7 +107,7 @@ jobs:
             aws_cmake_flags: "-DCMAKE_CXX_FLAGS_INIT=-stdlib=libc++",
             boost_cmake_flags: "-DCMAKE_CXX_FLAGS_INIT=-stdlib=libc++",
             label: "RelWithDebInfo-clang17",
-            run_clang_tidy: true,
+            run_clang_tidy: true
           }
         - {
             name: "Clang 17 ASan",
@@ -118,9 +118,13 @@ jobs:
             aws_cmake_flags: "-DCMAKE_CXX_FLAGS_INIT=-stdlib=libc++ -DENABLE_ADDRESS_SANITIZER=ON",
             boost_cmake_flags: "-DCMAKE_CXX_FLAGS_INIT=\"-stdlib=libc++ -fsanitize=address\"",
             sanitizer_cmake_flags: "-DWITH_ASAN=ON",
-            label: "ASan-clang17",
-            run_mtr: true,
-            mtr_options: "--sanitize"
+            label: "ASan-clang17"
+            # TODO: re-enable running MTR under "Clang 17 ASan"
+            #   run_mtr: true,
+            #   mtr_options: "--sanitize"
+            # when "-stdlib=libc++ -fsanitize=address" alloc-dealloc-mismatch issue is fixed
+            # (https://github.com/llvm/llvm-project/issues/59432)
+            # or CI is upgraded to Clang 18
           }
 
     steps:

CMakeLists.txt

@@ -143,6 +143,12 @@ set(source_files
   src/binsrv/event/rotate_post_header_impl.hpp
   src/binsrv/event/rotate_post_header_impl.cpp
 
+  src/binsrv/event/stop_body_impl_fwd.hpp
+  src/binsrv/event/stop_body_impl.hpp
+
+  src/binsrv/event/stop_post_header_impl_fwd.hpp
+  src/binsrv/event/stop_post_header_impl.hpp
+
   src/binsrv/event/unknown_body_fwd.hpp
   src/binsrv/event/unknown_body.hpp
   src/binsrv/event/unknown_body.cpp

mtr/binlog_streaming/include/diff_with_storage_object.inc

@@ -2,18 +2,18 @@
 # Compares a file on a local filesystem with an object from the backend storage.
 #
 # Usage:
-# --let $storage_backend = file
-# --let $local_file = $MYSQL_TMP_DIR/first
-# --let $storage_object = $MYSQL_TMP_DIR/second
-# --source diff_with_storage_object.inc
+#   --let $storage_backend = file
+#   --let $local_file = $MYSQL_TMP_DIR/first
+#   --let $storage_object = $MYSQL_TMP_DIR/second
+#   --source diff_with_storage_object.inc
 #
 # or
-# --let $aws_cli = AWS_ACCESS_KEY_ID=... AWS_SECRET_ACCESS_KEY=... aws
-# --let $storage_backend = s3
-# --let $local_file = $MYSQL_TMP_DIR/first
-# --let $aws_s3_bucket = my-bucket
-# --let $storage_object = /vault/second
-# --source diff_with_storage_object.inc
+#   --let $aws_cli = AWS_ACCESS_KEY_ID=... AWS_SECRET_ACCESS_KEY=... aws
+#   --let $storage_backend = s3
+#   --let $local_file = $MYSQL_TMP_DIR/first
+#   --let $aws_s3_bucket = my-bucket
+#   --let $storage_object = /vault/second
+#   --source diff_with_storage_object.inc
 #
 # $storage_backend - determines stortage backend type (either 'file' or 'fs')
 # $aws_cli         - path to AWS command line interface (cli) tools with AWS_ACCESS_KEY_ID /

mtr/binlog_streaming/include/identify_storage_backend.inc

@@ -0,0 +1,23 @@
+# The following environment variables must be defined to use AWS S3 as a
+# storage backend:
+# - $MTR_BINSRV_AWS_ACCESS_KEY_ID
+# - $MTR_BINSRV_AWS_SECRET_ACCESS_KEY
+# - $MTR_BINSRV_AWS_S3_BUCKET
+# - $MTR_BINSRV_AWS_S3_REGION (optional)
+
+--let $storage_backend = file
+if ($MTR_BINSRV_AWS_ACCESS_KEY_ID != '')
+{
+  if ($MTR_BINSRV_AWS_SECRET_ACCESS_KEY != '')
+  {
+    if ($MTR_BINSRV_AWS_S3_BUCKET != '')
+    {
+      --let $storage_backend = s3
+      --let $aws_cli = AWS_ACCESS_KEY_ID=$MTR_BINSRV_AWS_ACCESS_KEY_ID AWS_SECRET_ACCESS_KEY=$MTR_BINSRV_AWS_SECRET_ACCESS_KEY aws
+      if ($MTR_BINSRV_AWS_S3_REGION != '')
+      {
+        --let $aws_cli = $aws_cli --region $MTR_BINSRV_AWS_S3_REGION
+      }
+    }
+  }
+}

mtr/binlog_streaming/include/set_up_binsrv_environment.inc

@@ -0,0 +1,94 @@
+#
+# Creates a data directory and a JSON configuration file for the Binlog Server Utility
+#
+# Usage:
+#   --let $binsrv_connect_timeout = 20
+#   --let $binsrv_read_timeout = 60
+#   --let $binsrv_idle_time = 10
+#   --source set_up_binsrv_environment.inc
+
+--echo
+--echo *** Generating a configuration file in JSON format for the Binlog
+--echo *** Server utility.
+
+# temporarily disabling MySQL general query log so that AWS credentials
+# will not appear in plain in recorded SQL queries
+--disable_query_log
+SET @old_sql_log_off = @@sql_log_off;
+SET sql_log_off = ON;
+
+if ($storage_backend == file)
+{
+  --let $binsrv_storage_path = $MYSQL_TMP_DIR/storage
+  eval SET @storage_uri = CONCAT('file://', '$binsrv_storage_path');
+}
+if ($storage_backend == s3)
+{
+  --let $qualified_bucket = $MTR_BINSRV_AWS_S3_BUCKET
+  if ($MTR_BINSRV_AWS_S3_REGION)
+  {
+    --let $qualified_bucket = $qualified_bucket.$MTR_BINSRV_AWS_S3_REGION
+  }
+  --let $binsrv_storage_path = `SELECT CONCAT('/mtr-', UUID())`
+  eval SET @storage_uri = CONCAT('s3://', '$MTR_BINSRV_AWS_ACCESS_KEY_ID', ':', '$MTR_BINSRV_AWS_SECRET_ACCESS_KEY', '@', '$qualified_bucket',  '$binsrv_storage_path');
+  --let $aws_s3_bucket = $MTR_BINSRV_AWS_S3_BUCKET
+}
+
+--let $binsrv_log_path = $MYSQL_TMP_DIR/binsrv_utility.log
+eval SET @log_path = '$binsrv_log_path';
+
+SET @delimiter_pos = INSTR(USER(), '@');
+SET @connection_user = SUBSTRING(USER(), 1, @delimiter_pos - 1);
+SET @connection_host = SUBSTRING(USER(), @delimiter_pos + 1);
+SET @connection_host = IF(@connection_host = 'localhost', '127.0.0.1', @connection_host);
+
+eval SET @binsrv_config_json = JSON_OBJECT(
+  'logger', JSON_OBJECT(
+    'level', 'trace',
+    'file', @log_path
+  ),
+  'connection', JSON_OBJECT(
+     'host', @connection_host,
+     'port', @@global.port,
+     'user', @connection_user,
+     'password', '',
+     'connect_timeout', $binsrv_connect_timeout,
+     'read_timeout', $binsrv_read_timeout,
+     'write_timeout', 60
+  ),
+  'replication', JSON_OBJECT(
+     'server_id', @@server_id + 1,
+     'idle_time', $binsrv_idle_time
+  ),
+  'storage', JSON_OBJECT(
+     'uri', @storage_uri
+  )
+);
+
+--let $binsrv_config_file_path = $MYSQL_TMP_DIR/binsrv_config.json
+--let $write_var = `SELECT @binsrv_config_json`
+--let $write_to_file = $binsrv_config_file_path
+--source include/write_var_to_file.inc
+
+SET sql_log_off = @old_sql_log_off;
+--enable_query_log
+
+--echo
+--echo *** Determining binlog file directory from the server.
+--disable_query_log
+SET @path_separator = '/';
+--source include/check_windows.inc
+if ($have_windows) {
+  SET @path_separator = '\\';
+}
+--let $binlog_base_dir = `SELECT LEFT(@@global.log_bin_basename, CHAR_LENGTH(@@global.log_bin_basename) - CHAR_LENGTH(SUBSTRING_INDEX(@@global.log_bin_basename, @path_separator, -1)))`
+--enable_query_log
+
+
+--echo
+--echo *** Creating a temporary directory <BINSRV_STORAGE_PATH> for storing
+--echo *** binlog files downloaded via the Binlog Server utility.
+if ($storage_backend == file)
+{
+  --mkdir $binsrv_storage_path
+}

mtr/binlog_streaming/include/tear_down_binsrv_environment.inc

@@ -0,0 +1,18 @@
+--echo
+--echo *** Removing the Binlog Server utility storage directory.
+if ($storage_backend == file)
+{
+  --force-rmdir $binsrv_storage_path
+}
+if ($storage_backend == s3)
+{
+  --exec $aws_cli s3 rm s3://$aws_s3_bucket$binsrv_storage_path/ --recursive > /dev/null
+}
+
+--echo
+--echo *** Removing the Binlog Server utility log file.
+--remove_file $binsrv_log_path
+
+--echo
+--echo *** Removing the Binlog Server utility configuration file.
+--remove_file $binsrv_config_file_path

mtr/binlog_streaming/r/binsrv.result

@@ -1,11 +1,8 @@
-*** Flushing binary logs at the very beginning of the test.
-FLUSH BINARY LOGS;
+*** Resetting replication at the very beginning of the test.
+RESET MASTER;
 
 *** Determining the first fresh binary log name.
 
-*** Purging all binary logs before the first fresh one.
-PURGE BINARY LOGS TO '<FIRST_BINLOG>';
-
 *** Creating a simple table and filling it with some data.
 CREATE TABLE t1(id INT UNSIGNED NOT NULL AUTO_INCREMENT, PRIMARY KEY(id)) ENGINE=InnoDB;
 INSERT INTO t1 VALUES(DEFAULT);

mtr/binlog_streaming/r/pull_mode.result

@@ -0,0 +1,55 @@
+*** Resetting replication at the very beginning of the test.
+RESET MASTER;
+
+*** Generating a configuration file in JSON format for the Binlog
+*** Server utility.
+
+*** Determining binlog file directory from the server.
+
+*** Creating a temporary directory <BINSRV_STORAGE_PATH> for storing
+*** binlog files downloaded via the Binlog Server utility.
+
+*** Starting Binlog Server Utility in background in pull mode
+include/read_file_to_var.inc
+
+*** Determining the first fresh binary log name.
+
+*** Creating a simple table and filling it with some data.
+CREATE TABLE t1(id INT UNSIGNED NOT NULL AUTO_INCREMENT, PRIMARY KEY(id)) ENGINE=InnoDB;
+INSERT INTO t1 VALUES(DEFAULT);
+
+*** Restarting the server with a pause to test for the STOP event in
+*** the binary log and reconnection logic.
+# restart
+
+*** Determining the second binary log name.
+
+*** Filling the table with some more data and dropping the table.
+INSERT INTO t1 VALUES(DEFAULT);
+DROP TABLE t1;
+
+*** FLUSHING the binlog one more time to make sure that the second one
+*** is no longer open.
+FLUSH BINARY LOGS;
+
+*** Determining the third binary log name.
+
+*** Waiting till Binlog Server Utility starts processing the third
+*** binary log.
+
+*** Sending SIGTERM signal to the Binlog Server Utility and waiting for
+*** the process to terminate
+
+*** Checking that the Binlog Server utility detected an empty storage
+include/assert_grep.inc [Binlog storage must be initialized on an empty directory]
+
+*** Comparing server and downloaded versions of the first binlog file
+
+*** Comparing server and downloaded versions of the second binlog file
+
+*** Removing the Binlog Server utility storage directory.
+
+*** Removing the Binlog Server utility log file.
+
+*** Removing the Binlog Server utility configuration file.
+KILL CONNECTION <CONNECTION_ID>;