MDEV-37605 Extend mariadb-binlog to convert innodb based binary logs to legacy

mysql-test/include/have_innodb_binlog.inc

@@ -1 +1,2 @@
+--source include/not_embedded.inc
 --source include/have_innodb.inc

mysql-test/suite/binlog_in_engine/mysqlbinlog_convert_engine_binlog_basic.result

@@ -0,0 +1,26 @@
+include/reset_master.inc
+set TIMESTAMP= UNIX_TIMESTAMP("1970-01-21 15:32:22");
+*** Generate a small workload into binlog-000000.ibb
+CREATE TABLE t1 (a INT PRIMARY KEY, b INT) ENGINE=InnoDB;
+INSERT INTO t1 VALUES (1, 0), (2, 0), (3, 0);
+UPDATE t1 SET b=1 WHERE a=1;
+DELETE FROM t1 WHERE a=2;
+REPLACE INTO t1 VALUES (3, 3);
+SELECT * FROM t1 ORDER BY a;
+a	b
+1	1
+3	3
+FLUSH BINARY LOGS;
+*** Convert binlog-000000.ibb to legacy format
+*** Converted file contains the synthesized header events
+FOUND 1 /Gtid list/ in conv_listing.txt
+FOUND 1 /Binlog checkpoint/ in conv_listing.txt
+FOUND 1 /Start: binlog v 4/ in conv_listing.txt
+NOT FOUND /Rotate to/ in conv_listing.txt
+*** Round-trip: replay the converted file and compare data
+DROP TABLE t1;
+SELECT * FROM t1 ORDER BY a;
+a	b
+1	1
+3	3
+DROP TABLE t1;

mysql-test/suite/binlog_in_engine/mysqlbinlog_convert_engine_binlog_basic.test

@@ -0,0 +1,63 @@
+# mysqlbinlog_convert.test
+#
+# Purpose:
+#   Basic test for MDEV-37605: mariadb-binlog conversion of InnoDB-format
+#   binlogs (.ibb) into legacy-format binary log 
+#
+
+--source include/have_innodb_binlog.inc
+
+--let $datadir= `SELECT @@datadir`
+
+--source include/reset_master.inc
+
+# Fixed timestamp for deterministic event listings (with --timezone=GMT-3
+# from the -master.opt file, matching binlog_in_engine.mysqlbinlog).
+set TIMESTAMP= UNIX_TIMESTAMP("1970-01-21 15:32:22");
+
+--echo *** Generate a small workload into binlog-000000.ibb
+CREATE TABLE t1 (a INT PRIMARY KEY, b INT) ENGINE=InnoDB;
+INSERT INTO t1 VALUES (1, 0), (2, 0), (3, 0);
+UPDATE t1 SET b=1 WHERE a=1;
+DELETE FROM t1 WHERE a=2;
+REPLACE INTO t1 VALUES (3, 3);
+SELECT * FROM t1 ORDER BY a;
+
+# Rotate so binlog-000000.ibb is complete on disk. Waiting for the *next*
+# pre-allocated file (binlog-000002.ibb, fully pre-allocated and empty)
+# guarantees both 000000 and the new active 000001 exist.
+FLUSH BINARY LOGS;
+--let $binlog_name= binlog-000002.ibb
+--let $binlog_size= 262144
+--source include/wait_for_engine_binlog.inc
+
+--echo *** Convert binlog-000000.ibb to legacy format
+--exec $MYSQL_BINLOG --convert-engine-binlog --result-file=$MYSQL_TMP_DIR/conv $datadir/binlog-000000.ibb
+
+
+--echo *** Converted file contains the synthesized header events
+--exec $MYSQL_BINLOG --verbose $MYSQL_TMP_DIR/conv.000001 > $MYSQL_TMP_DIR/conv_listing.txt
+--let SEARCH_FILE= $MYSQL_TMP_DIR/conv_listing.txt
+--let SEARCH_PATTERN= Gtid list
+--source include/search_pattern_in_file.inc
+--let SEARCH_PATTERN= Binlog checkpoint
+--source include/search_pattern_in_file.inc
+--let SEARCH_PATTERN= Start: binlog v 4
+--source include/search_pattern_in_file.inc
+# it should not contain the Rotate event in the converted file
+--let SEARCH_PATTERN= Rotate to
+--source include/search_pattern_in_file.inc
+--remove_file $MYSQL_TMP_DIR/conv_listing.txt
+
+--echo *** Round-trip: replay the converted file and compare data
+# The replayed GTIDs duplicate the originals, so the replay must not use
+# GTID strict mode (same as in binlog_in_engine.mysqlbinlog).
+--exec $MYSQL_BINLOG --gtid-strict-mode=0 $MYSQL_TMP_DIR/conv.000001 > $MYSQLTEST_VARDIR/tmp/convert_replay.sql
+DROP TABLE t1;
+--exec $MYSQL --abort-source-on-error -e "source $MYSQLTEST_VARDIR/tmp/convert_replay.sql;" test
+--remove_file $MYSQLTEST_VARDIR/tmp/convert_replay.sql
+SELECT * FROM t1 ORDER BY a;
+
+# Cleanup
+--remove_file $MYSQL_TMP_DIR/conv.000001
+DROP TABLE t1;

mysql-test/suite/binlog_in_engine/mysqlbinlog_convert_engine_binlog_gtid_list_event.result

@@ -0,0 +1,48 @@
+SET @@session.sql_log_bin= 0;
+CREATE TABLE t1 (a INT PRIMARY KEY) ENGINE=InnoDB;
+CREATE TABLE t2 (a INT PRIMARY KEY) ENGINE=InnoDB;
+CREATE TABLE t3 (a INT PRIMARY KEY) ENGINE=InnoDB;
+CREATE TABLE t4 (a INT PRIMARY KEY) ENGINE=InnoDB;
+SET @@session.sql_log_bin= 1;
+include/reset_master.inc
+*** Generate GTIDs across several InnoDB binlog files
+SET @@session.gtid_domain_id= 0;
+SET @@session.server_id= 1;
+INSERT INTO t1 VALUES (2);
+SET @@session.gtid_domain_id= 1;
+SET @@session.server_id= 2;
+INSERT INTO t2 VALUES (1);
+FLUSH BINARY LOGS;
+SET @@session.gtid_domain_id= 0;
+SET @@session.server_id= 1;
+INSERT INTO t1 VALUES (1);
+SET @@session.gtid_domain_id= 2;
+SET @@session.server_id= 3;
+INSERT INTO t3 VALUES (1);
+FLUSH BINARY LOGS;
+SET @@session.gtid_domain_id= 1;
+SET @@session.server_id= 1;
+INSERT INTO t2 VALUES (2);
+SET @@session.gtid_domain_id= 3;
+SET @@session.server_id= 4;
+INSERT INTO t4 VALUES (1);
+# restart
+SET @@session.gtid_domain_id= 0;
+SET @@session.server_id= 1;
+INSERT INTO t1 VALUES (3);
+FLUSH BINARY LOGS;
+*** Convert the generated InnoDB binlogs to legacy format
+*** Verify synthesized Gtid_list events in converted legacy binlogs
+FOUND 1 /Gtid list \[\]/ in gtid_conv_listing.txt
+FOUND 1 /Gtid list \[0-1-1,\n# 1-2-1\]/ in gtid_conv_listing.txt
+FOUND 1 /Gtid list \[0-1-2,\n# 1-2-1,\n# 2-3-1\]/ in gtid_conv_listing.txt
+FOUND 1 /Rotate to gtid_conv.000004/ in gtid_conv_listing.txt
+FOUND 1 /Gtid list \[0-1-2,\n# 1-2-1,\n# 1-1-2,\n# 2-3-1,\n# 3-4-1\]/ in gtid_conv_listing.txt
+*** Convert the generated InnoDB binlog(binlog-000002.ibb) to legacy format in random order
+*** Verify synthesized Gtid_list events in converted legacy binlogs
+NOT FOUND /Gtid list \[0-1-2,\n# 1-2-1,\n# 1-1-2,\n# 2-3-1,\n# 3-4-1\]/ in gtid_conv_listing.txt
+FOUND 1 /Rotate to gtid_conv.000002/ in gtid_conv_listing.txt
+DROP TABLE t1;
+DROP TABLE t2;
+DROP TABLE t3;
+DROP TABLE t4;

mysql-test/suite/binlog_in_engine/mysqlbinlog_convert_engine_binlog_gtid_list_event.test

@@ -0,0 +1,139 @@
+#
+# Purpose:
+#   Verify that mariadb-binlog populates synthesized GTID_LIST_EVENTs correctly
+#   when converting InnoDB-format binlogs (.ibb) to legacy binlog files.
+#   Also verify that the generated legacy binlogs contain the expected ROTATE_EVENT.
+#
+# Methodology:
+#   Generate several .ibb files with GTIDs across multiple domains and server
+#   ids, including GTIDs that are not ordered by server id. Restart the server
+#   in the middle so the converter sees another FORMAT_DESCRIPTION_EVENT and
+#   rotates the generated legacy output. Convert all completed .ibb files and
+#   verify each generated legacy binlog contains the expected Gtid_list state.
+#   Convert the .ibb files again in random order and verify the generated 
+#   legacy binlogs contain the expected Gtid_list state.
+#
+
+--source include/have_binlog_format_row.inc
+--source include/have_innodb_binlog.inc
+
+--let $datadir= `SELECT @@datadir`
+
+# start of binlog-000000.ibb
+
+SET @@session.sql_log_bin= 0;
+CREATE TABLE t1 (a INT PRIMARY KEY) ENGINE=InnoDB;
+CREATE TABLE t2 (a INT PRIMARY KEY) ENGINE=InnoDB;
+CREATE TABLE t3 (a INT PRIMARY KEY) ENGINE=InnoDB;
+CREATE TABLE t4 (a INT PRIMARY KEY) ENGINE=InnoDB;
+SET @@session.sql_log_bin= 1;
+
+--source include/reset_master.inc
+
+--echo *** Generate GTIDs across several InnoDB binlog files
+SET @@session.gtid_domain_id= 0;
+SET @@session.server_id= 1;
+INSERT INTO t1 VALUES (2);
+
+SET @@session.gtid_domain_id= 1;
+SET @@session.server_id= 2;
+INSERT INTO t2 VALUES (1);
+
+# end of binlog-000000.ibb which will be converted to gtid_conv.000001
+
+FLUSH BINARY LOGS;
+
+# start of binlog-000001.ibb
+
+SET @@session.gtid_domain_id= 0;
+SET @@session.server_id= 1;
+INSERT INTO t1 VALUES (1);
+
+SET @@session.gtid_domain_id= 2;
+SET @@session.server_id= 3;
+INSERT INTO t3 VALUES (1);
+
+# end of binlog-000001.ibb which will be converted to gtid_conv.000002
+
+FLUSH BINARY LOGS;
+
+# start of binlog-000002.ibb
+
+SET @@session.gtid_domain_id= 1;
+SET @@session.server_id= 1;
+INSERT INTO t2 VALUES (2);
+
+SET @@session.gtid_domain_id= 3;
+SET @@session.server_id= 4;
+INSERT INTO t4 VALUES (1);
+
+# this will trigger a ROTATE_EVENT and will be converted to/end of gtid_conv.000003
+
+--source include/restart_mysqld.inc
+
+# start of converted gtid_conv.000004
+
+SET @@session.gtid_domain_id= 0;
+SET @@session.server_id= 1;
+INSERT INTO t1 VALUES (3);
+
+# end of binlog-000002.ibb which will be converted to gtid_conv.000004
+FLUSH BINARY LOGS;
+
+--let $binlog_name= binlog-000003.ibb
+--let $binlog_size= 262144
+--source include/wait_for_engine_binlog.inc
+
+
+--echo *** Convert the generated InnoDB binlogs to legacy format
+--exec $MYSQL_BINLOG --convert-engine-binlog --result-file=$MYSQL_TMP_DIR/gtid_conv $datadir/binlog-000000.ibb $datadir/binlog-000001.ibb $datadir/binlog-000002.ibb
+
+--echo *** Verify synthesized Gtid_list events in converted legacy binlogs
+--exec $MYSQL_BINLOG --verbose $MYSQL_TMP_DIR/gtid_conv.000001 > $MYSQL_TMP_DIR/gtid_conv_listing.txt
+--let SEARCH_FILE= $MYSQL_TMP_DIR/gtid_conv_listing.txt
+--let SEARCH_PATTERN= Gtid list \[\]
+--source include/search_pattern_in_file.inc
+
+--exec $MYSQL_BINLOG --verbose $MYSQL_TMP_DIR/gtid_conv.000002 > $MYSQL_TMP_DIR/gtid_conv_listing.txt
+--let SEARCH_PATTERN= Gtid list \[0-1-1,\n# 1-2-1\]
+--source include/search_pattern_in_file.inc
+
+--exec $MYSQL_BINLOG --verbose $MYSQL_TMP_DIR/gtid_conv.000003 > $MYSQL_TMP_DIR/gtid_conv_listing.txt
+--let SEARCH_PATTERN= Gtid list \[0-1-2,\n# 1-2-1,\n# 2-3-1\]
+--source include/search_pattern_in_file.inc
+
+--let SEARCH_PATTERN= Rotate to gtid_conv.000004
+--source include/search_pattern_in_file.inc
+
+--exec $MYSQL_BINLOG --verbose $MYSQL_TMP_DIR/gtid_conv.000004 > $MYSQL_TMP_DIR/gtid_conv_listing.txt
+--let SEARCH_PATTERN= Gtid list \[0-1-2,\n# 1-2-1,\n# 1-1-2,\n# 2-3-1,\n# 3-4-1\]
+--source include/search_pattern_in_file.inc
+
+--remove_file $MYSQL_TMP_DIR/gtid_conv_listing.txt
+--remove_file $MYSQL_TMP_DIR/gtid_conv.000001
+--remove_file $MYSQL_TMP_DIR/gtid_conv.000002
+--remove_file $MYSQL_TMP_DIR/gtid_conv.000003
+--remove_file $MYSQL_TMP_DIR/gtid_conv.000004
+
+
+--echo *** Convert the generated InnoDB binlog(binlog-000002.ibb) to legacy format in random order
+--exec $MYSQL_BINLOG --convert-engine-binlog --result-file=$MYSQL_TMP_DIR/gtid_conv $datadir/binlog-000002.ibb
+
+--echo *** Verify synthesized Gtid_list events in converted legacy binlogs
+--exec $MYSQL_BINLOG --verbose $MYSQL_TMP_DIR/gtid_conv.000001 > $MYSQL_TMP_DIR/gtid_conv_listing.txt
+--let SEARCH_FILE= $MYSQL_TMP_DIR/gtid_conv_listing.txt
+--let SEARCH_PATTERN= Gtid list \[0-1-2,\n# 1-2-1,\n# 1-1-2,\n# 2-3-1,\n# 3-4-1\]
+# TODO: Tarun this search will fail now as this functionality is not yet implemented
+--source include/search_pattern_in_file.inc
+
+--let SEARCH_PATTERN= Rotate to gtid_conv.000002
+--source include/search_pattern_in_file.inc
+
+--remove_file $MYSQL_TMP_DIR/gtid_conv_listing.txt
+--remove_file $MYSQL_TMP_DIR/gtid_conv.000001
+
+
+DROP TABLE t1;
+DROP TABLE t2;
+DROP TABLE t3;
+DROP TABLE t4;

mysql-test/suite/binlog_in_engine/mysqlbinlog_convert_engine_binlog_max_size.result

@@ -0,0 +1,15 @@
+include/reset_master.inc
+CREATE TABLE t1 (a INT PRIMARY KEY, b LONGTEXT) ENGINE=InnoDB;
+INSERT INTO t1 VALUES (1, REPEAT('a', 48 * 1024));
+INSERT INTO t1 VALUES (2, REPEAT('b', 48 * 1024));
+INSERT INTO t1 VALUES (3, REPEAT('c', 48 * 1024));
+FLUSH BINARY LOGS;
+*** Convert with a 32K maximum output binlog size
+*** Rotated files form a replayable binlog sequence
+DROP TABLE t1;
+SELECT a, LENGTH(b) FROM t1 ORDER BY a;
+a	LENGTH(b)
+1	49152
+2	49152
+3	49152
+DROP TABLE t1;

mysql-test/suite/binlog_in_engine/mysqlbinlog_convert_engine_binlog_max_size.test

@@ -0,0 +1,40 @@
+--source include/have_binlog_format_row.inc
+--source include/have_innodb_binlog.inc
+
+--let $datadir= `SELECT @@datadir`
+
+--source include/reset_master.inc
+
+
+CREATE TABLE t1 (a INT PRIMARY KEY, b LONGTEXT) ENGINE=InnoDB;
+INSERT INTO t1 VALUES (1, REPEAT('a', 48 * 1024));
+INSERT INTO t1 VALUES (2, REPEAT('b', 48 * 1024));
+INSERT INTO t1 VALUES (3, REPEAT('c', 48 * 1024));
+
+# Complete binlog-000000.ibb and wait until it is available on disk.
+FLUSH BINARY LOGS;
+--let $binlog_name= binlog-000002.ibb
+--let $binlog_size= 262144
+--source include/wait_for_engine_binlog.inc
+
+--echo *** Convert with a 32K maximum output binlog size
+--exec $MYSQL_BINLOG --convert-engine-binlog --max-binlog-size=32768 --result-file=$MYSQL_TMP_DIR/max_size_conv $datadir/binlog-000000.ibb
+
+# Rotation is checked at transaction boundaries. The first large INSERT stays
+# in the initial file; each following INSERT starts a new output file.
+--file_exists $MYSQL_TMP_DIR/max_size_conv.000001
+--file_exists $MYSQL_TMP_DIR/max_size_conv.000002
+--file_exists $MYSQL_TMP_DIR/max_size_conv.000003
+
+--echo *** Rotated files form a replayable binlog sequence
+--exec $MYSQL_BINLOG --gtid-strict-mode=0 $MYSQL_TMP_DIR/max_size_conv.000001 $MYSQL_TMP_DIR/max_size_conv.000002 $MYSQL_TMP_DIR/max_size_conv.000003 > $MYSQLTEST_VARDIR/tmp/max_size_replay.sql
+DROP TABLE t1;
+--exec $MYSQL --abort-source-on-error -e "source $MYSQLTEST_VARDIR/tmp/max_size_replay.sql;" test
+--remove_file $MYSQLTEST_VARDIR/tmp/max_size_replay.sql
+
+SELECT a, LENGTH(b) FROM t1 ORDER BY a;
+
+--remove_file $MYSQL_TMP_DIR/max_size_conv.000001
+--remove_file $MYSQL_TMP_DIR/max_size_conv.000002
+--remove_file $MYSQL_TMP_DIR/max_size_conv.000003
+DROP TABLE t1;