From 0ca32faeecd245535e6e288c1289d9c0dcdb4abb Mon Sep 17 00:00:00 2001 From: Ivan Lazarev Date: Thu, 19 May 2022 05:36:33 +0300 Subject: [PATCH 1/5] PBCKP-145 added backup size-check test --- tests/exclude.py | 88 ++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 88 insertions(+) diff --git a/tests/exclude.py b/tests/exclude.py index b98a483d0..663e59da5 100644 --- a/tests/exclude.py +++ b/tests/exclude.py @@ -261,6 +261,94 @@ def test_exclude_unlogged_tables_2(self): # Clean after yourself self.del_test_dir(module_name, fname) + # @unittest.skip("skip") + #TODO REVIEW time consuming test, but not more than 25 secs and 850M disk space :( + def test_exclude_unlogged_table_and_check_backup_size_unchanged(self): + """ + - make backup on empty db, capture full backup stats + - fill unlogged table by 160M, capture "full" backup, check its size is almost the same as for empty one + - increase unlogged table by 160M again, check "delta" backup size increment is less than 1M + - increase unlogged table by 160M again, check "page" backup size increment is less than 1M + """ + + fname = self.id().split('.')[3] + backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + node = self.make_simple_node( + base_dir=os.path.join(module_name, fname, 'node'), + set_replication=True, + initdb_params=['--data-checksums'], + pg_options={ + "shared_buffers": "10MB"}) + + self.init_pb(backup_dir) + self.add_instance(backup_dir, 'node', node) + self.set_archiving(backup_dir, 'node', node) + node.slow_start() + + # init full backup and stats on empty db + backup_id_empty = self.backup_node( + backup_dir, 'node', node, backup_type="full", + return_id=True, + options=["-j", "4", "--stream"] + ) + + show_pb_empty = self.show_pb( + backup_dir, 'node', backup_id=backup_id_empty) # ['recovery-time'] + + # fill unlogged table by 400M, ensure second "full" backup is almost the same size + node.safe_psql( + "postgres", + "create unlogged table t_logged as select i" + " as id from generate_series(0,4005000) i") + + backup_id_full = self.backup_node( + backup_dir, 'node', node, backup_type="full", + return_id=True, + options=["-j", "4", "--stream"] + ) + + show_pb_full = self.show_pb( + backup_dir, 'node', backup_id=backup_id_full) + + self.assertTrue(show_pb_full["data-bytes"] - show_pb_empty["data-bytes"] < 1024*1024) + + # ensure "delta" backup skips 400M increment to unlogged table + node.safe_psql( + "postgres", + "insert into t_logged " + " select from generate_series(0,4005000)") + + backup_id_delta = self.backup_node( + backup_dir, 'node', node, backup_type="delta", + return_id=True, + options=["-j", "4", "--stream"] + ) + + show_pb_delta = self.show_pb( + backup_dir, 'node', backup_id=backup_id_delta) + + self.assertTrue(show_pb_delta["data-bytes"] < 1024*1024) + + # ensure "page" backup skips 400M increment to unlogged table + node.safe_psql( + "postgres", + "insert into t_logged " + " select from generate_series(0,4005000)") + + backup_id_page = self.backup_node( + backup_dir, 'node', node, backup_type="page", + return_id=True, + options=["-j", "4", "--stream"] + ) + + show_pb_page = self.show_pb( + backup_dir, 'node', backup_id=backup_id_page) + show_archive = self.show_archive(backup_dir, 'node') + self.assertTrue(show_pb_page["data-bytes"] < 1024*1024) + + # Clean after yourself + self.del_test_dir(module_name, fname) + # @unittest.skip("skip") def test_exclude_log_dir(self): """ From 20fe51647be6866051e3ca51dba3ccc5797fdeb8 Mon Sep 17 00:00:00 2001 From: Ivan Lazarev Date: Thu, 19 May 2022 13:03:39 +0300 Subject: [PATCH 2/5] PBCKP-145 fixup --- tests/exclude.py | 1 - 1 file changed, 1 deletion(-) diff --git a/tests/exclude.py b/tests/exclude.py index 663e59da5..931dfc0ca 100644 --- a/tests/exclude.py +++ b/tests/exclude.py @@ -343,7 +343,6 @@ def test_exclude_unlogged_table_and_check_backup_size_unchanged(self): show_pb_page = self.show_pb( backup_dir, 'node', backup_id=backup_id_page) - show_archive = self.show_archive(backup_dir, 'node') self.assertTrue(show_pb_page["data-bytes"] < 1024*1024) # Clean after yourself From b09b22c103b7ad068298dac263539ce0eb76d336 Mon Sep 17 00:00:00 2001 From: Ivan Lazarev Date: Fri, 20 May 2022 05:37:01 +0300 Subject: [PATCH 3/5] PBCKP-145 added table restore check. unlogged table should be restored but always empty. --- tests/exclude.py | 105 +++++++++-------------------------------------- 1 file changed, 20 insertions(+), 85 deletions(-) diff --git a/tests/exclude.py b/tests/exclude.py index 931dfc0ca..456f2969e 100644 --- a/tests/exclude.py +++ b/tests/exclude.py @@ -203,8 +203,10 @@ def test_exclude_unlogged_tables_1(self): # @unittest.skip("skip") def test_exclude_unlogged_tables_2(self): """ - make node, create unlogged, take FULL, check - that unlogged was not backed up + 1. make node, create unlogged, take FULL, DELTA, PAGE, + check that unlogged table files was not backed up + 2. restore FULL, DELTA, PAGE to empty db, + ensure unlogged table exist and is epmty """ fname = self.id().split('.')[3] backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') @@ -220,6 +222,8 @@ def test_exclude_unlogged_tables_2(self): self.set_archiving(backup_dir, 'node', node) node.slow_start() + backup_ids = [] + for backup_type in ['full', 'delta', 'page']: if backup_type == 'full': @@ -239,6 +243,8 @@ def test_exclude_unlogged_tables_2(self): backup_dir, 'node', node, backup_type=backup_type, options=['--stream']) + backup_ids.append(backup_id) + filelist = self.get_backup_filelist( backup_dir, 'node', backup_id) @@ -258,96 +264,25 @@ def test_exclude_unlogged_tables_2(self): rel_path + '.3', filelist, "Unlogged table was not excluded") - # Clean after yourself - self.del_test_dir(module_name, fname) + # ensure restoring retrieves back only empty unlogged table + for backup_id in backup_ids: + node.stop() + node.cleanup() - # @unittest.skip("skip") - #TODO REVIEW time consuming test, but not more than 25 secs and 850M disk space :( - def test_exclude_unlogged_table_and_check_backup_size_unchanged(self): - """ - - make backup on empty db, capture full backup stats - - fill unlogged table by 160M, capture "full" backup, check its size is almost the same as for empty one - - increase unlogged table by 160M again, check "delta" backup size increment is less than 1M - - increase unlogged table by 160M again, check "page" backup size increment is less than 1M - """ + self.restore_node(backup_dir, 'node', node, backup_id=backup_id) - fname = self.id().split('.')[3] - backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') - node = self.make_simple_node( - base_dir=os.path.join(module_name, fname, 'node'), - set_replication=True, - initdb_params=['--data-checksums'], - pg_options={ - "shared_buffers": "10MB"}) + node.slow_start() - self.init_pb(backup_dir) - self.add_instance(backup_dir, 'node', node) - self.set_archiving(backup_dir, 'node', node) - node.slow_start() - - # init full backup and stats on empty db - backup_id_empty = self.backup_node( - backup_dir, 'node', node, backup_type="full", - return_id=True, - options=["-j", "4", "--stream"] - ) - - show_pb_empty = self.show_pb( - backup_dir, 'node', backup_id=backup_id_empty) # ['recovery-time'] - - # fill unlogged table by 400M, ensure second "full" backup is almost the same size - node.safe_psql( - "postgres", - "create unlogged table t_logged as select i" - " as id from generate_series(0,4005000) i") - - backup_id_full = self.backup_node( - backup_dir, 'node', node, backup_type="full", - return_id=True, - options=["-j", "4", "--stream"] - ) - - show_pb_full = self.show_pb( - backup_dir, 'node', backup_id=backup_id_full) - - self.assertTrue(show_pb_full["data-bytes"] - show_pb_empty["data-bytes"] < 1024*1024) - - # ensure "delta" backup skips 400M increment to unlogged table - node.safe_psql( - "postgres", - "insert into t_logged " - " select from generate_series(0,4005000)") - - backup_id_delta = self.backup_node( - backup_dir, 'node', node, backup_type="delta", - return_id=True, - options=["-j", "4", "--stream"] - ) - - show_pb_delta = self.show_pb( - backup_dir, 'node', backup_id=backup_id_delta) - - self.assertTrue(show_pb_delta["data-bytes"] < 1024*1024) - - # ensure "page" backup skips 400M increment to unlogged table - node.safe_psql( - "postgres", - "insert into t_logged " - " select from generate_series(0,4005000)") - - backup_id_page = self.backup_node( - backup_dir, 'node', node, backup_type="page", - return_id=True, - options=["-j", "4", "--stream"] - ) - - show_pb_page = self.show_pb( - backup_dir, 'node', backup_id=backup_id_page) - self.assertTrue(show_pb_page["data-bytes"] < 1024*1024) + self.assertEqual( + node.safe_psql( + 'postgres', + 'select count(*) from test').decode('utf-8').rstrip(), + '0') # Clean after yourself self.del_test_dir(module_name, fname) + # @unittest.skip("skip") def test_exclude_log_dir(self): """ From 61d3ff8569b87fc3cabf67a40423e83ff31577d8 Mon Sep 17 00:00:00 2001 From: Ivan Lazarev Date: Fri, 27 May 2022 03:40:55 +0300 Subject: [PATCH 4/5] [PBCKP-145] demo for node.execute() instead of node.safe_psql() to avoid .decode('utf-8').rstrip() conversion --- tests/exclude.py | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/tests/exclude.py b/tests/exclude.py index 456f2969e..257d605f5 100644 --- a/tests/exclude.py +++ b/tests/exclude.py @@ -235,9 +235,9 @@ def test_exclude_unlogged_tables_2(self): 'postgres', 'insert into test select generate_series(0,20050000)::text') - rel_path = node.safe_psql( + rel_path = node.execute( 'postgres', - "select pg_relation_filepath('test')").decode('utf-8').rstrip() + "select pg_relation_filepath('test')")[0][0] backup_id = self.backup_node( backup_dir, 'node', node, @@ -274,10 +274,11 @@ def test_exclude_unlogged_tables_2(self): node.slow_start() self.assertEqual( - node.safe_psql( + node.execute( 'postgres', - 'select count(*) from test').decode('utf-8').rstrip(), - '0') + # TODO REVIEW unfortunately this make auto conversion from string to int, it's weird for me + 'select count(*) from test')[0][0], + 0) # Clean after yourself self.del_test_dir(module_name, fname) From d27bf42f1d9c347f4651f8c809fea8ba09fc347c Mon Sep 17 00:00:00 2001 From: "Mikhail A. Kulagin" Date: Fri, 27 May 2022 18:55:09 +0300 Subject: [PATCH 5/5] [ci skip] remove comment --- tests/exclude.py | 1 - 1 file changed, 1 deletion(-) diff --git a/tests/exclude.py b/tests/exclude.py index 257d605f5..2c4925881 100644 --- a/tests/exclude.py +++ b/tests/exclude.py @@ -276,7 +276,6 @@ def test_exclude_unlogged_tables_2(self): self.assertEqual( node.execute( 'postgres', - # TODO REVIEW unfortunately this make auto conversion from string to int, it's weird for me 'select count(*) from test')[0][0], 0)