From e10520a939003a7a2dac36df389eedc8176a9c65 Mon Sep 17 00:00:00 2001 From: Dmitry Ivanov Date: Wed, 21 Feb 2018 16:23:33 +0300 Subject: [PATCH 1/4] Generate unique system ids in cached_initdb() --- testgres/cache.py | 19 ++++++++++++++++++- testgres/config.py | 2 ++ testgres/consts.py | 3 +++ testgres/utils.py | 21 +++++++++++++++++++++ tests/test_simple.py | 34 +++++++++++++++++++++++++++------- 5 files changed, 71 insertions(+), 8 deletions(-) diff --git a/testgres/cache.py b/testgres/cache.py index d0eaac9c..7b738411 100644 --- a/testgres/cache.py +++ b/testgres/cache.py @@ -1,5 +1,6 @@ # coding: utf-8 +import io import os import shutil @@ -7,13 +8,16 @@ from .config import testgres_config +from .consts import XLOG_CONTROL_FILE + from .exceptions import \ InitNodeException, \ ExecUtilException from .utils import \ get_bin_path, \ - execute_utility + execute_utility, \ + generate_system_id def cached_initdb(data_dir, logfile=None, params=None): @@ -42,5 +46,18 @@ def call_initdb(initdb_dir, log=None): try: # Copy cached initdb to current data dir shutil.copytree(cached_data_dir, data_dir) + + # Assign this node a unique system id if asked to + if testgres_config.cached_initdb_unique: + # XXX: write new unique system id to control file + # Some users might rely upon unique system ids, but + # our initdb caching mechanism breaks this contract. + pg_control = os.path.join(data_dir, XLOG_CONTROL_FILE) + with io.open(pg_control, "r+b") as f: + f.write(generate_system_id()) # overwrite id + + # XXX: build new WAL segment with our system id + _params = [get_bin_path("pg_resetwal"), "-D", data_dir, "-f"] + execute_utility(_params, logfile) except Exception as e: raise_from(InitNodeException("Failed to spawn a node"), e) diff --git a/testgres/config.py b/testgres/config.py index d192b6d5..4baf180f 100644 --- a/testgres/config.py +++ b/testgres/config.py @@ -15,6 +15,7 @@ class GlobalConfig(object): Attributes: cache_initdb: shall we use cached initdb instance? cached_initdb_dir: shall we create a temp dir for cached initdb? + cached_initdb_unique: shall we assign new node a unique system id? cache_pg_config: shall we cache pg_config results? @@ -30,6 +31,7 @@ class GlobalConfig(object): cache_initdb = True _cached_initdb_dir = None + cached_initdb_unique = False cache_pg_config = True diff --git a/testgres/consts.py b/testgres/consts.py index e8dd8c99..9f83b2d7 100644 --- a/testgres/consts.py +++ b/testgres/consts.py @@ -4,6 +4,9 @@ DATA_DIR = "data" LOGS_DIR = "logs" +# path to control file +XLOG_CONTROL_FILE = "global/pg_control" + # names for config files RECOVERY_CONF_FILE = "recovery.conf" PG_AUTO_CONF_FILE = "postgresql.auto.conf" diff --git a/testgres/utils.py b/testgres/utils.py index 01befbc5..2ffaf0a0 100644 --- a/testgres/utils.py +++ b/testgres/utils.py @@ -68,6 +68,27 @@ def generate_app_name(): return 'testgres-{}'.format(str(uuid.uuid4())) +def generate_system_id(): + """ + Generate a new 64-bit unique system identifier for node. + """ + + import datetime + import struct + + date = datetime.datetime.now() + secs = int(date.timestamp()) + usecs = date.microsecond + + system_id = 0 + system_id |= (secs << 32) + system_id |= (usecs << 12) + system_id |= (os.getpid() & 0xFFF) + + # pack ULL in native byte order + return struct.pack('=Q', system_id) + + def execute_utility(args, logfile=None): """ Execute utility (pg_ctl, pg_dump etc). diff --git a/tests/test_simple.py b/tests/test_simple.py index 6714ee32..c38e6808 100755 --- a/tests/test_simple.py +++ b/tests/test_simple.py @@ -38,15 +38,15 @@ get_pg_config from testgres import bound_ports +from testgres.utils import pg_version_ge def util_is_executable(util): def good_properties(f): - return ( - os.path.exists(f) and - os.path.isfile(f) and - os.access(f, os.X_OK) - ) + # yapf: disable + return (os.path.exists(f) and + os.path.isfile(f) and + os.access(f, os.X_OK)) # try to resolve it if good_properties(get_bin_path(util)): @@ -91,6 +91,27 @@ def test_init_after_cleanup(self): node.cleanup() node.init().start().execute('select 1') + @unittest.skipUnless(pg_version_ge('9.6'), 'query works on 9.6+') + def test_init_unique_system_id(self): + with scoped_config( + cache_initdb=True, cached_initdb_unique=True) as config: + + self.assertTrue(config.cache_initdb) + self.assertTrue(config.cached_initdb_unique) + + # spawn two nodes; ids must be different + with get_new_node().init().start() as node1, \ + get_new_node().init().start() as node2: + + # this function exists in PostgreSQL 9.6+ + query = 'select system_identifier from pg_control_system()' + + id1 = node1.execute(query)[0] + id2 = node2.execute(query)[0] + + # ids must increase + self.assertGreater(id2, id1) + def test_node_exit(self): base_dir = None @@ -486,8 +507,7 @@ def test_logging(self): master.restart() self.assertTrue(master._logger.is_alive()) - @unittest.skipUnless( - util_is_executable("pgbench"), "pgbench may be missing") + @unittest.skipUnless(util_is_executable('pgbench'), 'might be missing') def test_pgbench(self): with get_new_node().init().start() as node: From c696004909987b1743f341d77b8fe062abd35b9b Mon Sep 17 00:00:00 2001 From: Dmitry Ivanov Date: Wed, 21 Feb 2018 17:52:21 +0300 Subject: [PATCH 2/4] fixes in code & tests --- testgres/cache.py | 5 +++++ testgres/utils.py | 8 +++++--- tests/test_simple.py | 5 +++-- 3 files changed, 13 insertions(+), 5 deletions(-) diff --git a/testgres/cache.py b/testgres/cache.py index 7b738411..1c0d98ae 100644 --- a/testgres/cache.py +++ b/testgres/cache.py @@ -59,5 +59,10 @@ def call_initdb(initdb_dir, log=None): # XXX: build new WAL segment with our system id _params = [get_bin_path("pg_resetwal"), "-D", data_dir, "-f"] execute_utility(_params, logfile) + + except ExecUtilException as e: + msg = "Failed to reset WAL for system id" + raise_from(InitNodeException(msg), e) + except Exception as e: raise_from(InitNodeException("Failed to spawn a node"), e) diff --git a/testgres/utils.py b/testgres/utils.py index 2ffaf0a0..1a3d4f21 100644 --- a/testgres/utils.py +++ b/testgres/utils.py @@ -76,9 +76,11 @@ def generate_system_id(): import datetime import struct - date = datetime.datetime.now() - secs = int(date.timestamp()) - usecs = date.microsecond + date1 = datetime.datetime.utcfromtimestamp(0) + date2 = datetime.datetime.utcnow() + + secs = int((date2 - date1).total_seconds()) + usecs = date2.microsecond system_id = 0 system_id |= (secs << 32) diff --git a/tests/test_simple.py b/tests/test_simple.py index c38e6808..21bb9616 100755 --- a/tests/test_simple.py +++ b/tests/test_simple.py @@ -41,7 +41,7 @@ from testgres.utils import pg_version_ge -def util_is_executable(util): +def util_exists(util): def good_properties(f): # yapf: disable return (os.path.exists(f) and @@ -91,6 +91,7 @@ def test_init_after_cleanup(self): node.cleanup() node.init().start().execute('select 1') + @unittest.skipUnless(util_exists('pg_resetwal'), 'might be missing') @unittest.skipUnless(pg_version_ge('9.6'), 'query works on 9.6+') def test_init_unique_system_id(self): with scoped_config( @@ -507,7 +508,7 @@ def test_logging(self): master.restart() self.assertTrue(master._logger.is_alive()) - @unittest.skipUnless(util_is_executable('pgbench'), 'might be missing') + @unittest.skipUnless(util_exists('pgbench'), 'might be missing') def test_pgbench(self): with get_new_node().init().start() as node: From 83d122f116b28ffe62e62e7af36fdc463fd47f2e Mon Sep 17 00:00:00 2001 From: Dmitry Ivanov Date: Thu, 22 Feb 2018 12:11:38 +0300 Subject: [PATCH 3/4] add a comment in generate_system_id() --- testgres/utils.py | 1 + 1 file changed, 1 insertion(+) diff --git a/testgres/utils.py b/testgres/utils.py index 1a3d4f21..3f073e08 100644 --- a/testgres/utils.py +++ b/testgres/utils.py @@ -82,6 +82,7 @@ def generate_system_id(): secs = int((date2 - date1).total_seconds()) usecs = date2.microsecond + # see pg_resetwal.c : GuessControlValues() system_id = 0 system_id |= (secs << 32) system_id |= (usecs << 12) From c2f01569462e1493f339e016af3fbda9d65ab527 Mon Sep 17 00:00:00 2001 From: Dmitry Ivanov Date: Thu, 22 Feb 2018 12:28:52 +0300 Subject: [PATCH 4/4] make test_init_unique_system_id() more durable --- tests/test_simple.py | 16 +++++++++++----- 1 file changed, 11 insertions(+), 5 deletions(-) diff --git a/tests/test_simple.py b/tests/test_simple.py index 21bb9616..b907b5ec 100755 --- a/tests/test_simple.py +++ b/tests/test_simple.py @@ -94,8 +94,16 @@ def test_init_after_cleanup(self): @unittest.skipUnless(util_exists('pg_resetwal'), 'might be missing') @unittest.skipUnless(pg_version_ge('9.6'), 'query works on 9.6+') def test_init_unique_system_id(self): - with scoped_config( - cache_initdb=True, cached_initdb_unique=True) as config: + # this function exists in PostgreSQL 9.6+ + query = 'select system_identifier from pg_control_system()' + + with scoped_config(cache_initdb=False): + with get_new_node().init().start() as node0: + id0 = node0.execute(query)[0] + + # yapf: disable + with scoped_config(cache_initdb=True, + cached_initdb_unique=True) as config: self.assertTrue(config.cache_initdb) self.assertTrue(config.cached_initdb_unique) @@ -104,13 +112,11 @@ def test_init_unique_system_id(self): with get_new_node().init().start() as node1, \ get_new_node().init().start() as node2: - # this function exists in PostgreSQL 9.6+ - query = 'select system_identifier from pg_control_system()' - id1 = node1.execute(query)[0] id2 = node2.execute(query)[0] # ids must increase + self.assertGreater(id1, id0) self.assertGreater(id2, id1) def test_node_exit(self):