From 99e645e8bcce52ac85d4dabdc6ae1e4bdceb4a17 Mon Sep 17 00:00:00 2001 From: "d.kovalenko" Date: Tue, 6 May 2025 15:52:11 +0300 Subject: [PATCH 1/4] [#247] PortManager__Generic uses lock-dirs for reserved ports --- testgres/consts.py | 4 +++ testgres/impl/port_manager__generic.py | 44 ++++++++++++++++++++++++-- 2 files changed, 46 insertions(+), 2 deletions(-) diff --git a/testgres/consts.py b/testgres/consts.py index 89c49ab7..d3589205 100644 --- a/testgres/consts.py +++ b/testgres/consts.py @@ -10,6 +10,10 @@ TMP_CACHE = 'tgsc_' TMP_BACKUP = 'tgsb_' +TMP_TESTGRES = "testgres" + +TMP_TESTGRES_PORTS = TMP_TESTGRES + "/ports" + # path to control file XLOG_CONTROL_FILE = "global/pg_control" diff --git a/testgres/impl/port_manager__generic.py b/testgres/impl/port_manager__generic.py index a51af2bd..07cc2122 100755 --- a/testgres/impl/port_manager__generic.py +++ b/testgres/impl/port_manager__generic.py @@ -2,7 +2,9 @@ from ..port_manager import PortManager from ..exceptions import PortForException +from .. import consts +import os import threading import random import typing @@ -15,6 +17,8 @@ class PortManager__Generic(PortManager): _available_ports: typing.Set[int] _reserved_ports: typing.Set[int] + _lock_dir: str + def __init__(self, os_ops: OsOperations): assert os_ops is not None assert isinstance(os_ops, OsOperations) @@ -23,6 +27,12 @@ def __init__(self, os_ops: OsOperations): self._available_ports: typing.Set[int] = set(range(1024, 65535)) self._reserved_ports: typing.Set[int] = set() + temp_dir = os_ops.get_tempdir() + assert type(temp_dir) == str # noqa: E721 + self._lock_dir = os.path.join(temp_dir, consts.TMP_TESTGRES_PORTS) + assert type(self._lock_dir) == str # noqa: E721 + os_ops.makedirs(self._lock_dir) + def reserve_port(self) -> int: assert self._guard is not None assert type(self._available_ports) == set # noqa: E721t @@ -41,9 +51,23 @@ def reserve_port(self) -> int: if not self._os_ops.is_port_free(port): continue - self._reserved_ports.add(port) - self._available_ports.discard(port) + try: + lock_path = self.helper__make_lock_path(port) + self._os_ops.makedir(lock_path) + except: # noqa: 722 + continue + + assert self._os_ops.path_exists(lock_path) + + try: + self._reserved_ports.add(port) + except: # noqa: 722 + assert not (port in self._reserved_ports) + self._os_ops.rmdir(lock_path) + raise + assert port in self._reserved_ports + self._available_ports.discard(port) assert not (port in self._available_ports) return port @@ -55,6 +79,8 @@ def release_port(self, number: int) -> None: assert self._guard is not None assert type(self._reserved_ports) == set # noqa: E721 + lock_path = self.helper__make_lock_path(number) + with self._guard: assert number in self._reserved_ports assert not (number in self._available_ports) @@ -62,3 +88,17 @@ def release_port(self, number: int) -> None: self._reserved_ports.discard(number) assert not (number in self._reserved_ports) assert number in self._available_ports + + assert isinstance(self._os_ops, OsOperations) + assert self._os_ops.path_exists(lock_path) + self._os_ops.rmdir(lock_path) + + return + + def helper__make_lock_path(self, port_number: int) -> str: + assert type(port_number) == int # noqa: E721 + assert type(self._lock_dir) == str # noqa: E721 + + result = os.path.join(self._lock_dir, str(port_number) + ".lock") + assert type(result) == str # noqa: E721 + return result From c6f4b4df046aa37e3a72b5677a721f3ce6a60142 Mon Sep 17 00:00:00 2001 From: "d.kovalenko" Date: Wed, 7 May 2025 09:07:17 +0300 Subject: [PATCH 2/4] PortManager__Generic is refactored - [FIX] A creation of self._log_dir is moved to reserve_port method - Code normalization [copy&paste mistakes] - New asserts are added - New comments are added --- testgres/impl/port_manager__generic.py | 26 +++++++++++++++++--------- 1 file changed, 17 insertions(+), 9 deletions(-) diff --git a/testgres/impl/port_manager__generic.py b/testgres/impl/port_manager__generic.py index 07cc2122..cf60fa84 100755 --- a/testgres/impl/port_manager__generic.py +++ b/testgres/impl/port_manager__generic.py @@ -24,21 +24,28 @@ def __init__(self, os_ops: OsOperations): assert isinstance(os_ops, OsOperations) self._os_ops = os_ops self._guard = threading.Lock() - self._available_ports: typing.Set[int] = set(range(1024, 65535)) - self._reserved_ports: typing.Set[int] = set() - - temp_dir = os_ops.get_tempdir() - assert type(temp_dir) == str # noqa: E721 - self._lock_dir = os.path.join(temp_dir, consts.TMP_TESTGRES_PORTS) - assert type(self._lock_dir) == str # noqa: E721 - os_ops.makedirs(self._lock_dir) + self._available_ports = set(range(1024, 65535)) + self._reserved_ports = set() + self._lock_dir = None def reserve_port(self) -> int: assert self._guard is not None assert type(self._available_ports) == set # noqa: E721t assert type(self._reserved_ports) == set # noqa: E721 + assert isinstance(self._os_ops, OsOperations) with self._guard: + if self._lock_dir is None: + temp_dir = self._os_ops.get_tempdir() + assert type(temp_dir) == str # noqa: E721 + lock_dir = os.path.join(temp_dir, consts.TMP_TESTGRES_PORTS) + assert type(lock_dir) == str # noqa: E721 + self._os_ops.makedirs(lock_dir) + self._lock_dir = lock_dir + + assert self._lock_dir is not None + assert type(self._lock_dir) == str # noqa: E721 + t = tuple(self._available_ports) assert len(t) == len(self._available_ports) sampled_ports = random.sample(t, min(len(t), 100)) @@ -53,7 +60,7 @@ def reserve_port(self) -> int: try: lock_path = self.helper__make_lock_path(port) - self._os_ops.makedir(lock_path) + self._os_ops.makedir(lock_path) # raise except: # noqa: 722 continue @@ -97,6 +104,7 @@ def release_port(self, number: int) -> None: def helper__make_lock_path(self, port_number: int) -> str: assert type(port_number) == int # noqa: E721 + # You have to call the reserve_port at first! assert type(self._lock_dir) == str # noqa: E721 result = os.path.join(self._lock_dir, str(port_number) + ".lock") From f085b70d52922b4f7ac3af9e064ae1296f725621 Mon Sep 17 00:00:00 2001 From: "d.kovalenko" Date: Wed, 7 May 2025 09:37:43 +0300 Subject: [PATCH 3/4] [#256] A used port range is [1024 ... 65535] PortManager__Generic is updated. --- testgres/impl/port_manager__generic.py | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/testgres/impl/port_manager__generic.py b/testgres/impl/port_manager__generic.py index cf60fa84..fdf9e7a5 100755 --- a/testgres/impl/port_manager__generic.py +++ b/testgres/impl/port_manager__generic.py @@ -24,7 +24,7 @@ def __init__(self, os_ops: OsOperations): assert isinstance(os_ops, OsOperations) self._os_ops = os_ops self._guard = threading.Lock() - self._available_ports = set(range(1024, 65535)) + self._available_ports = set(range(1024, 65536)) self._reserved_ports = set() self._lock_dir = None @@ -52,9 +52,13 @@ def reserve_port(self) -> int: t = None for port in sampled_ports: + assert type(port) == int # noqa: E721 assert not (port in self._reserved_ports) assert port in self._available_ports + assert port >= 0 + assert port <= 65535 + if not self._os_ops.is_port_free(port): continue @@ -82,6 +86,8 @@ def reserve_port(self) -> int: def release_port(self, number: int) -> None: assert type(number) == int # noqa: E721 + assert number >= 0 + assert number <= 65535 assert self._guard is not None assert type(self._reserved_ports) == set # noqa: E721 From c9b4bbf9a84b2f4f041e12afa8669f00287f9ec0 Mon Sep 17 00:00:00 2001 From: "d.kovalenko" Date: Wed, 7 May 2025 10:10:54 +0300 Subject: [PATCH 4/4] PortManager__Generic is refactored [consts, asserts] --- testgres/impl/port_manager__generic.py | 22 +++++++++++++++++----- 1 file changed, 17 insertions(+), 5 deletions(-) diff --git a/testgres/impl/port_manager__generic.py b/testgres/impl/port_manager__generic.py index fdf9e7a5..3f312c13 100755 --- a/testgres/impl/port_manager__generic.py +++ b/testgres/impl/port_manager__generic.py @@ -11,6 +11,9 @@ class PortManager__Generic(PortManager): + C_MIN_PORT_NUMBER = 1024 + C_MAX_PORT_NUMBER = 65535 + _os_ops: OsOperations _guard: object # TODO: is there better to use bitmap fot _available_ports? @@ -20,11 +23,20 @@ class PortManager__Generic(PortManager): _lock_dir: str def __init__(self, os_ops: OsOperations): + assert __class__.C_MIN_PORT_NUMBER <= __class__.C_MAX_PORT_NUMBER + assert os_ops is not None assert isinstance(os_ops, OsOperations) self._os_ops = os_ops self._guard = threading.Lock() - self._available_ports = set(range(1024, 65536)) + + self._available_ports = set( + range(__class__.C_MIN_PORT_NUMBER, __class__.C_MAX_PORT_NUMBER + 1) + ) + assert len(self._available_ports) == ( + (__class__.C_MAX_PORT_NUMBER - __class__.C_MIN_PORT_NUMBER) + 1 + ) + self._reserved_ports = set() self._lock_dir = None @@ -56,8 +68,8 @@ def reserve_port(self) -> int: assert not (port in self._reserved_ports) assert port in self._available_ports - assert port >= 0 - assert port <= 65535 + assert port >= __class__.C_MIN_PORT_NUMBER + assert port <= __class__.C_MAX_PORT_NUMBER if not self._os_ops.is_port_free(port): continue @@ -86,8 +98,8 @@ def reserve_port(self) -> int: def release_port(self, number: int) -> None: assert type(number) == int # noqa: E721 - assert number >= 0 - assert number <= 65535 + assert number >= __class__.C_MIN_PORT_NUMBER + assert number <= __class__.C_MAX_PORT_NUMBER assert self._guard is not None assert type(self._reserved_ports) == set # noqa: E721