Tests are based on pytest (#192)

* Using pytest [pytest.raises]

* Using pytest [pytest.skip]

* Using pytest [assertIsNotNone]

* Using pytest [assertFalse]

* Using pytest [assertTrue]

* Using pytest [assertEqual]

* Using pytest [assertNotEqual]

* Using pytest [assertGreaterEqual]

* Using pytest [assertGreater]

* Using pytest [assertIn]

* Using pytest [assertListEqual]

* unittest is not used

* Code style (flake8)

* Execution signature is removed

* run_tests.sh installs pytest

* run_tests.sh is updated

run tests through pytest explicitly

* Total refactoring of tests

- TestgresRemoteTests does not use global variables and code
- Explicit work with ..testgres folder

* Code style (flake8)

* Root __init__.py is added

It is required for tests.

* Code style (flake8)

* pytest.ini is added

* TestgresTests::test_ports_management is corrected

Let's send warning about a garbage in the container "bound_ports" and continue working.

* coding: utf-8

* Cleanup

* CI runs all the tests of testgres.

* Add install ssh

(cherry picked from commit fec1e7a)

* Revert "Add install ssh"

This reverts commit 537a9ac.

* Revert "CI runs all the tests of testgres."

This reverts commit 2d2532c.

* Test of probackup plugin is restored

It works now (was runned with a fresh probackup2 and vanilla 18devel).

* The test suite of a probackup plugin is based on pytest

* Probackup plugin is updated

Probackup plugin tests
 - They are skipped if PGPROBACKUPBIN is not defined

Global variable init_params is None when PGPROBACKUPBIN is not defined or version is not processed

* CI test use 4 cores

* testgres.plugins.probackup2.Init was restored [thanks to Yuri Sokolov]

* pytest.ini is updated [testpaths]

Enumeration of all the known folders with tests.

* test_child_pids (local, remote) is updated

Multiple attempts and logging are added.

* test_child_process_dies is updated

Multiple attempts are added.

---------

Co-authored-by: vshepard <[email protected]>

Author: dmitry-lipetsk

__init__.py

@@ -0,0 +1,9 @@
+[pytest]
+testpaths = ["./tests", "./testgres/plugins/pg_probackup2/pg_probackup2/tests"]
+addopts = --strict-markers
+markers =    
+#log_file = logs/pytest.log
+log_file_level = NOTSET
+log_file_format = %(levelname)8s [%(asctime)s] %(message)s
+log_file_date_format=%Y-%m-%d %H:%M:%S
+

pytest.ini

@@ -22,11 +22,11 @@ export VIRTUAL_ENV_DISABLE_PROMPT=1
 source $VENV_PATH/bin/activate
 
 # install utilities
-$PIP install coverage flake8 psutil Sphinx
+$PIP install coverage flake8 psutil Sphinx pytest pytest-xdist psycopg2 six psutil
 
 # install testgres' dependencies
 export PYTHONPATH=$(pwd)
-$PIP install .
+# $PIP install .
 
 # test code quality
 flake8 .
@@ -38,21 +38,19 @@ rm -f $COVERAGE_FILE
 
 
 # run tests (PATH)
-time coverage run -a tests/test_simple.py
+time coverage run -a -m pytest -l -v -n 4 -k "TestgresTests"
 
 
 # run tests (PG_BIN)
 time \
 	PG_BIN=$(dirname $(which pg_config)) \
-	ALT_CONFIG=1 \
-	coverage run -a tests/test_simple.py
+	coverage run -a -m pytest -l -v -n 4 -k "TestgresTests"
 
 
 # run tests (PG_CONFIG)
 time \
 	PG_CONFIG=$(which pg_config) \
-	ALT_CONFIG=1 \
-	coverage run -a tests/test_simple.py
+	coverage run -a -m pytest -l -v -n 4 -k "TestgresTests"
 
 
 # show coverage

run_tests.sh

@@ -1,7 +1,7 @@
-from pg_probackup2.gdb import GDBobj
-from pg_probackup2.app import ProbackupApp, ProbackupException
-from pg_probackup2.init_helpers import init_params
-from pg_probackup2.storage.fs_backup import FSTestBackupDir
+from .pg_probackup2.pg_probackup2.gdb import GDBobj
+from .pg_probackup2.pg_probackup2.app import ProbackupApp, ProbackupException
+from .pg_probackup2.pg_probackup2.init_helpers import init_params
+from .pg_probackup2.pg_probackup2.storage.fs_backup import FSTestBackupDir
 
 __all__ = [
     "ProbackupApp", "ProbackupException", "init_params", "FSTestBackupDir", "GDBobj"

testgres/plugins/__init__.py

@@ -121,8 +121,7 @@ def __init__(self):
                     self.probackup_path = probackup_path_tmp
 
         if not self.probackup_path:
-            logging.error('pg_probackup binary is not found')
-            exit(1)
+            raise Exception('pg_probackup binary is not found')
 
         if os.name == 'posix':
             self.EXTERNAL_DIRECTORY_DELIMITER = ':'
@@ -213,11 +212,15 @@ def __init__(self):
         if self.probackup_version.split('.')[0].isdigit():
             self.major_version = int(self.probackup_version.split('.')[0])
         else:
-            logging.error('Can\'t process pg_probackup version \"{}\": the major version is expected to be a number'.format(self.probackup_version))
-            sys.exit(1)
+            raise Exception('Can\'t process pg_probackup version \"{}\": the major version is expected to be a number'.format(self.probackup_version))
 
     def test_env(self):
         return self._test_env.copy()
 
 
-init_params = Init()
+try:
+    init_params = Init()
+except Exception as e:
+    logging.error(str(e))
+    logging.warning("testgres.plugins.probackup2.init_params is set to None.")
+    init_params = None

testgres/plugins/pg_probackup2/pg_probackup2/init_helpers.py

@@ -0,0 +1,95 @@
+from __future__ import annotations
+
+import os
+import shutil
+import pytest
+
+from ...... import testgres
+from ...pg_probackup2.app import ProbackupApp
+from ...pg_probackup2.init_helpers import Init, init_params
+from ..storage.fs_backup import FSTestBackupDir
+
+
+class ProbackupTest:
+    pg_node: testgres.PostgresNode
+
+    @staticmethod
+    def probackup_is_available() -> bool:
+        p = os.environ.get("PGPROBACKUPBIN")
+
+        if p is None:
+            return False
+
+        if not os.path.exists(p):
+            return False
+
+        return True
+
+    @pytest.fixture(autouse=True, scope="function")
+    def implicit_fixture(self, request: pytest.FixtureRequest):
+        assert isinstance(request, pytest.FixtureRequest)
+        self.helper__setUp(request)
+        yield
+        self.helper__tearDown()
+
+    def helper__setUp(self, request: pytest.FixtureRequest):
+        assert isinstance(request, pytest.FixtureRequest)
+
+        self.helper__setup_test_environment(request)
+        self.helper__setup_test_paths()
+        self.helper__setup_backup_dir()
+        self.helper__setup_probackup()
+
+    def helper__setup_test_environment(self, request: pytest.FixtureRequest):
+        assert isinstance(request, pytest.FixtureRequest)
+
+        self.output = None
+        self.cmd = None
+        self.nodes_to_cleanup = []
+        self.module_name, self.fname = request.node.cls.__name__, request.node.name
+        self.test_env = Init().test_env()
+
+    def helper__setup_test_paths(self):
+        self.rel_path = os.path.join(self.module_name, self.fname)
+        self.test_path = os.path.join(init_params.tmp_path, self.rel_path)
+        os.makedirs(self.test_path, exist_ok=True)
+        self.pb_log_path = os.path.join(self.test_path, "pb_log")
+
+    def helper__setup_backup_dir(self):
+        self.backup_dir = self.helper__build_backup_dir('backup')
+        self.backup_dir.cleanup()
+
+    def helper__setup_probackup(self):
+        self.pg_node = testgres.NodeApp(self.test_path, self.nodes_to_cleanup)
+        self.pb = ProbackupApp(self, self.pg_node, self.pb_log_path, self.test_env,
+                               auto_compress_alg='zlib', backup_dir=self.backup_dir)
+
+    def helper__tearDown(self):
+        if os.path.exists(self.test_path):
+            shutil.rmtree(self.test_path)
+
+    def helper__build_backup_dir(self, backup='backup'):
+        return FSTestBackupDir(rel_path=self.rel_path, backup=backup)
+
+
+@pytest.mark.skipif(not ProbackupTest.probackup_is_available(), reason="Check that PGPROBACKUPBIN is defined and is valid.")
+class TestBasic(ProbackupTest):
+    def test_full_backup(self):
+        # Setting up a simple test node
+        node = self.pg_node.make_simple('node', pg_options={"fsync": "off", "synchronous_commit": "off"})
+
+        # Initialize and configure Probackup
+        self.pb.init()
+        self.pb.add_instance('node', node)
+        self.pb.set_archiving('node', node)
+
+        # Start the node and initialize pgbench
+        node.slow_start()
+        node.pgbench_init(scale=100, no_vacuum=True)
+
+        # Perform backup and validation
+        backup_id = self.pb.backup_node('node', node)
+        out = self.pb.validate('node', backup_id)
+
+        # Check if the backup is valid
+        assert f"INFO: Backup {backup_id} is valid" in out

testgres/plugins/pg_probackup2/pg_probackup2/tests/__init__.py

@@ -1,3 +1,4 @@
+# coding: utf-8
 import pytest
 import platform
 

testgres/plugins/pg_probackup2/pg_probackup2/tests/basic_test.py

@@ -1,12 +1,13 @@
+# coding: utf-8
 import os
 
 import pytest
 import re
 import tempfile
 
-from testgres import ExecUtilException
-from testgres import InvalidOperationException
-from testgres import LocalOperations
+from ..testgres import ExecUtilException
+from ..testgres import InvalidOperationException
+from ..testgres import LocalOperations
 
 from .helpers.run_conditions import RunConditions
 

testgres/plugins/pg_probackup2/pg_probackup2/tests/test_basic.py

@@ -1,13 +1,14 @@
+# coding: utf-8
 import os
 
 import pytest
 import re
 import tempfile
 
-from testgres import ExecUtilException
-from testgres import InvalidOperationException
-from testgres import RemoteOperations
-from testgres import ConnectionParams
+from ..testgres import ExecUtilException
+from ..testgres import InvalidOperationException
+from ..testgres import RemoteOperations
+from ..testgres import ConnectionParams
 
 
 class TestRemoteOperations: