diff --git a/.travis.yml b/.travis.yml index c3edfdf..b6bd63a 100644 --- a/.travis.yml +++ b/.travis.yml @@ -21,26 +21,26 @@ notifications: # keep in sync with codecov.yml number of builds env: - - PG_VERSION=15 PG_BRANCH=master TEST_CASE=tap - - PG_VERSION=15 PG_BRANCH=master TEST_CASE=tap MODE=legacy - - PG_VERSION=15 PG_BRANCH=master TEST_CASE=all - - PG_VERSION=15 PG_BRANCH=master TEST_CASE=all MODE=paranoia - - PG_VERSION=14 PG_BRANCH=REL_14_STABLE TEST_CASE=tap - - PG_VERSION=14 PG_BRANCH=REL_14_STABLE TEST_CASE=tap MODE=legacy - - PG_VERSION=14 PG_BRANCH=REL_14_STABLE TEST_CASE=all - - PG_VERSION=14 PG_BRANCH=REL_14_STABLE TEST_CASE=all MODE=paranoia - - PG_VERSION=13 PG_BRANCH=REL_13_STABLE TEST_CASE=tap - - PG_VERSION=13 PG_BRANCH=REL_13_STABLE TEST_CASE=tap MODE=legacy - - PG_VERSION=13 PG_BRANCH=REL_13_STABLE TEST_CASE=all - - PG_VERSION=13 PG_BRANCH=REL_13_STABLE TEST_CASE=all MODE=paranoia - - PG_VERSION=12 PG_BRANCH=REL_12_STABLE TEST_CASE=tap - - PG_VERSION=12 PG_BRANCH=REL_12_STABLE TEST_CASE=tap MODE=legacy - - PG_VERSION=12 PG_BRANCH=REL_12_STABLE TEST_CASE=all - - PG_VERSION=12 PG_BRANCH=REL_12_STABLE TEST_CASE=all MODE=paranoia - - PG_VERSION=11 PG_BRANCH=REL_11_STABLE TEST_CASE=tap - - PG_VERSION=11 PG_BRANCH=REL_11_STABLE TEST_CASE=tap MODE=legacy - - PG_VERSION=11 PG_BRANCH=REL_11_STABLE TEST_CASE=all - - PG_VERSION=11 PG_BRANCH=REL_11_STABLE TEST_CASE=all MODE=paranoia + - PG_BRANCH=master TEST_CASE=tap + - PG_BRANCH=master TEST_CASE=tap MODE=legacy +# - PG_BRANCH=master TEST_CASE=all + - PG_BRANCH=master TEST_CASE=all MODE=paranoia + - PG_BRANCH=REL_14_STABLE TEST_CASE=tap + - PG_BRANCH=REL_14_STABLE TEST_CASE=tap MODE=legacy +# - PG_BRANCH=REL_14_STABLE TEST_CASE=all + - PG_BRANCH=REL_14_STABLE TEST_CASE=all MODE=paranoia + - PG_BRANCH=REL_13_STABLE TEST_CASE=tap + - PG_BRANCH=REL_13_STABLE TEST_CASE=tap MODE=legacy +# - PG_BRANCH=REL_13_STABLE TEST_CASE=all + - PG_BRANCH=REL_13_STABLE TEST_CASE=all MODE=paranoia + - PG_BRANCH=REL_12_STABLE TEST_CASE=tap + - PG_BRANCH=REL_12_STABLE TEST_CASE=tap MODE=legacy +# - PG_BRANCH=REL_12_STABLE TEST_CASE=all + - PG_BRANCH=REL_12_STABLE TEST_CASE=all MODE=paranoia + - PG_BRANCH=REL_11_STABLE TEST_CASE=tap + - PG_BRANCH=REL_11_STABLE TEST_CASE=tap MODE=legacy +# - PG_BRANCH=REL_11_STABLE TEST_CASE=all + - PG_BRANCH=REL_11_STABLE TEST_CASE=all MODE=paranoia jobs: allow_failures: diff --git a/Dockerfile.in b/Dockerfile.in index 39541da..c2b0ffd 100644 --- a/Dockerfile.in +++ b/Dockerfile.in @@ -5,21 +5,21 @@ RUN apt-get update RUN apt-get -yq install python python-pip python-virtualenv # Environment -ENV PG_MAJOR=${PG_VERSION} PG_BRANCH=${PG_BRANCH} -ENV LANG=C.UTF-8 PGHOME=/pg/testdir/pgbin +ENV PG_BRANCH=${PG_BRANCH} +ENV LANG=C.UTF-8 PGHOME=/testdir/pgbin ENV MODE=${MODE} TEST_CASE=${TEST_CASE} TEST_REPEATS=${TEST_REPEATS} # Make directories -RUN mkdir -p /pg/testdir +RUN mkdir -p /testdir COPY run_tests.sh /run.sh RUN chmod 755 /run.sh -COPY . /pg/testdir -WORKDIR /pg/testdir +COPY . /testdir/ptrack +WORKDIR /testdir # Grant privileges -RUN chown -R postgres:postgres /pg/testdir +RUN chown -R postgres:postgres /testdir USER postgres ENTRYPOINT /run.sh diff --git a/README.md b/README.md index b6b35fd..898df12 100644 --- a/README.md +++ b/README.md @@ -151,7 +151,6 @@ Feel free to [send pull requests](https://github.com/postgrespro/ptrack/compare) Everything is tested automatically with [travis-ci.com](https://travis-ci.com/postgrespro/ptrack) and [codecov.io](https://codecov.io/gh/postgrespro/ptrack), but you can also run tests locally via `Docker`: ```sh -export PG_VERSION=14 export PG_BRANCH=REL_14_STABLE export TEST_CASE=all export MODE=paranoia diff --git a/docker-compose.yml b/docker-compose.yml index 544e59f..fc65455 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -1,3 +1,17 @@ -tests: - privileged: true - build: . +version: "3.7" +services: + tests: + build: + context: . + + cap_add: + - SYS_PTRACE + + security_opt: + - seccomp=unconfined + + # don't work + #sysctls: + # kernel.yama.ptrace_scope: 0 + privileged: true + diff --git a/make_dockerfile.sh b/make_dockerfile.sh index 52543e8..409a5b9 100755 --- a/make_dockerfile.sh +++ b/make_dockerfile.sh @@ -1,10 +1,5 @@ #!/usr/bin/env sh -if [ -z ${PG_VERSION+x} ]; then - echo PG_VERSION is not set! - exit 1 -fi - if [ -z ${PG_BRANCH+x} ]; then echo PG_BRANCH is not set! exit 1 @@ -28,11 +23,9 @@ else echo TEST_REPEATS=${TEST_REPEATS} fi -echo PG_VERSION=${PG_VERSION} echo PG_BRANCH=${PG_BRANCH} sed \ - -e 's/${PG_VERSION}/'${PG_VERSION}/g \ -e 's/${PG_BRANCH}/'${PG_BRANCH}/g \ -e 's/${MODE}/'${MODE}/g \ -e 's/${TEST_CASE}/'${TEST_CASE}/g \ diff --git a/run_tests.sh b/run_tests.sh index c47617f..c52d9ed 100755 --- a/run_tests.sh +++ b/run_tests.sh @@ -1,55 +1,47 @@ #!/usr/bin/env bash # -# Copyright (c) 2019-2020, Postgres Professional +# Copyright (c) 2019-2021, Postgres Professional # -PG_SRC=$PWD/postgres +PTRACK_SRC=${PWD}/ptrack +PG_SRC=${PWD}/postgres +PBK_SRC=${PWD}/pg_probackup status=0 -# curl "https://ftp.postgresql.org/pub/source/v$PG_VERSION/postgresql-$PG_VERSION.tar.bz2" -o postgresql.tar.bz2 -# echo "$PG_SHA256 *postgresql.tar.bz2" | sha256sum -c - - -# mkdir $PG_SRC - -# tar \ -# --extract \ -# --file postgresql.tar.bz2 \ -# --directory $PG_SRC \ -# --strip-components 1 - +######################################################### # Clone Postgres echo "############### Getting Postgres sources" -git clone https://github.com/postgres/postgres.git -b $PG_BRANCH --depth=1 +git clone https://github.com/postgres/postgres.git --depth=1 --branch=${PG_BRANCH} ${PG_SRC} # Clone pg_probackup echo "############### Getting pg_probackup sources" -git clone https://github.com/postgrespro/pg_probackup.git --depth=1 -b master -# git clone https://github.com/ololobus/pg_probackup.git --depth=1 -b ptrack-tests +git clone https://github.com/postgrespro/pg_probackup.git --depth=1 --branch=master ${PBK_SRC} +######################################################### # Compile and install Postgres -cd postgres # Go to postgres dir +cd ${PG_SRC} # Go to postgres dir echo "############### Applying ptrack patch" -git apply -v -3 ../patches/$PG_BRANCH-ptrack-core.diff +git apply --verbose --3way ${PTRACK_SRC}/patches/${PG_BRANCH}-ptrack-core.diff -if [ "$MODE" = "paranoia" ]; then +if [ "${MODE}" = "paranoia" ]; then echo "############### Paranoia mode: applying turn-off-hint-bits.diff" - git apply -v -3 ../patches/turn-off-hint-bits.diff + git apply --verbose --3way ${PTRACK_SRC}/patches/turn-off-hint-bits.diff fi echo "############### Compiling Postgres" -if [ "$TEST_CASE" = "tap" ] && [ "$MODE" = "legacy" ]; then - ./configure CFLAGS='-DEXEC_BACKEND' --disable-atomics --prefix=$PGHOME --enable-debug --enable-cassert --enable-depend --enable-tap-tests +if [ "${TEST_CASE}" = "tap" ] && [ "${MODE}" = "legacy" ]; then + ./configure CFLAGS='-DEXEC_BACKEND' --disable-atomics --prefix=${PGHOME} --enable-debug --enable-cassert --enable-depend --enable-tap-tests --quiet else - ./configure --prefix=$PGHOME --enable-debug --enable-cassert --enable-depend --enable-tap-tests + ./configure --prefix=${PGHOME} --enable-debug --enable-cassert --enable-depend --enable-tap-tests --quiet fi -make -s -j$(nproc) install -make -s -j$(nproc) -C contrib/ install +make --quiet --jobs=$(nproc) install +make --quiet --jobs=$(nproc) --directory=contrib/ install # Override default Postgres instance -export PATH=$PGHOME/bin:$PATH -export LD_LIBRARY_PATH=$PGHOME/lib +export PATH=${PGHOME}/bin:${PATH} +export LD_LIBRARY_PATH=${PGHOME}/lib export PG_CONFIG=$(which pg_config) # Show pg_config path (just in case) @@ -60,48 +52,37 @@ which pg_config echo "############### pg_config" pg_config -# Get amcheck if missing -if [ ! -d "contrib/amcheck" ]; then - echo "############### Getting missing amcheck" - git clone https://github.com/petergeoghegan/amcheck.git --depth=1 contrib/amcheck - make USE_PGXS=1 -C contrib/amcheck install -fi - -# Get back to testdir -cd .. - +######################################################### # Build and install ptrack extension echo "############### Compiling and installing ptrack extension" +cp --recursive ${PTRACK_SRC} ${PG_SRC}/contrib/ptrack +make USE_PGXS=1 PG_CPPFLAGS="-coverage" SHLIB_LINK="-coverage" --directory=${PG_SRC}/contrib/ptrack/ install -# XXX: Hackish way to make possible to run tap tests -mkdir $PG_SRC/contrib/ptrack -cp * $PG_SRC/contrib/ptrack/ -cp -R t $PG_SRC/contrib/ptrack/ - -make USE_PGXS=1 PG_CPPFLAGS="-coverage" SHLIB_LINK="-coverage" -C $PG_SRC/contrib/ptrack/ install - -if [ "$TEST_CASE" = "tap" ]; then +if [ "${TEST_CASE}" = "tap" ]; then # Run tap tests echo "############### Running tap tests" - if [ "$MODE" = "legacy" ]; then + if [ "${MODE}" = "legacy" ]; then # There is a known issue with attaching shared memory segment using the same # address each time, when EXEC_BACKEND mechanism is turned on. It happens due # to the ASLR address space randomization, so we are trying to attach a segment # to the already occupied location. That way we simply turning off ASLR here. # # Postgres comment: https://github.com/postgres/postgres/blob/5cbfce562f7cd2aab0cdc4694ce298ec3567930e/src/backend/postmaster/postmaster.c#L4929 - setarch x86_64 --addr-no-randomize make -C postgres/contrib/ptrack check || status=$? + setarch x86_64 --addr-no-randomize make --directory=${PG_SRC}/contrib/ptrack check || status=$? else - make -C postgres/contrib/ptrack check || status=$? + make --directory=${PG_SRC}/contrib/ptrack check || status=$? fi else + # Set kernel params (used for debugging -- probackup tests) + echo "############### setting kernel params" + sudo sh -c 'echo 0 > /proc/sys/kernel/yama/ptrace_scope' # Build and install pg_probackup echo "############### Compiling and installing pg_probackup" - cd pg_probackup # Go to pg_probackup dir - make USE_PGXS=1 top_srcdir=$PG_SRC install + cd ${PBK_SRC} # Go to pg_probackup dir + make USE_PGXS=1 top_srcdir=${PG_SRC} install # Setup python environment echo "############### Setting up python env" @@ -110,35 +91,36 @@ else pip install testgres==1.8.2 echo "############### Testing" - if [ "$MODE" = "basic" ]; then + export PG_PROBACKUP_PTRACK=ON + if [ "${MODE}" = "basic" ]; then export PG_PROBACKUP_TEST_BASIC=ON - elif [ "$MODE" = "paranoia" ]; then + elif [ "${MODE}" = "paranoia" ]; then export PG_PROBACKUP_PARANOIA=ON fi - export PG_PROBACKUP_PTRACK=ON - if [ "$TEST_CASE" = "all" ]; then + if [ "${TEST_CASE}" = "all" ]; then # Run all pg_probackup ptrack tests - python3 -m unittest -v tests.ptrack || status=$? + PBK_TEST_CASE=tests.ptrack else - for i in `seq $TEST_REPEATS`; do - python3 -m unittest -v tests.ptrack.PtrackTest.$TEST_CASE || status=$? - done + PBK_TEST_CASE=tests.ptrack.PtrackTest.${TEST_CASE} fi + for i in `seq ${TEST_REPEATS}`; do + python3 -m unittest -v ${PBK_TEST_CASE} || status=$? + done # Exit virtualenv deactivate - - # Get back to testdir - cd .. - fi +######################################################### +# codecov +echo "############### Codecov" +cd ${PTRACK_SRC} # Generate *.gcov files -gcov $PG_SRC/contrib/ptrack/*.c $PG_SRC/contrib/ptrack/*.h +gcov ${PG_SRC}/contrib/ptrack/*.c ${PG_SRC}/contrib/ptrack/*.h # Send coverage stats to Codecov bash <(curl -s https://codecov.io/bash) # Something went wrong, exit with code 1 -if [ $status -ne 0 ]; then exit 1; fi +if [ ${status} -ne 0 ]; then exit 1; fi