From b2f513126271045727c54506a167d4694d0686cc Mon Sep 17 00:00:00 2001 From: anastasia Date: Wed, 19 Aug 2020 18:53:08 +0300 Subject: [PATCH 1/3] add basic tests to check compatibility with REL_11 in travis --- .travis.yml | 2 ++ codecov.yml | 2 +- 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/.travis.yml b/.travis.yml index 7fe4b10..f1d802b 100644 --- a/.travis.yml +++ b/.travis.yml @@ -27,3 +27,5 @@ env: - PG_VERSION=12 PG_BRANCH=REL_12_STABLE TEST_CASE=all MODE=paranoia - PG_VERSION=12 PG_BRANCH=REL_12_STABLE TEST_CASE=test_ptrack_multiple_segments TEST_REPEATS=5 MODE=paranoia - PG_VERSION=12 PG_BRANCH=REL_12_STABLE TEST_CASE=test_ptrack_eat_my_data TEST_REPEATS=4 + - PG_VERSION=11 PG_BRANCH=REL_11_STABLE TEST_CASE=tap + - PG_VERSION=11 PG_BRANCH=REL_11_STABLE TEST_CASE=tap MODE=legacy diff --git a/codecov.yml b/codecov.yml index 2d54077..ed32e3b 100644 --- a/codecov.yml +++ b/codecov.yml @@ -1,6 +1,6 @@ codecov: notify: - after_n_builds: 5 # keep in sync with .travis.yml number of builds + after_n_builds: 8 # keep in sync with .travis.yml number of builds # datapagemap.c/.h are copied from Postgres, so let's remove it # from report. Otherwise, we would have to remove some currently From 05b033e66514f4fdc02719b5f2c4064eed0fe61d Mon Sep 17 00:00:00 2001 From: anastasia Date: Fri, 21 Aug 2020 20:09:37 +0300 Subject: [PATCH 2/3] fix compatibility fixes for backport to v11. bring back checkpoint hook. --- ptrack.c | 12 ++---------- 1 file changed, 2 insertions(+), 10 deletions(-) diff --git a/ptrack.c b/ptrack.c index 8476b25..783bd9b 100644 --- a/ptrack.c +++ b/ptrack.c @@ -46,6 +46,7 @@ #if PG_VERSION_NUM >= 120000 #include "storage/md.h" #endif +#include "storage/smgr.h" #include "storage/reinit.h" #include "utils/builtins.h" #include "utils/guc.h" @@ -64,9 +65,7 @@ int ptrack_map_size_tmp; static copydir_hook_type prev_copydir_hook = NULL; static mdwrite_hook_type prev_mdwrite_hook = NULL; static mdextend_hook_type prev_mdextend_hook = NULL; -#if PG_VERSION_NUM >= 120000 static ProcessSyncRequests_hook_type prev_ProcessSyncRequests_hook = NULL; -#endif void _PG_init(void); void _PG_fini(void); @@ -76,9 +75,8 @@ static void ptrack_mdwrite_hook(RelFileNodeBackend smgr_rnode, ForkNumber forkno, BlockNumber blkno); static void ptrack_mdextend_hook(RelFileNodeBackend smgr_rnode, ForkNumber forkno, BlockNumber blkno); -#if PG_VERSION_NUM >= 120000 static void ptrack_ProcessSyncRequests_hook(void); -#endif + static void ptrack_gather_filelist(List **filelist, char *path, Oid spcOid, Oid dbOid); static int ptrack_filelist_getnext(PtScanCtx * ctx); @@ -120,10 +118,8 @@ _PG_init(void) mdwrite_hook = ptrack_mdwrite_hook; prev_mdextend_hook = mdextend_hook; mdextend_hook = ptrack_mdextend_hook; -#if PG_VERSION_NUM >= 120000 prev_ProcessSyncRequests_hook = ProcessSyncRequests_hook; ProcessSyncRequests_hook = ptrack_ProcessSyncRequests_hook; -#endif } /* @@ -136,9 +132,7 @@ _PG_fini(void) copydir_hook = prev_copydir_hook; mdwrite_hook = prev_mdwrite_hook; mdextend_hook = prev_mdextend_hook; -#if PG_VERSION_NUM >= 120000 ProcessSyncRequests_hook = prev_ProcessSyncRequests_hook; -#endif } /* @@ -218,7 +212,6 @@ ptrack_mdextend_hook(RelFileNodeBackend smgr_rnode, prev_mdextend_hook(smgr_rnode, forknum, blocknum); } -#if PG_VERSION_NUM >= 120000 static void ptrack_ProcessSyncRequests_hook() { @@ -227,7 +220,6 @@ ptrack_ProcessSyncRequests_hook() if (prev_ProcessSyncRequests_hook) prev_ProcessSyncRequests_hook(); } -#endif /* * Recursively walk through the path and add all data files to filelist. From 335e0f31edbb143c54b6c8d5f6fc9de7618ff984 Mon Sep 17 00:00:00 2001 From: anastasia Date: Fri, 21 Aug 2020 20:23:54 +0300 Subject: [PATCH 3/3] add core-patch for REL_11_STABLE to test ptrack compatibility with REL_11 in travis --- patches/REL_11_STABLE-ptrack-core.diff | 263 +++++++++++++++++++++++++ 1 file changed, 263 insertions(+) create mode 100644 patches/REL_11_STABLE-ptrack-core.diff diff --git a/patches/REL_11_STABLE-ptrack-core.diff b/patches/REL_11_STABLE-ptrack-core.diff new file mode 100644 index 0000000..a8207f5 --- /dev/null +++ b/patches/REL_11_STABLE-ptrack-core.diff @@ -0,0 +1,263 @@ +diff --git a/src/backend/replication/basebackup.c b/src/backend/replication/basebackup.c +index 3e53b3df6fb..f76bfc2a646 100644 +--- a/src/backend/replication/basebackup.c ++++ b/src/backend/replication/basebackup.c +@@ -209,6 +209,13 @@ static const struct exclude_list_item excludeFiles[] = + {"postmaster.pid", false}, + {"postmaster.opts", false}, + ++ /* ++ * Skip all transient ptrack files, but do copy ptrack.map, since it may ++ * be successfully used immediately after backup. TODO: check, test? ++ */ ++ {"ptrack.map.mmap", false}, ++ {"ptrack.map.tmp", false}, ++ + /* end of list */ + {NULL, false} + }; +@@ -224,6 +231,10 @@ static const struct exclude_list_item noChecksumFiles[] = { + {"pg_filenode.map", false}, + {"pg_internal.init", true}, + {"PG_VERSION", false}, ++ {"ptrack.map.mmap", false}, ++ {"ptrack.map", false}, ++ {"ptrack.map.tmp", false}, ++ + #ifdef EXEC_BACKEND + {"config_exec_params", true}, + #endif +diff --git a/src/backend/storage/file/copydir.c b/src/backend/storage/file/copydir.c +index 4a0d23b11e3..d59009a4c8c 100644 +--- a/src/backend/storage/file/copydir.c ++++ b/src/backend/storage/file/copydir.c +@@ -27,6 +27,8 @@ + #include "miscadmin.h" + #include "pgstat.h" + ++copydir_hook_type copydir_hook = NULL; ++ + /* + * copydir: copy a directory + * +@@ -78,6 +80,9 @@ copydir(char *fromdir, char *todir, bool recurse) + } + FreeDir(xldir); + ++ if (copydir_hook) ++ copydir_hook(todir); ++ + /* + * Be paranoid here and fsync all files to ensure the copy is really done. + * But if fsync is disabled, we're done. +diff --git a/src/backend/storage/smgr/md.c b/src/backend/storage/smgr/md.c +index 200cc7f657a..d0dcb5c0287 100644 +--- a/src/backend/storage/smgr/md.c ++++ b/src/backend/storage/smgr/md.c +@@ -39,6 +39,7 @@ + #include "utils/memutils.h" + #include "pg_trace.h" + ++ProcessSyncRequests_hook_type ProcessSyncRequests_hook = NULL; + + /* intervals for calling AbsorbFsyncRequests in mdsync and mdpostckpt */ + #define FSYNCS_PER_ABSORB 10 +@@ -114,6 +115,8 @@ typedef struct _MdfdVec + + static MemoryContext MdCxt; /* context for all MdfdVec objects */ + ++mdextend_hook_type mdextend_hook = NULL; ++mdwrite_hook_type mdwrite_hook = NULL; + + /* + * In some contexts (currently, standalone backends and the checkpointer) +@@ -558,6 +561,9 @@ mdextend(SMgrRelation reln, ForkNumber forknum, BlockNumber blocknum, + register_dirty_segment(reln, forknum, v); + + Assert(_mdnblocks(reln, forknum, v) <= ((BlockNumber) RELSEG_SIZE)); ++ ++ if (mdextend_hook) ++ mdextend_hook(reln->smgr_rnode, forknum, blocknum); + } + + /* +@@ -851,6 +857,9 @@ mdwrite(SMgrRelation reln, ForkNumber forknum, BlockNumber blocknum, + + if (!skipFsync && !SmgrIsTemp(reln)) + register_dirty_segment(reln, forknum, v); ++ ++ if (mdwrite_hook) ++ mdwrite_hook(reln->smgr_rnode, forknum, blocknum); + } + + /* +@@ -1329,6 +1338,9 @@ mdsync(void) + CheckpointStats.ckpt_longest_sync = longest; + CheckpointStats.ckpt_agg_sync_time = total_elapsed; + ++ if (ProcessSyncRequests_hook) ++ ProcessSyncRequests_hook(); ++ + /* Flag successful completion of mdsync */ + mdsync_in_progress = false; + } +diff --git a/src/bin/pg_resetwal/pg_resetwal.c b/src/bin/pg_resetwal/pg_resetwal.c +index 6fb403a5a8a..6e31ccb3e0f 100644 +--- a/src/bin/pg_resetwal/pg_resetwal.c ++++ b/src/bin/pg_resetwal/pg_resetwal.c +@@ -84,6 +84,7 @@ static void RewriteControlFile(void); + static void FindEndOfXLOG(void); + static void KillExistingXLOG(void); + static void KillExistingArchiveStatus(void); ++static void KillExistingPtrack(void); + static void WriteEmptyXLOG(void); + static void usage(void); + +@@ -516,6 +517,7 @@ main(int argc, char *argv[]) + RewriteControlFile(); + KillExistingXLOG(); + KillExistingArchiveStatus(); ++ KillExistingPtrack(); + WriteEmptyXLOG(); + + printf(_("Write-ahead log reset\n")); +@@ -1201,6 +1203,57 @@ KillExistingArchiveStatus(void) + } + } + ++/* ++ * Remove existing ptrack files ++ */ ++static void ++KillExistingPtrack(void) ++{ ++#define PTRACKDIR "global" ++ ++ DIR *xldir; ++ struct dirent *xlde; ++ char path[MAXPGPATH + sizeof(PTRACKDIR)]; ++ ++ xldir = opendir(PTRACKDIR); ++ if (xldir == NULL) ++ { ++ fprintf(stderr, _("%s: could not open directory \"%s\": %s\n"), ++ progname, PTRACKDIR, strerror(errno)); ++ exit(1); ++ } ++ ++ while (errno = 0, (xlde = readdir(xldir)) != NULL) ++ { ++ if (strcmp(xlde->d_name, "ptrack.map.mmap") == 0 || ++ strcmp(xlde->d_name, "ptrack.map") == 0 || ++ strcmp(xlde->d_name, "ptrack.map.tmp") == 0) ++ { ++ snprintf(path, sizeof(path), "%s/%s", PTRACKDIR, xlde->d_name); ++ if (unlink(path) < 0) ++ { ++ fprintf(stderr, _("%s: could not delete file \"%s\": %s\n"), ++ progname, path, strerror(errno)); ++ exit(1); ++ } ++ } ++ } ++ ++ if (errno) ++ { ++ fprintf(stderr, _("%s: could not read directory \"%s\": %s\n"), ++ progname, PTRACKDIR, strerror(errno)); ++ exit(1); ++ } ++ ++ if (closedir(xldir)) ++ { ++ fprintf(stderr, _("%s: could not close directory \"%s\": %s\n"), ++ progname, PTRACKDIR, strerror(errno)); ++ exit(1); ++ } ++} ++ + + /* + * Write an empty XLOG file, containing only the checkpoint record +diff --git a/src/bin/pg_rewind/filemap.c b/src/bin/pg_rewind/filemap.c +index 197163d5544..fc846e78175 100644 +--- a/src/bin/pg_rewind/filemap.c ++++ b/src/bin/pg_rewind/filemap.c +@@ -118,6 +118,10 @@ static const struct exclude_list_item excludeFiles[] = + {"postmaster.pid", false}, + {"postmaster.opts", false}, + ++ {"ptrack.map.mmap", false}, ++ {"ptrack.map", false}, ++ {"ptrack.map.tmp", false}, ++ + /* end of list */ + {NULL, false} + }; +diff --git a/src/include/miscadmin.h b/src/include/miscadmin.h +index 80241455357..50dca7bf6f4 100644 +--- a/src/include/miscadmin.h ++++ b/src/include/miscadmin.h +@@ -367,7 +367,7 @@ typedef enum ProcessingMode + NormalProcessing /* normal processing */ + } ProcessingMode; + +-extern ProcessingMode Mode; ++extern PGDLLIMPORT ProcessingMode Mode; + + #define IsBootstrapProcessingMode() (Mode == BootstrapProcessing) + #define IsInitProcessingMode() (Mode == InitProcessing) +diff --git a/src/include/port/pg_crc32c.h b/src/include/port/pg_crc32c.h +index 9a26295c8e8..dc72b27a10d 100644 +--- a/src/include/port/pg_crc32c.h ++++ b/src/include/port/pg_crc32c.h +@@ -69,8 +69,11 @@ extern pg_crc32c pg_comp_crc32c_armv8(pg_crc32c crc, const void *data, size_t le + #define FIN_CRC32C(crc) ((crc) ^= 0xFFFFFFFF) + + extern pg_crc32c pg_comp_crc32c_sb8(pg_crc32c crc, const void *data, size_t len); +-extern pg_crc32c (*pg_comp_crc32c) (pg_crc32c crc, const void *data, size_t len); +- ++extern ++#ifndef FRONTEND ++PGDLLIMPORT ++#endif ++pg_crc32c (*pg_comp_crc32c) (pg_crc32c crc, const void *data, size_t len); + #ifdef USE_SSE42_CRC32C_WITH_RUNTIME_CHECK + extern pg_crc32c pg_comp_crc32c_sse42(pg_crc32c crc, const void *data, size_t len); + #endif +diff --git a/src/include/storage/copydir.h b/src/include/storage/copydir.h +index 4fef3e21072..e55430879c3 100644 +--- a/src/include/storage/copydir.h ++++ b/src/include/storage/copydir.h +@@ -13,6 +13,9 @@ + #ifndef COPYDIR_H + #define COPYDIR_H + ++typedef void (*copydir_hook_type) (const char *path); ++extern PGDLLIMPORT copydir_hook_type copydir_hook; ++ + extern void copydir(char *fromdir, char *todir, bool recurse); + extern void copy_file(char *fromfile, char *tofile); + +diff --git a/src/include/storage/smgr.h b/src/include/storage/smgr.h +index 0298ed1a2bc..24c684771d0 100644 +--- a/src/include/storage/smgr.h ++++ b/src/include/storage/smgr.h +@@ -116,6 +116,17 @@ extern void AtEOXact_SMgr(void); + /* internals: move me elsewhere -- ay 7/94 */ + + /* in md.c */ ++ ++typedef void (*mdextend_hook_type) (RelFileNodeBackend smgr_rnode, ++ ForkNumber forknum, BlockNumber blocknum); ++extern PGDLLIMPORT mdextend_hook_type mdextend_hook; ++typedef void (*mdwrite_hook_type) (RelFileNodeBackend smgr_rnode, ++ ForkNumber forknum, BlockNumber blocknum); ++extern PGDLLIMPORT mdwrite_hook_type mdwrite_hook; ++ ++typedef void (*ProcessSyncRequests_hook_type) (void); ++extern PGDLLIMPORT ProcessSyncRequests_hook_type ProcessSyncRequests_hook; ++ + extern void mdinit(void); + extern void mdclose(SMgrRelation reln, ForkNumber forknum); + extern void mdcreate(SMgrRelation reln, ForkNumber forknum, bool isRedo);