Merge pull request #500 from postgrespro/PBCKP-153

PBCKP-153

Author: dlepikhova

src/dir.c

@@ -1036,13 +1036,20 @@ opt_externaldir_map(ConfigOption *opt, const char *arg)
  */
 void
 create_data_directories(parray *dest_files, const char *data_dir, const char *backup_dir,
-						bool extract_tablespaces, bool incremental, fio_location location)
+						bool extract_tablespaces, bool incremental, fio_location location, 
+						const char* waldir_path)
 {
 	int			i;
 	parray		*links = NULL;
 	mode_t		pg_tablespace_mode = DIR_PERMISSION;
 	char		to_path[MAXPGPATH];
 
+	if (waldir_path && !dir_is_empty(waldir_path, location))
+	{
+		elog(ERROR, "WAL directory location is not empty: \"%s\"", waldir_path);
+	}
+
+
 	/* get tablespace map */
 	if (extract_tablespaces)
 	{
@@ -1107,6 +1114,27 @@ create_data_directories(parray *dest_files, const char *data_dir, const char *ba
 		/* skip external directory content */
 		if (dir->external_dir_num != 0)
 			continue;
+		/* Create WAL directory and symlink if waldir_path is setting */
+		if (waldir_path && strcmp(dir->rel_path, PG_XLOG_DIR) == 0) {
+			/* get full path to PG_XLOG_DIR */
+
+			join_path_components(to_path, data_dir, PG_XLOG_DIR);
+
+			elog(VERBOSE, "Create directory \"%s\" and symbolic link \"%s\"",
+				waldir_path, to_path);
+
+			/* create tablespace directory from waldir_path*/
+			fio_mkdir(waldir_path, pg_tablespace_mode, location);
+
+			/* create link to linked_path */
+			if (fio_symlink(waldir_path, to_path, incremental, location) < 0)
+				elog(ERROR, "Could not create symbolic link \"%s\": %s",
+					to_path, strerror(errno));
+
+			continue;
+
+
+		}
 
 		/* tablespace_map exists */
 		if (links)

src/help.c

@@ -169,6 +169,7 @@ help_pg_probackup(void)
 	printf(_("                 [-T OLDDIR=NEWDIR] [--progress]\n"));
 	printf(_("                 [--external-mapping=OLDDIR=NEWDIR]\n"));
 	printf(_("                 [--skip-external-dirs] [--no-sync]\n"));
+	printf(_("                 [-X WALDIR | --waldir=WALDIR]\n"));
 	printf(_("                 [-I | --incremental-mode=none|checksum|lsn]\n"));
 	printf(_("                 [--db-include | --db-exclude]\n"));
 	printf(_("                 [--remote-proto] [--remote-host]\n"));
@@ -435,6 +436,7 @@ help_restore(void)
 	printf(_("                 [-T OLDDIR=NEWDIR]\n"));
 	printf(_("                 [--external-mapping=OLDDIR=NEWDIR]\n"));
 	printf(_("                 [--skip-external-dirs]\n"));
+	printf(_("                 [-X WALDIR | --waldir=WALDIR]\n"));
 	printf(_("                 [-I | --incremental-mode=none|checksum|lsn]\n"));
 	printf(_("                 [--db-include dbname | --db-exclude dbname]\n"));
 	printf(_("                 [--recovery-target-time=time|--recovery-target-xid=xid\n"));
@@ -472,6 +474,10 @@ help_restore(void)
 	printf(_("                                   relocate the external directory from OLDDIR to NEWDIR\n"));
 	printf(_("      --skip-external-dirs         do not restore all external directories\n"));
 
+
+	printf(_("  -X, --waldir=WALDIR              location for the write-ahead log directory\n"));
+	
+
 	printf(_("\n  Incremental restore options:\n"));
 	printf(_("  -I, --incremental-mode=none|checksum|lsn\n"));
 	printf(_("                                   reuse valid pages available in PGDATA if they have not changed\n"));

src/merge.c

@@ -614,7 +614,7 @@ merge_chain(InstanceState *instanceState,
 
 	/* Create directories */
 	create_data_directories(dest_backup->files, full_database_dir,
-							dest_backup->root_dir, false, false, FIO_BACKUP_HOST);
+							dest_backup->root_dir, false, false, FIO_BACKUP_HOST, NULL);
 
 	/* External directories stuff */
 	if (dest_backup->external_dir_str)

src/pg_probackup.c

@@ -122,6 +122,7 @@ static parray *datname_include_list = NULL;
 /* arrays for --exclude-path's */
 static parray *exclude_absolute_paths_list = NULL;
 static parray *exclude_relative_paths_list = NULL;
+static char* gl_waldir_path = NULL;
 
 /* checkdb options */
 bool need_amcheck = false;
@@ -238,6 +239,7 @@ static ConfigOption cmd_options[] =
 	{ 's', 160, "primary-conninfo",	&primary_conninfo,	SOURCE_CMD_STRICT },
 	{ 's', 'S', "primary-slot-name",&replication_slot,	SOURCE_CMD_STRICT },
 	{ 'f', 'I', "incremental-mode", opt_incr_restore_mode,	SOURCE_CMD_STRICT },
+	{ 's', 'X', "waldir",		&gl_waldir_path,	SOURCE_CMD_STRICT },
 	/* checkdb options */
 	{ 'b', 195, "amcheck",			&need_amcheck,		SOURCE_CMD_STRICT },
 	{ 'b', 196, "heapallindexed",	&heapallindexed,	SOURCE_CMD_STRICT },
@@ -754,6 +756,21 @@ main(int argc, char *argv[])
 			restore_params->partial_restore_type = INCLUDE;
 			restore_params->partial_db_list = datname_include_list;
 		}
+
+		if (gl_waldir_path)
+		{
+			/* clean up xlog directory name, check it's absolute */
+			canonicalize_path(gl_waldir_path);
+			if (!is_absolute_path(gl_waldir_path))
+			{
+				elog(ERROR, "WAL directory location must be an absolute path");
+			}
+			if (strlen(gl_waldir_path) > MAXPGPATH)
+				elog(ERROR, "Value specified to --waldir is too long");
+
+		}
+		restore_params->waldir = gl_waldir_path;
+
 	}
 
 	/*

src/pg_probackup.h

@@ -566,6 +566,8 @@ typedef struct pgRestoreParams
 	/* options for partial restore */
 	PartialRestoreType partial_restore_type;
 	parray *partial_db_list;
+	
+	char* waldir;
 } pgRestoreParams;
 
 /* Options needed for set-backup command */
@@ -1022,7 +1024,8 @@ extern void create_data_directories(parray *dest_files,
 										const char *backup_dir,
 										bool extract_tablespaces,
 										bool incremental,
-										fio_location location);
+										fio_location location,
+										const char *waldir_path);
 
 extern void read_tablespace_map(parray *links, const char *backup_dir);
 extern void opt_tablespace_map(ConfigOption *opt, const char *arg);

src/restore.c

@@ -801,7 +801,7 @@ restore_chain(pgBackup *dest_backup, parray *parent_chain,
 	create_data_directories(dest_files, instance_config.pgdata,
 							dest_backup->root_dir, backup_has_tblspc,
 							params->incremental_mode != INCR_NONE,
-							FIO_DB_HOST);
+							FIO_DB_HOST, params->waldir);
 
 	/*
 	 * Restore dest_backup external directories.

tests/expected/option_help.out

@@ -86,6 +86,7 @@ pg_probackup - utility to manage backup/recovery of PostgreSQL database.
                  [-T OLDDIR=NEWDIR] [--progress]
                  [--external-mapping=OLDDIR=NEWDIR]
                  [--skip-external-dirs] [--no-sync]
+                 [-X WALDIR | --waldir=WALDIR]
                  [-I | --incremental-mode=none|checksum|lsn]
                  [--db-include | --db-exclude]
                  [--remote-proto] [--remote-host]

tests/expected/option_help_ru.out

@@ -86,6 +86,7 @@ pg_probackup - утилита для управления резервным к
                  [-T OLDDIR=NEWDIR] [--progress]
                  [--external-mapping=OLDDIR=NEWDIR]
                  [--skip-external-dirs] [--no-sync]
+                 [-X WALDIR | --waldir=WALDIR]
                  [-I | --incremental-mode=none|checksum|lsn]
                  [--db-include | --db-exclude]
                  [--remote-proto] [--remote-host]

tests/restore.py

@@ -3916,3 +3916,59 @@ def test_restore_issue_313(self):
 
         # Clean after yourself
         self.del_test_dir(module_name, fname)
+
+    # @unittest.skip("skip")
+    def test_restore_with_waldir(self):
+        """recovery using tablespace-mapping option and page backup"""
+        fname = self.id().split('.')[3]
+        node = self.make_simple_node(
+            base_dir=os.path.join(module_name, fname, 'node'),
+            initdb_params=['--data-checksums'])
+
+        backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup')
+        self.init_pb(backup_dir)
+        self.add_instance(backup_dir, 'node', node)
+        self.set_archiving(backup_dir, 'node', node)
+        node.slow_start()
+
+
+        with node.connect("postgres") as con:
+            con.execute(
+                "CREATE TABLE tbl AS SELECT * "
+                "FROM generate_series(0,3) AS integer")
+            con.commit()
+
+        # Full backup
+        backup_id = self.backup_node(backup_dir, 'node', node)
+
+        node.stop()
+        node.cleanup()
+
+        # Create waldir
+        waldir_path = os.path.join(node.base_dir, "waldir")
+        os.makedirs(waldir_path)
+
+        # Test recovery from latest
+        self.assertIn(
+            "INFO: Restore of backup {0} completed.".format(backup_id),
+            self.restore_node(
+                backup_dir, 'node', node,
+                options=[
+                    "-X", "%s" % (waldir_path)]),
+            '\n Unexpected Error Message: {0}\n CMD: {1}'.format(
+                repr(self.output), self.cmd))
+        node.slow_start()
+
+        count = node.execute("postgres", "SELECT count(*) FROM tbl")
+        self.assertEqual(count[0][0], 4)
+
+	# check pg_wal is symlink
+        if node.major_version >= 10:
+            wal_path=os.path.join(node.data_dir, "pg_wal")
+        else:
+            wal_path=os.path.join(node.data_dir, "pg_xlog")
+
+        self.assertEqual(os.path.islink(wal_path), True)
+
+        # Clean after yourself
+        self.del_test_dir(module_name, fname)