Tool for quicker replicated recovery

db/comdb2.h

@@ -3742,6 +3742,10 @@ extern int gbl_server_admin_mode;
 extern int gbl_epoch_time;
 extern int gbl_watchdog_disable_at_start;
 
+extern int gbl_recovery_timestamp;
+extern int gbl_recovery_lsn_file;
+extern int gbl_recovery_lsn_offset;
+
 extern int64_t gbl_legacy_requests;
 
 void csc2_free_all(void);

db/db_tunables.c

@@ -1138,6 +1138,9 @@ int ctrace_set_rollat(void *unused, void *value);
  */
 int get_commit_lsn_map_switch_value()
 {
+    if (gbl_recovery_timestamp || gbl_recovery_lsn_file) {
+        return 0;
+    }
     const int dependencies_are_enabled = gbl_utxnid_log;
     const int enabled_dependent_exists = 
         gbl_test_commit_lsn_map || gbl_use_modsnap_for_snapshot;

tests/Makefile

@@ -95,7 +95,7 @@ basicops_nokey: init
 basicops: basicops_nokey
 	$(shell TESTDIR="${TESTDIR}" CLUSTER="${CLUSTER}" SKIPSSL="${SKIPSSL}" $(TESTSROOTDIR)/tools/keygen.sh)
 	$(shell TOTAL="${TOTAL}" $(TESTSROOTDIR)/tools/smalldelay.sh)
-	$(shell TESTDIR="${TESTDIR}" CLUSTER="${CLUSTER}" TESTSROOTDIR="${TESTSROOTDIR}" COMDB2_EXE=${COMDB2_EXE} CDB2SQL_EXE=${CDB2SQL_EXE} COPYCOMDB2_EXE=${COPYCOMDB2_EXE} COMDB2AR_EXE=${COMDB2AR_EXE} CDB2VERIFY_EXE=${CDB2VERIFY_EXE} PMUX_EXE=${PMUX_EXE} PMUXPORT=${PMUXPORT} SKIP_COPY_EXE="${SKIP_COPY_EXE}" ${TESTSROOTDIR}/tools/copy_files_to_cluster.sh > ${TESTDIR}/copy_files_to_cluster.log 2>&1 || echo "exit 1 copy_files_to_cluster failed, see ${TESTDIR}/copy_files_to_cluster.log" )
+	$(shell TESTDIR="${TESTDIR}" CLUSTER="${CLUSTER}" TESTSROOTDIR="${TESTSROOTDIR}" COMDB2_EXE=${COMDB2_EXE} CDB2SQL_EXE=${CDB2SQL_EXE} COPYCOMDB2_EXE=${COPYCOMDB2_EXE} COMDB2AR_EXE=${COMDB2AR_EXE} CDB2VERIFY_EXE=${CDB2VERIFY_EXE} CDB2_COPYLOGS_START_EXE=${CDB2_COPYLOGS_START_EXE} PMUX_EXE=${PMUX_EXE} PMUXPORT=${PMUXPORT} SKIP_COPY_EXE="${SKIP_COPY_EXE}" ${TESTSROOTDIR}/tools/copy_files_to_cluster.sh > ${TESTDIR}/copy_files_to_cluster.log 2>&1 || echo "exit 1 copy_files_to_cluster failed, see ${TESTDIR}/copy_files_to_cluster.log" )
 
 showparams: 
 	@echo MAKECMDGOALS=${MAKECMDGOALS}
@@ -107,6 +107,7 @@ showparams:
 	@echo CLUSTER=${CLUSTER}
 	@echo BUILDDIR=${BUILDDIR}
 	@echo COMDB2AR_EXE=${COMDB2AR_EXE}
+	@echo CDB2_COPYLOGS_START_EXE=${CDB2_COPYLOGS_START_EXE}
 	@echo CDB2VERIFY_EXE=${CDB2VERIFY_EXE}
 	@echo COMDB2_EXE=${COMDB2_EXE}
 	@echo CDB2SQL_EXE=${CDB2SQL_EXE}

tests/Makefile.common

@@ -13,6 +13,7 @@ export CDB2DUMP_EXE?=${BUILDDIR}/db/cdb2_dump
 export CDB2VERIFY_EXE?=${BUILDDIR}/db/cdb2_verify
 export CDB2_SQLREPLAY_EXE?=${BUILDDIR}/tools/cdb2_sqlreplay/cdb2_sqlreplay
 export PMUX_EXE?=${BUILDDIR}/tools/pmux/pmux
+export CDB2_COPYLOGS_START_EXE?=${BUILDDIR}/tools/cdb2_copylogs_start/cdb2_copylogs_start
 export pmux_port?=5105
 export MAKEFILE_COMMON_INCLUDED=1
 endif

tests/copylogs_start.test/Makefile

@@ -0,0 +1,8 @@
+ifeq ($(TESTSROOTDIR),)
+  include ../testcase.mk
+else
+  include $(TESTSROOTDIR)/testcase.mk
+endif
+ifeq ($(TEST_TIMEOUT),)
+  export TEST_TIMEOUT=5m
+endif

tests/copylogs_start.test/runit

@@ -0,0 +1,63 @@
+#!/usr/bin/env bash
+bash -n "$0" | exit 1
+
+# Cursor moves testcase for comdb2
+################################################################################
+
+TMPDIR=${TMPDIR:-/tmp}
+DBNAME=$1
+
+SSH_OPT="-o StrictHostKeyChecking=no"
+export LOGDIR=$TESTDIR/logs
+
+cdb2sql ${CDB2_OPTIONS} $DBNAME default "create table if not exists t (a blob);"
+cdb2sql ${CDB2_OPTIONS} $DBNAME default "truncate t";
+
+# generate some log volume
+cdb2sql ${CDB2_OPTIONS} $DBNAME default "insert into t select randomblob(10000) from generate_series(1, 5000);"
+victim=$(cdb2sql --tabs ${CDB2_OPTIONS} $DBNAME default "select host from comdb2_cluster where is_master='N' limit 1;")
+master=$(cdb2sql --tabs ${CDB2_OPTIONS} $DBNAME default "select host from comdb2_cluster where is_master='Y' limit 1;")
+
+echo "Victim is $victim"
+
+rmtpid=$(cat ${TMPDIR}/${DBNAME}.${victim}.pid)
+
+[[ -z "$rmtpid" ]] && {
+    echo "Could not find pid in ${TMPDIR}/${DBNAME}.${victim}.pid"
+    exit 1
+}
+
+echo "Victim pid is $rmtpid"
+kill -9 $rmtpid
+
+# generate more log volume to force the now down node to be behind
+# by a few dozen logs
+cdb2sql ${CDB2_OPTIONS} $DBNAME default "insert into t select randomblob(10000) from generate_series(1, 30000);"
+
+# try to recover by copying the logs
+ssh ${SSH_OPT} $victim $CDB2_COPYLOGS_START_EXE --direct --lrl ${DBDIR}/${DBNAME}.lrl --dbbinary=${COMDB2_EXE} $DBNAME $master 2>&1
+rc=$?
+if [[ $rc -ne 0 ]]; then
+    echo "cdb2_copylogs_start failed with rc=$rc"
+    exit 1
+fi
+
+echo "Starting up db on $victim"
+ssh ${SSH_OPT} -n -tt $victim ${COMDB2_EXE} --no-global-lrl --lrl ${DBDIR}/${DBNAME}.lrl $DBNAME $master --pidfile ${TMPDIR}/${DBNAME}.${victim}.pid >> ${TMPDIR}/${DBNAME}.${victim}.db &> $LOGDIR/${DBNAME}.${victim}.db &
+echo $! > ${TMPDIR}/${DBNAME}.${victim}.pid
+
+sleep 10
+
+# wait until we can query it again (or timeout...)
+out=
+n=1
+while [[ "$out" != "1" ]]; do
+    out=$($CDB2SQL_EXE ${CDB2_OPTIONS} --host $victim --tabs $DBNAME default 'select 1' 2> /dev/null)
+    if [[ "$out" == "1" ]]; then
+        break
+    fi
+    echo "Attempt #$n: waiting for $victim to come up..."
+    sleep 1
+done
+echo "SUCCESS"
+exit 0

tests/setup.common

@@ -2,4 +2,5 @@ d1=$(dirname $COMDB2AR_EXE)
 d2=$(dirname $COMDB2_EXE)
 d3=$(dirname $CDB2SQL_EXE)
 d4=$(dirname $PMUX_EXE)
-paths="$d1:$d2:$d3:$d4"
+d5=$(dirname $CDB2_COPYLOGS_START_EXE)
+paths="$d1:$d2:$d3:$d4:$d5"

tests/testcase.mk

@@ -81,7 +81,7 @@ endif
 ifneq ($(INSETUP),)
   # we are in setup or running make from within a testdir
   $(shell TESTDIR="${TESTDIR}" CLUSTER="${CLUSTER}" SKIPSSL="${SKIPSSL}" ${TESTSROOTDIR}/tools/keygen.sh )
-  $(shell TESTDIR="${TESTDIR}" CLUSTER="${CLUSTER}" TESTSROOTDIR="${TESTSROOTDIR}" COMDB2_EXE=${COMDB2_EXE} CDB2SQL_EXE=${CDB2SQL_EXE} COMDB2AR_EXE=${COMDB2AR_EXE} COPYCOMDB2_EXE=${COPYCOMDB2_EXE} PMUX_EXE=${PMUX_EXE} PMUXPORT=${PMUXPORT} SKIP_COPY_EXE="${SKIP_COPY_EXE}" ${TESTSROOTDIR}/tools/copy_files_to_cluster.sh > ${TESTDIR}/copy_files_to_cluster.log 2>&1 )
+  $(shell TESTDIR="${TESTDIR}" CLUSTER="${CLUSTER}" TESTSROOTDIR="${TESTSROOTDIR}" COMDB2_EXE=${COMDB2_EXE} CDB2SQL_EXE=${CDB2SQL_EXE} COMDB2AR_EXE=${COMDB2AR_EXE} CDB2_COPYLOGS_START_EXE=${CDB2_COPYLOGS_START_EXE} COPYCOMDB2_EXE=${COPYCOMDB2_EXE} PMUX_EXE=${PMUX_EXE} PMUXPORT=${PMUXPORT} SKIP_COPY_EXE="${SKIP_COPY_EXE}" ${TESTSROOTDIR}/tools/copy_files_to_cluster.sh > ${TESTDIR}/copy_files_to_cluster.log 2>&1 )
 endif
 
 test:: tool unit

tests/tools/comdb2makecluster

@@ -66,6 +66,7 @@ COMDB2_EXE=${COMDB2_EXE:-comdb2}
 CDB2SQL_EXE=${CDB2SQL_EXE:-cdb2sql}
 COMDB2AR_EXE=${COMDB2AR_EXE:-comdb2ar}
 COPYCOMDB2_EXE=${COPYCOMDB2_EXE:-copycomdb2}
+CDB2_COPYLOGS_START_EXE=${CDB2_COPYLOGS_START_EXE:-cdb2_copylogs_start}
 export comdb2ar=$COMDB2AR_EXE  # used by copycomdb2
 export COMDB2AR_EXOPTS="-x $COMDB2_EXE"
 export COMDB2_ROOT=${DBDIR}
@@ -77,6 +78,7 @@ check_executables_for_cluster()
 {
     which $CDB2SQL_EXE  > /dev/null || failexit "$CDB2SQL_EXE not in path"
     which $COMDB2AR_EXE  > /dev/null || failexit "$COMDB2AR_EXE not in path"
+    which $CDB2_COPYLOGS_START_EXE  > /dev/null || failexit "$CDB2_COPYLOGS_START_EXE not in path"
     which $COPYCOMDB2_EXE  > /dev/null || failexit "$COPYCOMDB2_EXE not in path"
     for node in $CLUSTER; do
         if [ $node == $myhostname ] ; then

tests/tools/copy_files_to_cluster.sh

@@ -4,7 +4,7 @@
 
 #set -x
 
-vars="HOSTNAME TESTSROOTDIR TESTDIR COMDB2_EXE CDB2SQL_EXE COMDB2AR_EXE COPYCOMDB2_EXE CDB2VERIFY_EXE PMUX_EXE"
+vars="HOSTNAME TESTSROOTDIR TESTDIR COMDB2_EXE CDB2SQL_EXE COMDB2AR_EXE COPYCOMDB2_EXE CDB2VERIFY_EXE PMUX_EXE CDB2_COPYLOGS_START_EXE"
 for required in $vars; do
     q=${!required}
     if [[ -z "$q" ]]; then
@@ -15,7 +15,7 @@ for required in $vars; do
 done
 
 #make sure files exist
-files="COMDB2_EXE CDB2SQL_EXE COMDB2AR_EXE COPYCOMDB2_EXE CDB2VERIFY_EXE PMUX_EXE"
+files="COMDB2_EXE CDB2SQL_EXE COMDB2AR_EXE COPYCOMDB2_EXE CDB2VERIFY_EXE PMUX_EXE CDB2_COPYLOGS_START_EXE"
 for file in $files; do
     f=${!file}
     if [[ ! -f $f ]] ; then
@@ -54,10 +54,11 @@ copy_files_to_node() {
     trap "close_master_ssh_session \"closing\"" INT EXIT
 
     ssh $SSH_OPT $SSH_MSTR -MNf $node   #start master ssh session for node
-    ssh $SSH_OPT $SSH_MSTR $node "mkdir -p $d1 $d2 $d3 $d4 $TESTDIR/logs/ $TESTDIR/var/log/cdb2 $TESTDIR/tmp/cdb2 $TESTDIR/etc/cdb2/config.d" < /dev/null
+    ssh $SSH_OPT $SSH_MSTR $node "mkdir -p $d1 $d2 $d3 $d4 $d5 $TESTDIR/logs/ $TESTDIR/var/log/cdb2 $TESTDIR/tmp/cdb2 $TESTDIR/etc/cdb2/config.d" < /dev/null
 
     if [[ "$SKIP_COPY_EXE" != "1" ]] ; then
         scp $SSH_OPT $SSH_MSTR $COMDB2AR_EXE $node:$COMDB2AR_EXE
+        scp $SSH_OPT $SSH_MSTR $CDB2_COPYLOGS_START_EXE $node:$CDB2_COPYLOGS_START_EXE
         scp $SSH_OPT $SSH_MSTR $COPYCOMDB2_EXE $node:$COPYCOMDB2_EXE
         scp $SSH_OPT $SSH_MSTR $CDB2VERIFY_EXE $node:$CDB2VERIFY_EXE
         scp $SSH_OPT $SSH_MSTR $COMDB2_EXE $node:$COMDB2_EXE

tools/CMakeLists.txt

@@ -4,3 +4,4 @@ add_subdirectory(cdb2sockpool)
 add_subdirectory(cdb2_sqlreplay)
 add_subdirectory(cdb2sql)
 add_subdirectory(pmux)
+add_subdirectory(cdb2_copylogs_start)

tools/cdb2_copylogs_start/CMakeLists.txt

@@ -0,0 +1,21 @@
+add_executable(cdb2_copylogs_start cdb2_copylogs_start.c)
+
+include_directories(
+  ${PROJECT_SOURCE_DIR}/cdb2api
+)
+set(libs
+  cdb2api
+  ${PROTOBUF-C_LIBRARY}
+  ${OPENSSL_LIBRARIES}
+  ${ZLIB_LIBRARIES}
+  ${CMAKE_DL_LIBS}
+)
+
+list(APPEND libs ${UNWIND_LIBRARY})
+
+target_link_libraries(cdb2_copylogs_start ${libs})
+if(COMDB2_BUILD_STATIC)
+  set(CMAKE_EXE_LINKER_FLAGS "${CMAKE_EXE_LINKER_FLAGS} -static-libgcc -static-libstdc++")
+endif()
+
+install(TARGETS cdb2_copylogs_start RUNTIME DESTINATION bin)