Added some DB-access routines to help rsync keep extra filesystem info
about the files it is dealing with. This adds both the --db=CONFIG_FILE
option and the "db config" daemon parameter.
Future improvements may include:
- Updating of MD4 checksums when transferring any file, even w/o -c.
To make that work we'd need to make the sender force checksum_seed to
0 when using a DB and having the receiving side check to see if it
got a 0 checksum_seed.
- Caching of path info that allows for the finding of files to use for
moving/linking/copying/alternate-basis-use.
- Extend DB support beyond MySQL and SQLite (PostgreSQL?).
To use this patch, run these commands for a successful build:
patch -p1 <patches/db.diff
./prepare-source
./configure
make
based-on: d73762eea3f15f2c56bb3fa9394ad1883c25c949
diff --git a/.gitignore b/.gitignore
--- a/.gitignore
+++ b/.gitignore
@@ -16,6 +16,7 @@ aclocal.m4
/proto.h
/proto.h-tstamp
/rsync.1
+/rsyncdb.1
/rsyncd.conf.5
/autom4te*.cache
/confdefs.h
@@ -24,6 +25,7 @@ aclocal.m4
/getgroups
/gmon.out
/rsync
+/rsyncdb
/rsync-ssl
/stunnel-rsync
/stunnel-rsyncd.conf
diff --git a/Makefile.in b/Makefile.in
--- a/Makefile.in
+++ b/Makefile.in
@@ -6,6 +6,7 @@ datarootdir=@datarootdir@
exec_prefix=@exec_prefix@
stunnel4=@STUNNEL4@
bindir=@bindir@
+sbindir=@sbindir@
mandir=@mandir@
LIBS=@LIBS@
@@ -29,7 +30,7 @@ VERSION=@RSYNC_VERSION@
.SUFFIXES:
.SUFFIXES: .c .o
-GENFILES=configure.sh aclocal.m4 config.h.in proto.h proto.h-tstamp rsync.1 rsyncd.conf.5
+GENFILES=configure.sh aclocal.m4 config.h.in proto.h proto.h-tstamp rsync.1 rsyncdb.1 rsyncd.conf.5
HEADERS=byteorder.h config.h errcode.h proto.h rsync.h ifuncs.h itypes.h inums.h \
lib/pool_alloc.h
LIBOBJ=lib/wildmatch.o lib/compat.o lib/snprintf.o lib/mdfour.o lib/md5.o \
@@ -39,7 +40,7 @@ zlib_OBJS=zlib/deflate.o zlib/inffast.o zlib/inflate.o zlib/inftrees.o \
OBJS1=flist.o rsync.o generator.o receiver.o cleanup.o sender.o exclude.o \
util.o util2.o main.o checksum.o match.o syscall.o log.o backup.o delete.o
OBJS2=options.o io.o compat.o hlink.o token.o uidlist.o socket.o hashtable.o \
- fileio.o batch.o clientname.o chmod.o acls.o xattrs.o
+ fileio.o batch.o clientname.o chmod.o db.o acls.o xattrs.o
OBJS3=progress.o pipe.o
DAEMON_OBJ = params.o loadparm.o clientserver.o access.o connection.o authenticate.o
popt_OBJS=popt/findme.o popt/popt.o popt/poptconfig.o \
@@ -63,14 +64,17 @@ CHECK_OBJS=tls.o testrun.o getgroups.o getfsdev.o t_stub.o t_unsafe.o trimslash.
$(CC) -I. -I$(srcdir) $(CFLAGS) $(CPPFLAGS) -c $< @CC_SHOBJ_FLAG@
@OBJ_RESTORE@
-all: Makefile rsync$(EXEEXT) rsync-ssl stunnel-rsync stunnel-rsyncd.conf @MAKE_MAN@
+all: Makefile rsync$(EXEEXT) rsyncdb$(EXEEXT) rsync-ssl stunnel-rsync stunnel-rsyncd.conf @MAKE_MAN@
install: all
- -${MKDIR_P} ${DESTDIR}${bindir}
+ -${MKDIR_P} ${DESTDIR}${bindir} ${DESTDIR}${sbindir}
${INSTALLCMD} ${INSTALL_STRIP} -m 755 rsync$(EXEEXT) ${DESTDIR}${bindir}
+ rsync -ilt rsyncdb$(EXEEXT) ${DESTDIR}${bindir}/
+ ${INSTALLCMD} -m 755 rsyncdb-mountinfo ${DESTDIR}${sbindir}
-${MKDIR_P} ${DESTDIR}${mandir}/man1
-${MKDIR_P} ${DESTDIR}${mandir}/man5
if test -f rsync.1; then ${INSTALLMAN} -m 644 rsync.1 ${DESTDIR}${mandir}/man1; fi
+ if test -f rsyncdb.1; then ${INSTALLMAN} -m 644 rsyncdb.1 ${DESTDIR}${mandir}/man1; fi
if test -f rsyncd.conf.5; then ${INSTALLMAN} -m 644 rsyncd.conf.5 ${DESTDIR}${mandir}/man5; fi
install-ssl-client: rsync-ssl stunnel-rsync
@@ -93,6 +97,9 @@ install-strip:
rsync$(EXEEXT): $(OBJS)
$(CC) $(CFLAGS) $(LDFLAGS) -o $@ $(OBJS) $(LIBS)
+rsyncdb$(EXEEXT): rsync$(EXEEXT)
+ ln -s rsync$(EXEEXT) rsyncdb$(EXEEXT)
+
$(OBJS): $(HEADERS)
$(CHECK_OBJS): $(HEADERS)
@@ -212,22 +219,27 @@ proto.h: proto.h-tstamp
proto.h-tstamp: $(srcdir)/*.c $(srcdir)/lib/compat.c config.h
perl $(srcdir)/mkproto.pl $(srcdir)/*.c $(srcdir)/lib/compat.c
-man: rsync.1 rsyncd.conf.5 man-copy
+man: rsync.1 rsyncdb.1 rsyncd.conf.5 man-copy
man-copy:
@if test -f rsync.1; then :; elif test -f $(srcdir)/rsync.1; then echo 'Copying srcdir rsync.1'; cp -p $(srcdir)/rsync.1 .; else echo "NOTE: rsync.1 cannot be created."; fi
+ @if test -f rsyncdb.1; then :; elif test -f $(srcdir)/rsyncdb.1; then echo 'Copying srcdir rsyncdb.1'; cp -p $(srcdir)/rsyncdb.1 .; else echo "NOTE: rsyncdb.1 cannot be created."; fi
@if test -f rsyncd.conf.5; then :; elif test -f $(srcdir)/rsyncd.conf.5; then echo 'Copying srcdir rsyncd.conf.5'; cp -p $(srcdir)/rsyncd.conf.5 .; else echo "NOTE: rsyncd.conf.5 cannot be created."; fi
rsync.1: rsync.yo
yodl2man -o rsync.1 $(srcdir)/rsync.yo
-$(srcdir)/tweak_manpage rsync.1
+rsyncdb.1: rsyncdb.yo
+ yodl2man -o rsyncdb.1 $(srcdir)/rsyncdb.yo
+ -$(srcdir)/tweak_manpage rsyncdb.1
+
rsyncd.conf.5: rsyncd.conf.yo
yodl2man -o rsyncd.conf.5 $(srcdir)/rsyncd.conf.yo
-$(srcdir)/tweak_manpage rsyncd.conf.5
clean: cleantests
- rm -f *~ $(OBJS) $(CHECK_PROGS) $(CHECK_OBJS) $(CHECK_SYMLINKS) \
+ rm -f *~ $(OBJS) $(CHECK_PROGS) $(CHECK_OBJS) $(CHECK_SYMLINKS) rsyncdb$(EXEEXT) \
rounding rounding.h *.old
cleantests:
diff --git a/checksum.c b/checksum.c
--- a/checksum.c
+++ b/checksum.c
@@ -24,6 +24,7 @@
extern int checksum_seed;
extern int protocol_version;
extern int proper_seed_order;
+extern int use_db;
extern char *checksum_choice;
#define CSUM_NONE 0
@@ -220,6 +221,8 @@ void file_checksum(const char *fname, const STRUCT_STAT *st_p, char *sum)
md5_update(&m, (uchar *)map_ptr(buf, i, remainder), remainder);
md5_result(&m, (uchar *)sum);
+ if (use_db)
+ db_set_checksum(5, st_p, sum);
break;
case CSUM_MD4:
case CSUM_MD4_OLD:
@@ -241,6 +244,8 @@ void file_checksum(const char *fname, const STRUCT_STAT *st_p, char *sum)
mdfour_update(&m, (uchar *)map_ptr(buf, i, remainder), remainder);
mdfour_result(&m, (uchar *)sum);
+ if (use_db)
+ db_set_checksum(4, st_p, sum);
break;
default:
rprintf(FERROR, "invalid checksum-choice for the --checksum option (%d)\n", checksum_type);
diff --git a/cleanup.c b/cleanup.c
--- a/cleanup.c
+++ b/cleanup.c
@@ -27,6 +27,7 @@ extern int am_server;
extern int am_daemon;
extern int am_receiver;
extern int io_error;
+extern int use_db;
extern int keep_partial;
extern int got_xfer_error;
extern int protocol_version;
@@ -142,6 +143,12 @@ NORETURN void _exit_cleanup(int code, const char *file, int line)
#include "case_N.h"
switch_step++;
+ if (use_db)
+ db_disconnect(False);
+
+ /* FALLTHROUGH */
+#include "case_N.h"
+
if (cleanup_child_pid != -1) {
int status;
int pid = wait_process(cleanup_child_pid, &status, WNOHANG);
diff --git a/clientserver.c b/clientserver.c
--- a/clientserver.c
+++ b/clientserver.c
@@ -42,12 +42,15 @@ extern int numeric_ids;
extern int filesfrom_fd;
extern int remote_protocol;
extern int protocol_version;
+extern int always_checksum;
+extern int db_lax;
extern int io_timeout;
extern int no_detach;
extern int write_batch;
extern int default_af_hint;
extern int logfile_format_has_i;
extern int logfile_format_has_o_or_i;
+extern char *db_config;
extern char *bind_address;
extern char *config_file;
extern char *logfile_format;
@@ -687,6 +690,11 @@ static int rsync_module(int f_in, int f_out, int i, const char *addr, const char
log_init(1);
+ if (*lp_db_config(i)) {
+ db_read_config(FLOG, lp_db_config(i));
+ db_lax = lp_db_lax(i);
+ }
+
#ifdef HAVE_PUTENV
if (*lp_prexfer_exec(i) || *lp_postxfer_exec(i)) {
int status;
@@ -894,6 +902,8 @@ static int rsync_module(int f_in, int f_out, int i, const char *addr, const char
am_server = 1; /* Don't let someone try to be tricky. */
quiet = 0;
+ db_config = NULL;
+
if (lp_ignore_errors(module_id))
ignore_errors = 1;
if (write_batch < 0)
diff --git a/configure.ac b/configure.ac
--- a/configure.ac
+++ b/configure.ac
@@ -344,6 +344,7 @@ AC_CHECK_HEADERS(sys/fcntl.h sys/select.h fcntl.h sys/time.h sys/unistd.h \
unistd.h utime.h grp.h compat.h sys/param.h ctype.h sys/wait.h \
sys/ioctl.h sys/filio.h string.h stdlib.h sys/socket.h sys/mode.h \
sys/un.h sys/attr.h mcheck.h arpa/inet.h arpa/nameser.h locale.h \
+ mysql/mysql.h sqlite3.h \
netdb.h malloc.h float.h limits.h iconv.h libcharset.h langinfo.h \
sys/acl.h acl/libacl.h attr/xattr.h sys/xattr.h sys/extattr.h \
popt.h popt/popt.h linux/falloc.h netinet/in_systm.h netinet/ip.h \
@@ -1099,6 +1100,48 @@ if test x"$enable_acl_support" = x"no" -o x"$enable_xattr_support" = x"no" -o x"
fi
fi
+AC_MSG_CHECKING([whether to include mysql DB support])
+AC_ARG_ENABLE(mysql,
+ AC_HELP_STRING([--disable-mysql],
+ [disable mysql DB support]))
+
+if test x"$enable_mysql" = x"no"; then
+ AC_MSG_RESULT(no)
+else
+ AC_MSG_RESULT([yes])
+ AC_CHECK_PROG(MYSQL_CONFIG, mysql_config, 1, 0)
+ if test x$MYSQL_CONFIG = x1; then
+ AC_MSG_CHECKING(for mysql version >= 4)
+ mysql_version=`mysql_config --version`
+ mysql_major_version=`echo $mysql_version | sed 's/\..*//'`
+ if test $mysql_major_version -lt 4; then
+ AC_MSG_RESULT(no.. skipping MySQL)
+ else
+ AC_MSG_RESULT(yes)
+
+ MYSQL_CFLAGS=`mysql_config --cflags`
+ MYSQL_LIBS=`mysql_config --libs`
+
+ CPPFLAGS="$CPPFLAGS $MYSQL_CFLAGS"
+ LIBS="$MYSQL_LIBS $LIBS"
+
+ AC_CHECK_LIB(mysqlclient, mysql_init)
+ fi
+ fi
+fi
+
+AC_MSG_CHECKING([whether to include sqlite DB support])
+AC_ARG_ENABLE(sqlite,
+ AC_HELP_STRING([--disable-sqlite],
+ [disable sqlite DB support]))
+
+if test x"$enable_sqlite" = x"no"; then
+ AC_MSG_RESULT(no)
+else
+ AC_CHECK_LIB(sqlite3, sqlite3_open)
+ AC_CHECK_FUNCS(sqlite3_open_v2 sqlite3_prepare_v2)
+fi
+
case "$CC" in
' checker'*|checker*)
AC_DEFINE(FORCE_FD_ZERO_MEMSET, 1, [Used to make "checker" understand that FD_ZERO() clears memory.])
diff --git a/db.c b/db.c
new file mode 100644
--- /dev/null
+++ b/db.c
@@ -0,0 +1,1912 @@
+/*
+ * Routines to access extended file info via DB.
+ *
+ * Copyright (C) 2008-2013 Wayne Davison
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License along
+ * with this program; if not, visit the http://fsf.org website.
+ */
+
+#include "rsync.h"
+#include "ifuncs.h"
+#include "itypes.h"
+#include "inums.h"
+
+extern int recurse;
+extern int same_db;
+extern int am_receiver;
+extern int am_generator;
+extern int checksum_type;
+extern int db_clean, db_check, db_do_md4, db_do_md5, db_update, db_lax, db_init, db_mounts;
+extern int db_output_name, db_output_sum, db_output_info, db_output_unchanged, db_output_dirs, db_output_msgs;
+extern int saw_db_output_opt, saw_db_sum_opt;
+extern char *db_config;
+
+/* TODO: make this configurable */
+#define RSYNCDB_MOUNTS "/usr/sbin/rsyncdb-mountinfo"
+
+#if defined HAVE_MYSQL_MYSQL_H && defined HAVE_LIBMYSQLCLIENT
+#define USE_MYSQL
+#include <mysql/mysql.h>
+#include <mysql/errmsg.h>
+#endif
+
+#if defined HAVE_SQLITE3_H && defined HAVE_LIBSQLITE3
+#define USE_SQLITE
+#include <sqlite3.h>
+#ifndef HAVE_SQLITE3_OPEN_V2
+#define sqlite3_open_v2(dbname, dbhptr, flags, vfs) \
+ sqlite3_open(dbname, dbhptr)
+#endif
+#ifndef HAVE_SQLITE3_PREPARE_V2
+#define sqlite3_prepare_v2 sqlite3_prepare
+#endif
+#define MAX_LOCK_FAILURES 10
+#define LOCK_FAIL_MSLEEP 100
+#endif
+
+#define DB_TYPE_NONE 0
+#define DB_TYPE_MYSQL 1
+#define DB_TYPE_SQLITE 2
+
+int use_db = DB_TYPE_NONE;
+int select_many_sums = 0;
+
+#define PREP_NORM 0
+#define PREP_MOUNT 1
+
+static const char *dbhost = NULL, *dbuser = NULL, *dbpass = NULL, *dbname = NULL;
+static unsigned int dbport = 0;
+static int transaction_state = -1;
+
+static union {
+#ifdef USE_MYSQL
+ MYSQL *mysql;
+#endif
+#ifdef USE_SQLITE
+ sqlite3 *sqlite;
+#endif
+ void *all;
+} dbh;
+
+#define SEL_DEV 0
+#define SEL_SUM 1
+#define REP_SUM 2
+#define UPD_CTIME 3
+#define INS_MOUNT 4
+#define UPD_MOUNT 5 /* SQLite only */
+#define SEL_MOUNT 6
+#define UN_MOUNT 7
+#define DEL_SUMS 8
+#define INS_PRESENT 9
+#define MAX_PREP_CNT 10
+
+#define MAX_BIND_CNT 7
+#define MAX_RESULT_BINDS 32
+
+static union {
+#ifdef USE_MYSQL
+ MYSQL_STMT *mysql;
+#endif
+#ifdef USE_SQLITE
+ sqlite3_stmt *sqlite;
+#endif
+ void *all;
+} statements[MAX_PREP_CNT];
+
+static int md_num;
+static enum logcode log_code;
+
+#ifdef USE_MYSQL
+static unsigned int bind_disk_id, bind_mdnum;
+static int64 bind_devno, bind_ino, bind_size, bind_mtime, bind_ctime;
+static char bind_sum[MAX_DIGEST_LEN];
+static unsigned long result_length[MAX_RESULT_BINDS];
+static my_bool result_is_null[MAX_RESULT_BINDS], result_error[MAX_RESULT_BINDS];
+#else
+static int64 bind_mtime;
+#endif
+static char bind_thishost[128+1], bind_mount_uniq[128+1];
+static unsigned long bind_thishost_len, bind_mount_uniq_len;
+
+static char *error_log;
+#if defined USE_SQLITE && defined SQLITE_CONFIG_LOG
+static FILE *error_log_fp;
+#endif
+
+#define PTR_SIZE (sizeof (struct file_struct *))
+
+static void update_mounts(void);
+
+struct name_list {
+ struct name_list *next;
+ char name[1];
+} *dirs_list;
+
+int db_read_config(enum logcode code, const char *config_file)
+{
+ char buf[2048], *cp;
+ FILE *fp;
+ int lineno = 0;
+
+ log_code = code;
+
+ bind_thishost_len = strlcpy(bind_thishost, "localhost", sizeof bind_thishost);
+
+ if (!(fp = fopen(config_file, "r"))) {
+ rsyserr(log_code, errno, "unable to open %s", config_file);
+ return 0;
+ }
+ if (DEBUG_GTE(DB, 1))
+ rprintf(FCLIENT, "[%s] Reading DB config from %s\n", who_am_i(), config_file);
+ while (fgets(buf, sizeof buf, fp)) {
+ lineno++;
+ if ((cp = strchr(buf, '#')) == NULL
+ && (cp = strchr(buf, '\r')) == NULL
+ && (cp = strchr(buf, '\n')) == NULL)
+ cp = buf + strlen(buf);
+ while (cp != buf && isSpace(cp-1)) cp--;
+ *cp = '\0';
+
+ if (!*buf)
+ continue;
+
+ if (!(cp = strchr(buf, ':')))
+ goto invalid_line;
+ *cp++ = '\0';
+
+ while (isSpace(cp)) cp++;
+ if (strcasecmp(buf, "dbhost") == 0)
+ dbhost = strdup(cp);
+ else if (strcasecmp(buf, "dbuser") == 0)
+ dbuser = strdup(cp);
+ else if (strcasecmp(buf, "dbpass") == 0)
+ dbpass = strdup(cp);
+ else if (strcasecmp(buf, "dbname") == 0)
+ dbname = strdup(cp);
+ else if (strcasecmp(buf, "dbport") == 0)
+ dbport = atoi(cp);
+ else if (strcasecmp(buf, "transaction") == 0)
+ transaction_state = atoi(cp) ? 0 : -1;
+ else if (strcasecmp(buf, "errlog") == 0)
+ error_log = strdup(cp);
+ else if (strcasecmp(buf, "thishost") == 0)
+ bind_thishost_len = strlcpy(bind_thishost, cp, sizeof bind_thishost);
+ else if (strcasecmp(buf, "dbtype") == 0) {
+#ifdef USE_MYSQL
+ if (strcasecmp(cp, "mysql") == 0) {
+ use_db = DB_TYPE_MYSQL;
+ continue;
+ }
+#endif
+#ifdef USE_SQLITE
+ if (strcasecmp(cp, "sqlite") == 0) {
+ use_db = DB_TYPE_SQLITE;
+ continue;
+ }
+#endif
+ rprintf(log_code,
+ "Unsupported dbtype on line #%d in %s.\n",
+ lineno, config_file);
+ use_db = DB_TYPE_NONE;
+ return 0;
+ } else {
+ invalid_line:
+ rprintf(log_code, "Invalid line #%d in %s\n",
+ lineno, config_file);
+ use_db = DB_TYPE_NONE;
+ return 0;
+ }
+ }
+ fclose(fp);
+
+ if (bind_thishost_len >= (int)sizeof bind_thishost)
+ bind_thishost_len = sizeof bind_thishost - 1;
+
+ if (!use_db || !dbname) {
+ rprintf(log_code, "Please specify at least dbtype and dbname in %s.\n", config_file);
+ use_db = DB_TYPE_NONE;
+ return 0;
+ }
+
+ md_num = checksum_type == 5 ? 5 : 4;
+
+ if (error_log) {
+ if (use_db != DB_TYPE_SQLITE)
+ rprintf(log_code, "Ignoring errlog setting for non-SQLite DB.\n");
+#ifndef SQLITE_CONFIG_LOG
+ else
+ rprintf(log_code, "Your sqlite doesn't support SQLITE_CONFIG_LOG.\n");
+#endif
+ }
+
+ return 1;
+}
+
+#if defined USE_SQLITE && defined SQLITE_CONFIG_LOG
+static void errorLogCallback(UNUSED(void *pArg), int iErrCode, const char *zMsg)
+{
+ fprintf(error_log_fp, "[%d] %s (%d)\n", (int)getpid(), zMsg, iErrCode);
+}
+#endif
+
+static int run_sql(const char *fmt, ...)
+{
+ va_list ap;
+ char *query;
+ int ok = 0, qlen;
+
+ va_start(ap, fmt);
+ qlen = vasprintf(&query, fmt, ap);
+ va_end(ap);
+ if (qlen < 0)
+ out_of_memory("run_sql");
+ if (DEBUG_GTE(DB, 3))
+ rprintf(FCLIENT, "[%s] SQL being run: %s\n", who_am_i(), query);
+
+ switch (use_db) {
+#ifdef USE_MYSQL
+ case DB_TYPE_MYSQL:
+ if (mysql_query(dbh.mysql, query) < 0) {
+ rprintf(FERROR, "Failed to run sql: %s\n", mysql_error(dbh.mysql));
+ rprintf(FERROR, "%s\n", query);
+ } else
+ ok = 1;
+ break;
+#endif
+#ifdef USE_SQLITE
+ case DB_TYPE_SQLITE: {
+ int rc, lock_failures = 0;
+ while (1) {
+ if ((rc = sqlite3_exec(dbh.sqlite, query, NULL, NULL, NULL)) == 0)
+ break;
+ if (rc != SQLITE_BUSY && rc != SQLITE_LOCKED)
+ break;
+ if (++lock_failures > MAX_LOCK_FAILURES)
+ break;
+ msleep(LOCK_FAIL_MSLEEP);
+ }
+ if (rc) {
+ rprintf(FERROR, "[%s] Failed to run sql: %s\n", who_am_i(), sqlite3_errmsg(dbh.sqlite));
+ rprintf(FERROR, "%s\n", query);
+ } else
+ ok = 1;
+ break;
+ }
+#endif
+ }
+
+ free(query);
+
+ return ok;
+}
+
+#ifdef USE_MYSQL
+static int prepare_mysql(int ndx, MYSQL_BIND *binds, int bind_cnt, const char *fmt, ...)
+{
+ va_list ap;
+ char *query;
+ int qlen, param_cnt;
+ MYSQL_STMT *stmt = mysql_stmt_init(dbh.mysql);
+
+ if (stmt == NULL)
+ out_of_memory("prepare_mysql");
+
+ va_start(ap, fmt);
+ qlen = vasprintf(&query, fmt, ap);
+ va_end(ap);
+ if (qlen < 0)
+ out_of_memory("prepare_mysql");
+ if (DEBUG_GTE(DB, 3))
+ rprintf(FCLIENT, "[%s] SQL being prepared: %s\n", who_am_i(), query);
+
+ if (mysql_stmt_prepare(stmt, query, qlen) != 0) {
+ rprintf(log_code, "[%s] Prepare failed: %s\n", who_am_i(), mysql_stmt_error(stmt));
+ rprintf(log_code, "%s\n", query);
+ free(query);
+ return 0;
+ }
+
+ if ((param_cnt = mysql_stmt_param_count(stmt)) != bind_cnt) {
+ rprintf(log_code, "[%s] Parameters in statement = %d, bind vars = %d\n",
+ who_am_i(), param_cnt, bind_cnt);
+ rprintf(log_code, "%s\n", query);
+ free(query);
+ return 0;
+ }
+ if (bind_cnt)
+ mysql_stmt_bind_param(stmt, binds);
+
+ statements[ndx].mysql = stmt;
+ free(query);
+
+ return 1;
+}
+#endif
+
+#ifdef USE_MYSQL
+static int prepare_mysql_queries(int type)
+{
+ MYSQL_BIND binds[MAX_BIND_CNT];
+ char *sql;
+
+ switch (type) {
+ case PREP_NORM:
+ sql="SELECT disk_id"
+ " FROM disk"
+ " WHERE host = ? AND devno = ?";
+ memset(binds, 0, sizeof binds);
+ binds[0].buffer_type = MYSQL_TYPE_STRING;
+ binds[0].buffer = &bind_thishost;
+ binds[0].buffer_length = bind_thishost_len;
+ binds[1].buffer_type = MYSQL_TYPE_LONGLONG;
+ binds[1].buffer = &bind_devno;
+ if (!prepare_mysql(SEL_DEV, binds, 2, sql))
+ return 0;
+
+ memset(binds, 0, sizeof binds);
+ binds[0].buffer_type = MYSQL_TYPE_LONG;
+ binds[0].buffer = &bind_disk_id;
+ binds[1].buffer_type = MYSQL_TYPE_LONGLONG;
+ binds[1].buffer = &bind_ino;
+ if (select_many_sums) {
+ sql="SELECT checksum, sum_type, size, mtime, ctime"
+ " FROM inode_map"
+ " WHERE disk_id = ? AND ino = ?";
+ if (!prepare_mysql(SEL_SUM, binds, 2, sql))
+ return 0;
+ } else {
+ sql="SELECT checksum"
+ " FROM inode_map"
+ " WHERE disk_id = ? AND ino = ? AND sum_type = %d"
+ " AND size = ? AND mtime = ? %s";
+ binds[2].buffer_type = MYSQL_TYPE_LONGLONG;
+ binds[2].buffer = &bind_size;
+ binds[3].buffer_type = MYSQL_TYPE_LONGLONG;
+ binds[3].buffer = &bind_mtime;
+ if (!db_lax) {
+ binds[4].buffer_type = MYSQL_TYPE_LONGLONG;
+ binds[4].buffer = &bind_ctime;
+ }
+ if (!prepare_mysql(SEL_SUM, binds, 4 + !db_lax, sql, md_num, db_lax ? "" : "AND ctime = ?"))
+ return 0;
+ }
+
+ sql="INSERT INTO inode_map"
+ " SET disk_id = ?, ino = ?, sum_type = ?,"
+ " size = ?, mtime = ?, ctime = ?, checksum = ?"
+ " ON DUPLICATE KEY"
+ " UPDATE size = VALUES(size), mtime = VALUES(mtime),"
+ " ctime = VALUES(ctime), checksum = VALUES(checksum)";
+ memset(binds, 0, sizeof binds);
+ binds[0].buffer_type = MYSQL_TYPE_LONG;
+ binds[0].buffer = &bind_disk_id;
+ binds[1].buffer_type = MYSQL_TYPE_LONGLONG;
+ binds[1].buffer = &bind_ino;
+ binds[2].buffer_type = MYSQL_TYPE_LONG;
+ binds[2].buffer = &bind_mdnum;
+ binds[3].buffer_type = MYSQL_TYPE_LONGLONG;
+ binds[3].buffer = &bind_size;
+ binds[4].buffer_type = MYSQL_TYPE_LONGLONG;
+ binds[4].buffer = &bind_mtime;
+ binds[5].buffer_type = MYSQL_TYPE_LONGLONG;
+ binds[5].buffer = &bind_ctime;
+ binds[6].buffer_type = MYSQL_TYPE_BLOB;
+ binds[6].buffer = &bind_sum;
+ binds[6].buffer_length = MD5_DIGEST_LEN; /* Same as MD4_DIGEST_LEN */
+ if (!prepare_mysql(REP_SUM, binds, 7, sql))
+ return 0;
+
+ sql="UPDATE inode_map"
+ " SET ctime = ?"
+ " WHERE disk_id = ? AND ino = ? AND sum_type = ? AND size = ? AND mtime = ?";
+ memset(binds, 0, sizeof binds);
+ binds[0].buffer_type = MYSQL_TYPE_LONGLONG;
+ binds[0].buffer = &bind_ctime;
+ binds[1].buffer_type = MYSQL_TYPE_LONG;
+ binds[1].buffer = &bind_disk_id;
+ binds[2].buffer_type = MYSQL_TYPE_LONGLONG;
+ binds[2].buffer = &bind_ino;
+ binds[3].buffer_type = MYSQL_TYPE_LONG;
+ binds[3].buffer = &bind_mdnum;
+ binds[4].buffer_type = MYSQL_TYPE_LONGLONG;
+ binds[4].buffer = &bind_size;
+ binds[5].buffer_type = MYSQL_TYPE_LONGLONG;
+ binds[5].buffer = &bind_mtime;
+ if (!prepare_mysql(UPD_CTIME, binds, 6, sql))
+ return 0;
+ break;
+
+ case PREP_MOUNT:
+ sql="INSERT INTO disk"
+ " SET host = ?, last_seen = ?, mount_uniq = ?, devno = ?"
+ " ON DUPLICATE KEY"
+ " UPDATE last_seen = VALUES(last_seen), devno = VALUES(devno)";
+ memset(binds, 0, sizeof binds);
+ binds[0].buffer_type = MYSQL_TYPE_STRING;
+ binds[0].buffer = &bind_thishost;
+ binds[0].buffer_length = bind_thishost_len;
+ binds[1].buffer_type = MYSQL_TYPE_LONGLONG;
+ binds[1].buffer = &bind_mtime; /* we abuse mtime to hold the last_seen value */
+ binds[2].buffer_type = MYSQL_TYPE_STRING;
+ binds[2].buffer = &bind_mount_uniq;
+ binds[2].buffer_length = sizeof bind_mount_uniq;
+ binds[2].length = &bind_mount_uniq_len;
+ binds[3].buffer_type = MYSQL_TYPE_LONGLONG;
+ binds[3].buffer = &bind_devno;
+ if (!prepare_mysql(INS_MOUNT, binds, 4, sql))
+ return 0;
+
+ sql="SELECT mount_uniq"
+ " FROM disk"
+ " WHERE host = ? AND last_seen < ? AND devno != 0";
+ /* Reusing first 2 binds from INS_MOUNT */
+ if (!prepare_mysql(SEL_MOUNT, binds, 2, sql))
+ return 0;
+
+ sql="UPDATE disk"
+ " SET devno = 0"
+ " WHERE host = ? AND last_seen < ? AND devno != 0";
+ /* Reusing binds from SEL_MOUNT */
+ if (!prepare_mysql(UN_MOUNT, binds, 2, sql))
+ return 0;
+ break;
+ }
+
+ return 1;
+}
+#endif
+
+#ifdef USE_MYSQL
+static int db_connect_mysql(void)
+{
+ const char *open_dbname = db_init ? "mysql" : dbname;
+
+ if (!(dbh.mysql = mysql_init(NULL)))
+ out_of_memory("db_read_config");
+
+ if (DEBUG_GTE(DB, 1)) {
+ rprintf(FCLIENT, "[%s] connecting: host=%s user=%s db=%s port=%d\n",
+ who_am_i(), dbhost, dbuser, open_dbname, dbport);
+ }
+ if (!mysql_real_connect(dbh.mysql, dbhost, dbuser, dbpass, open_dbname, dbport, NULL, 0)) {
+ rprintf(log_code, "[%s] Unable to connect to DB: %s\n", who_am_i(), mysql_error(dbh.mysql));
+ return 0;
+ }
+
+ if (db_init) {
+ if (db_output_msgs)
+ rprintf(FCLIENT, "Creating DB %s (if it does not exist)\n", dbname);
+ if (!run_sql("CREATE DATABASE IF NOT EXISTS `%s`", dbname)
+ || !run_sql("USE `%s`", dbname))
+ exit_cleanup(RERR_IPC);
+
+ if (db_output_msgs)
+ rprintf(FCLIENT, "Dropping old tables (if they exist))\n");
+ if (!run_sql("DROP TABLE IF EXISTS disk")
+ || !run_sql("DROP TABLE IF EXISTS inode_map"))
+ exit_cleanup(RERR_IPC);
+
+ if (db_output_msgs)
+ rprintf(FCLIENT, "Creating empty tables ...\n");
+ if (!run_sql(
+ "CREATE TABLE disk (\n"
+ " disk_id integer unsigned NOT NULL PRIMARY KEY AUTO_INCREMENT,\n"
+ " host varchar(128) NOT NULL default 'localhost',\n"
+ " mount_uniq varchar(128) default NULL,\n"
+ " devno bigint unsigned NOT NULL,\n" /* This is 0 when not mounted */
+ " last_seen bigint NOT NULL,\n"
+ " UNIQUE KEY mount_lookup (host, mount_uniq),\n"
+ " KEY dev_lookup (devno, host)\n"
+ ")"))
+ exit_cleanup(RERR_IPC);
+
+ if (!run_sql(
+ "CREATE TABLE inode_map (\n"
+ " disk_id integer unsigned NOT NULL,\n"
+ " ino bigint unsigned NOT NULL,\n"
+ " sum_type tinyint NOT NULL default '0',\n"
+ " size bigint unsigned NOT NULL,\n"
+ " mtime bigint NOT NULL,\n"
+ " ctime bigint NOT NULL,\n"
+ " checksum binary(16) NOT NULL,\n"
+ " PRIMARY KEY (disk_id,ino,sum_type)\n"
+ ")"))
+ exit_cleanup(RERR_IPC);
+
+ if (!db_mounts)
+ exit_cleanup(0);
+ }
+
+ if (db_mounts) {
+ if (!prepare_mysql_queries(PREP_MOUNT))
+ exit_cleanup(RERR_IPC);
+ update_mounts();
+ exit_cleanup(0);
+ }
+
+ if (!prepare_mysql_queries(PREP_NORM))
+ return 0;
+
+ return 1;
+}
+#endif
+
+#ifdef USE_SQLITE
+static int prepare_sqlite(int ndx, const char *fmt, ...)
+{
+ va_list ap;
+ char *query;
+ int rc, qlen, lock_failures = 0;
+
+ va_start(ap, fmt);
+ qlen = vasprintf(&query, fmt, ap);
+ va_end(ap);
+ if (qlen < 0)
+ out_of_memory("prepare_sqlite");
+ if (DEBUG_GTE(DB, 3))
+ rprintf(FCLIENT, "[%s] SQL being prepared: %s\n", who_am_i(), query);
+
+ while ((rc = sqlite3_prepare_v2(dbh.sqlite, query, -1, &statements[ndx].sqlite, NULL)) != 0) {
+ if (DEBUG_GTE(DB, 4)) {
+ rprintf(FCLIENT, "[%s] sqlite3_prepare_v2(,%s,,) returned %d\n",
+ who_am_i(), query, rc);
+ }
+ if (rc != SQLITE_BUSY && rc != SQLITE_LOCKED)
+ break;
+ if (++lock_failures > MAX_LOCK_FAILURES)
+ break;
+ msleep(LOCK_FAIL_MSLEEP);
+ }
+ if (rc) {
+ rprintf(log_code, "[%s] Failed to prepare SQL: %s (%d)\n", who_am_i(), sqlite3_errmsg(dbh.sqlite), rc);
+ rprintf(log_code, "%s\n", query);
+ free(query);
+ return 0;
+ }
+ free(query);
+
+ return 1;
+}
+#endif
+
+#ifdef USE_SQLITE
+static int prepare_sqlite_queries(int type)
+{
+ char *sql;
+
+ switch (type) {
+ case PREP_NORM:
+ sql="SELECT disk_id"
+ " FROM disk"
+ " WHERE host = ? AND devno = ?";
+ if (!prepare_sqlite(SEL_DEV, sql))
+ return 0;
+
+ if (select_many_sums) {
+ sql="SELECT checksum, sum_type, size, mtime, ctime"
+ " FROM inode_map"
+ " WHERE disk_id = ? AND ino = ?";
+ if (!prepare_sqlite(SEL_SUM, sql))
+ return 0;
+ } else {
+ sql="SELECT checksum"
+ " FROM inode_map"
+ " WHERE disk_id = ? AND ino = ? AND sum_type = %d"
+ " AND size = ? AND mtime = ? %s";
+ if (!prepare_sqlite(SEL_SUM, sql, md_num, db_lax ? "" : "AND ctime = ?"))
+ return 0;
+ }
+
+ sql="INSERT OR REPLACE INTO inode_map"
+ " (disk_id, ino, sum_type, size, mtime, ctime, checksum)"
+ " VALUES (?, ?, ?, ?, ?, ?, ?)";
+ if (!prepare_sqlite(REP_SUM, sql))
+ return 0;
+
+ sql="UPDATE inode_map"
+ " SET ctime = ?"
+ " WHERE disk_id = ? AND ino = ? AND sum_type = ? AND size = ? AND mtime = ?";
+ if (!prepare_sqlite(UPD_CTIME, sql))
+ return 0;
+ break;
+
+ case PREP_MOUNT:
+ sql="INSERT OR IGNORE INTO disk"
+ " (host, last_seen, mount_uniq, devno)"
+ " VALUES (?, ?, ?, ?)";
+ if (!prepare_sqlite(INS_MOUNT, sql))
+ return 0;
+
+ sql="UPDATE disk"
+ " SET last_seen = ?, devno = ?"
+ " WHERE host = ? AND mount_uniq = ?";
+ if (!prepare_sqlite(UPD_MOUNT, sql))
+ return 0;
+
+ sql="SELECT mount_uniq"
+ " FROM disk"
+ " WHERE host = ? AND last_seen < ? AND devno != 0";
+ if (!prepare_sqlite(SEL_MOUNT, sql))
+ return 0;
+
+ sql="UPDATE disk"
+ " SET devno = 0"
+ " WHERE host = ? AND last_seen < ? AND devno != 0";
+ if (!prepare_sqlite(UN_MOUNT, sql))
+ return 0;
+ break;
+ }
+
+ return 1;
+}
+#endif
+
+#ifdef USE_SQLITE
+static int db_connect_sqlite(void)
+{
+ int lock_failures = 0;
+ int rc;
+
+#ifdef SQLITE_CONFIG_LOG
+ if (error_log) {
+ if (DEBUG_GTE(DB, 1))
+ rprintf(FCLIENT, "[%s] Setting sqlite errlog to %s\n", who_am_i(), error_log);
+ if (!(error_log_fp = fopen(error_log, "a"))) {
+ rsyserr(log_code, errno, "unable to append to logfile %s", error_log);
+ error_log = NULL;
+ } else if (sqlite3_config(SQLITE_CONFIG_LOG, errorLogCallback, NULL) != 0)
+ rprintf(log_code, "Failed to set errorLogCallback: %s\n", sqlite3_errmsg(dbh.sqlite));
+ }
+#endif
+
+ while (1) {
+ int open_flags = SQLITE_OPEN_READWRITE;
+ if (db_init)
+ open_flags |= SQLITE_OPEN_CREATE;
+ if (DEBUG_GTE(DB, 1))
+ rprintf(FCLIENT, "[%s] opening %s (%d)\n", who_am_i(), dbname, open_flags);
+ if ((rc = sqlite3_open_v2(dbname, &dbh.sqlite, open_flags, NULL)) == 0) {
+ break;
+ }
+ if (DEBUG_GTE(DB, 4)) {
+ rprintf(FCLIENT, "[%s] sqlite3_open_v2(%s,,%d,NULL) returned %d\n",
+ who_am_i(), dbname, open_flags, rc);
+ }
+ if (rc != SQLITE_BUSY && rc != SQLITE_LOCKED)
+ break;
+ if (++lock_failures > MAX_LOCK_FAILURES)
+ break;
+ msleep(LOCK_FAIL_MSLEEP);
+ }
+
+ if (rc) {
+ rprintf(log_code, "Unable to connect to DB: %s (%d)\n", sqlite3_errmsg(dbh.sqlite), rc);
+ return 0;
+ }
+
+ if (db_init) {
+ char *sql;
+ if (db_output_msgs)
+ rprintf(FCLIENT, "Dropping old tables (if they exist) ...\n");
+ if (!run_sql("DROP TABLE IF EXISTS disk")
+ || !run_sql("DROP TABLE IF EXISTS inode_map"))
+ exit_cleanup(RERR_IPC);
+
+ if (db_output_msgs)
+ rprintf(FCLIENT, "Creating empty tables ...\n");
+ sql="CREATE TABLE disk (\n"
+ " disk_id integer NOT NULL PRIMARY KEY AUTOINCREMENT,\n"
+ " host varchar(128) NOT NULL default 'localhost',\n"
+ " mount_uniq varchar(128) default NULL,\n"
+ " devno bigint NOT NULL,\n" /* This is 0 when not mounted */
+ " last_seen bigint NOT NULL,\n"
+ " UNIQUE (host, mount_uniq)\n"
+ ")";
+ if (!run_sql(sql))
+ exit_cleanup(RERR_IPC);
+
+ sql="CREATE TABLE inode_map (\n"
+ " disk_id integer NOT NULL,\n"
+ " ino bigint NOT NULL,\n"
+ " size bigint NOT NULL,\n"
+ " mtime bigint NOT NULL,\n"
+ " ctime bigint NOT NULL,\n"
+ " sum_type tinyint NOT NULL default '0',\n"
+ " checksum binary(16) NOT NULL,\n"
+ " PRIMARY KEY (disk_id,ino,sum_type)\n"
+ ")";
+ if (!run_sql(sql))
+ exit_cleanup(RERR_IPC);
+
+#if SQLITE_VERSION_NUMBER >= 3007000
+ /* Using WAL locking makes concurrency much better (requires sqlite 3.7.0). */
+ sql="PRAGMA journal_mode = wal";
+ run_sql(sql); /* We don't check this for success. */
+#endif
+
+ if (!db_mounts)
+ exit_cleanup(0);
+ }
+
+ if (db_mounts) {
+ if (!prepare_sqlite_queries(PREP_MOUNT))
+ exit_cleanup(RERR_IPC);
+ update_mounts();
+ exit_cleanup(0);
+ }
+
+ if (!prepare_sqlite_queries(PREP_NORM)) {
+ db_disconnect(False);
+ return 0;
+ }
+
+ return 1;
+}
+#endif
+
+int db_connect(int select_many)
+{
+ select_many_sums = select_many;
+
+ switch (use_db) {
+#ifdef USE_MYSQL
+ case DB_TYPE_MYSQL:
+ if (db_connect_mysql())
+ return 1;
+ break;
+#endif
+#ifdef USE_SQLITE
+ case DB_TYPE_SQLITE:
+ if (db_connect_sqlite())
+ return 1;
+ break;
+#endif
+ }
+
+ db_disconnect(False);
+
+ return 0;
+}
+
+void db_disconnect(BOOL commit)
+{
+ int ndx;
+
+ if (!dbh.all)
+ return;
+
+ if (transaction_state > 0) {
+ if (DEBUG_GTE(DB, 1)) {
+ rprintf(FCLIENT, "[%s] %s our DB transaction\n",
+ who_am_i(), commit ? "Committing" : "Rolling back");
+ }
+ transaction_state = 0;
+ if (commit)
+ run_sql("COMMIT");
+ else
+ run_sql("ROLLBACK");
+ }
+
+ if (DEBUG_GTE(DB, 1))
+ rprintf(FCLIENT, "[%s] Disconnecting from the DB\n", who_am_i());
+
+ for (ndx = 0; ndx < MAX_PREP_CNT; ndx++) {
+ if (statements[ndx].all) {
+ switch (use_db) {
+#ifdef USE_MYSQL
+ case DB_TYPE_MYSQL:
+ mysql_stmt_close(statements[ndx].mysql);
+ break;
+#endif
+#ifdef USE_SQLITE
+ case DB_TYPE_SQLITE:
+ sqlite3_finalize(statements[ndx].sqlite);
+ break;
+#endif
+ }
+ statements[ndx].all = NULL;
+ }
+ }
+
+ switch (use_db) {
+#ifdef USE_MYSQL
+ case DB_TYPE_MYSQL:
+ mysql_close(dbh.mysql);
+ break;
+#endif
+#ifdef USE_SQLITE
+ case DB_TYPE_SQLITE:
+ sqlite3_close(dbh.sqlite);
+ break;
+#endif
+ }
+
+ dbh.all = NULL;
+ use_db = DB_TYPE_NONE;
+}
+
+#ifdef USE_MYSQL
+static MYSQL_STMT *exec_mysql(int ndx)
+{
+ MYSQL_STMT *stmt = statements[ndx].mysql;
+ int rc;
+
+ if ((rc = mysql_stmt_execute(stmt)) == CR_SERVER_LOST) {
+ db_disconnect(False);
+ use_db = DB_TYPE_MYSQL;
+ if (db_connect(select_many_sums)) {
+ stmt = statements[ndx].mysql;
+ rc = mysql_stmt_execute(stmt);
+ }
+ }
+ if (rc != 0) {
+ rprintf(log_code, "SQL execute failed: %s\n", mysql_stmt_error(stmt));
+ return NULL;
+ }
+
+ return stmt;
+}
+#endif
+
+#ifdef USE_MYSQL
+/* This stores up to max_rows into the values pointed to by the bind data arrays.
+ * If max_rows is > 1, then all the buffer pointers MUST be set to an array long
+ * enough to hold the max count of rows. The buffer pointer will be incremented
+ * to read additional rows (but never past the end). If stmt_ptr is non-NULL, it
+ * will be set to the "stmt" pointer IFF we didn't run out of rows before hitting
+ * the max. In this case, the caller should call mysql_stmt_fetch() to read any
+ * remaining rows (the buffer pointers will point at the final array element) and
+ * then call mysql_stmt_free_result(). If *stmt_ptr is a NULL value, there were
+ * not enough rows to fill the max_rows arrays, and the stmt was already freed. */
+static int fetch_mysql(MYSQL_BIND *binds, int bind_cnt, int ndx, int max_rows, MYSQL_STMT **stmt_ptr)
+{
+ MYSQL_STMT *stmt;
+ int i, rc, rows = 0;
+
+ if (bind_cnt > MAX_RESULT_BINDS) {
+ fprintf(stderr, "Internal error: MAX_RESULT_BINDS overflow\n");
+ exit_cleanup(RERR_UNSUPPORTED);
+ }
+
+ if ((stmt = exec_mysql(ndx)) == NULL)
+ return 0;
+
+ for (i = 0; i < bind_cnt; i++) {
+ binds[i].is_null = &result_is_null[i];
+ binds[i].length = &result_length[i];
+ binds[i].error = &result_error[i];
+ }
+ mysql_stmt_bind_result(stmt, binds);
+
+ while (rows < max_rows) {
+ if ((rc = mysql_stmt_fetch(stmt)) != 0) {
+ if (rc != MYSQL_NO_DATA)
+ rprintf(log_code, "SELECT fetch failed: %s\n", mysql_stmt_error(stmt));
+ break;
+ }
+ if (++rows >= max_rows)
+ break;
+ for (i = 0; i < bind_cnt; i++) {
+ switch (binds[i].buffer_type) {
+ case MYSQL_TYPE_BLOB:
+ case MYSQL_TYPE_STRING:
+ binds[i].buffer += binds[i].buffer_length;
+ break;
+ case MYSQL_TYPE_LONG:
+ binds[i].buffer += sizeof (int);
+ break;
+ case MYSQL_TYPE_LONGLONG:
+ binds[i].buffer += sizeof (int64);
+ break;
+ default:
+ fprintf(stderr, "Unknown MYSQL_TYPE_* in multi-row read: %d.\n", binds[i].buffer_type);
+ exit_cleanup(RERR_UNSUPPORTED);
+ }
+ }
+ }
+
+ if (!stmt_ptr || rows < max_rows) {
+ mysql_stmt_free_result(stmt);
+ stmt = NULL;
+ }
+ if (stmt_ptr)
+ *stmt_ptr = stmt;
+
+ return rows;
+}
+#endif
+
+static void update_mounts(void)
+{
+ char buf[2048], *argv[2];
+ int f_from, f_to, len;
+ STRUCT_STAT st;
+ int pid, status;
+
+ if (DEBUG_GTE(DB, 2))
+ printf("Running %s to grab mount info\n", RSYNCDB_MOUNTS);
+ argv[0] = RSYNCDB_MOUNTS;
+ argv[1] = NULL;
+ pid = piped_child(argv, &f_from, &f_to);
+ close(f_to);
+
+ bind_mtime = time(NULL); /* abuse mtime slightly to hold our last_seen value */
+
+ /* Strict format has 2 items with one tab as separator: MOUNT_UNIQ\tPATH */
+ while ((len = read_line(f_from, buf, sizeof buf, 0)) > 0) {
+ char *mount_uniq, *path;
+
+ if (DEBUG_GTE(DB, 3))
+ printf("Parsing mount info: %s\n", buf);
+ mount_uniq = strtok(buf, "\t");
+ path = mount_uniq ? strtok(NULL, "\r\n") : NULL;
+ if (!path) {
+ fprintf(stderr, "Failed to parse line from %s output\n", RSYNCDB_MOUNTS);
+ exit_cleanup(RERR_SYNTAX);
+ }
+
+ if (lstat(path, &st) < 0) {
+ fprintf(stderr, "Failed to lstat(%s): %s\n", path, strerror(errno));
+ exit_cleanup(RERR_IPC);
+ }
+
+ bind_mount_uniq_len = strlcpy(bind_mount_uniq, mount_uniq, sizeof bind_mount_uniq);
+ if (bind_mount_uniq_len >= (int)sizeof bind_mount_uniq)
+ bind_mount_uniq_len = sizeof bind_mount_uniq - 1;
+
+ if (db_output_msgs) {
+ printf("Marking mount \"%s\" (%s) as a recent mount\n",
+ bind_mount_uniq, big_num(st.st_dev));
+ }
+ switch (use_db) {
+#ifdef USE_MYSQL
+ case DB_TYPE_MYSQL:
+ bind_devno = st.st_dev;
+ if (exec_mysql(INS_MOUNT) == NULL) {
+ fprintf(stderr, "Failed to update mount info for \"%s\" - %s\n",
+ bind_mount_uniq, mysql_error(dbh.mysql));
+ exit_cleanup(RERR_IPC);
+ }
+ break;
+#endif
+#ifdef USE_SQLITE
+ case DB_TYPE_SQLITE: {
+ int rc, change_cnt;
+ sqlite3_stmt *stmt = statements[INS_MOUNT].sqlite;
+ sqlite3_bind_text(stmt, 1, bind_thishost, bind_thishost_len, SQLITE_STATIC);
+ sqlite3_bind_int64(stmt, 2, bind_mtime);
+ sqlite3_bind_text(stmt, 3, bind_mount_uniq, bind_mount_uniq_len, SQLITE_STATIC);
+ sqlite3_bind_int64(stmt, 4, st.st_dev);
+ rc = sqlite3_step(stmt);
+ if (rc != SQLITE_DONE) {
+ fprintf(stderr, "Failed to insert mount info for \"%s\" - %s (%d)\n",
+ bind_mount_uniq, sqlite3_errmsg(dbh.sqlite), rc);
+ exit_cleanup(RERR_IPC);
+ }
+ change_cnt = sqlite3_changes(dbh.sqlite);
+ sqlite3_reset(stmt);
+ if (change_cnt == 0) {
+ stmt = statements[UPD_MOUNT].sqlite;
+ sqlite3_bind_int64(stmt, 1, bind_mtime);
+ sqlite3_bind_int64(stmt, 2, st.st_dev);
+ sqlite3_bind_text(stmt, 3, bind_thishost, bind_thishost_len, SQLITE_STATIC);
+ sqlite3_bind_text(stmt, 4, bind_mount_uniq, bind_mount_uniq_len, SQLITE_STATIC);
+ rc = sqlite3_step(stmt);
+ if (rc != SQLITE_DONE) {
+ fprintf(stderr, "Failed to update mount info for \"%s\" - %s (%d)\n",
+ bind_mount_uniq, sqlite3_errmsg(dbh.sqlite), rc);
+ exit_cleanup(RERR_IPC);
+ }
+ sqlite3_reset(stmt);
+ }
+ break;
+ }
+#endif
+ }
+ }
+ close(f_from);
+
+ waitpid(pid, &status, 0);
+
+ switch (use_db) {
+#ifdef USE_MYSQL
+ case DB_TYPE_MYSQL: {
+ if (db_output_msgs) {
+ MYSQL_BIND binds[1];
+ MYSQL_STMT *stmt;
+
+ binds[0].buffer_type = MYSQL_TYPE_BLOB;
+ binds[0].buffer = bind_mount_uniq;
+ binds[0].buffer_length = sizeof bind_mount_uniq;
+ if (fetch_mysql(binds, 1, SEL_MOUNT, 1, &stmt)) {
+ while (1) {
+ printf("Marking mount \"%s\" as unmounted.\n", bind_mount_uniq);
+ if (mysql_stmt_fetch(stmt) != 0)
+ break;
+ }
+ mysql_stmt_free_result(stmt);
+ }
+ }
+
+ if (exec_mysql(UN_MOUNT) == NULL) {
+ fprintf(stderr, "Failed to update old mount info - %s\n",
+ mysql_error(dbh.mysql));
+ exit_cleanup(RERR_IPC);
+ }
+ break;
+ }
+#endif
+#ifdef USE_SQLITE
+ case DB_TYPE_SQLITE: {
+ sqlite3_stmt *stmt;
+ int rc;
+
+ if (db_output_msgs) {
+ stmt = statements[SEL_MOUNT].sqlite;
+ sqlite3_bind_text(stmt, 1, bind_thishost, bind_thishost_len, SQLITE_STATIC);
+ sqlite3_bind_int64(stmt, 2, bind_mtime);
+ while (1) {
+ if (sqlite3_step(stmt) != SQLITE_ROW)
+ break;
+ printf("Marking mount \"%s\" as unmounted.\n", sqlite3_column_text(stmt, 0));
+ }
+ sqlite3_reset(stmt);
+ }
+
+ stmt = statements[UN_MOUNT].sqlite;
+ sqlite3_bind_text(stmt, 1, bind_thishost, bind_thishost_len, SQLITE_STATIC);
+ sqlite3_bind_int64(stmt, 2, bind_mtime);
+ rc = sqlite3_step(stmt);
+ sqlite3_reset(stmt);
+ if (rc != SQLITE_DONE) {
+ fprintf(stderr, "Failed to update old mount info - %s (%d)\n",
+ sqlite3_errmsg(dbh.sqlite), rc);
+ exit_cleanup(RERR_IPC);
+ }
+ break;
+ }
+#endif
+ }
+}
+
+unsigned int get_disk_id(int64 devno)
+{
+ static unsigned int prior_disk_id = 0;
+ static int64 prior_devno = 0;
+
+ if (prior_devno == devno && prior_disk_id) {
+ if (DEBUG_GTE(DB, 5))
+ rprintf(FCLIENT, "get_disk_id(%s,%s) = %d (cached)\n", bind_thishost, big_num(devno), prior_disk_id);
+ return prior_disk_id;
+ }
+ prior_devno = devno;
+
+ switch (use_db) {
+#ifdef USE_MYSQL
+ case DB_TYPE_MYSQL: {
+ MYSQL_BIND binds[1];
+
+ bind_devno = devno; /* The one changing SEL_DEV input value. */
+
+ /* Bind where to put the output. */
+ binds[0].buffer_type = MYSQL_TYPE_LONG;
+ binds[0].buffer = &prior_disk_id;
+ if (!fetch_mysql(binds, 1, SEL_DEV, 1, NULL))
+ prior_disk_id = 0;
+ break;
+ }
+#endif
+#ifdef USE_SQLITE
+ case DB_TYPE_SQLITE: {
+ sqlite3_stmt *stmt = statements[SEL_DEV].sqlite;
+ sqlite3_bind_text(stmt, 1, bind_thishost, bind_thishost_len, SQLITE_STATIC);
+ sqlite3_bind_int64(stmt, 2, devno);
+ if (sqlite3_step(stmt) == SQLITE_ROW)
+ prior_disk_id = sqlite3_column_int(stmt, 0);
+ else
+ prior_disk_id = 0;
+ sqlite3_reset(stmt);
+ break;
+ }
+#endif
+ }
+
+ if (DEBUG_GTE(DB, 2))
+ rprintf(FCLIENT, "get_disk_id(%s,%s) = %d\n", bind_thishost, big_num(devno), prior_disk_id);
+ return prior_disk_id;
+}
+
+int db_get_checksum(const STRUCT_STAT *st_p, char *sum)
+{
+ unsigned int disk_id = get_disk_id(st_p->st_dev);
+ int ok = 0;
+
+ if (disk_id == 0)
+ return 0;
+
+ switch (use_db) {
+#ifdef USE_MYSQL
+ case DB_TYPE_MYSQL: {
+ MYSQL_BIND binds[1];
+
+ bind_disk_id = disk_id;
+ bind_ino = st_p->st_ino;
+ bind_size = st_p->st_size;
+ bind_mtime = st_p->st_mtime;
+ if (!db_lax)
+ bind_ctime = st_p->st_ctime;
+
+ binds[0].buffer_type = MYSQL_TYPE_BLOB;
+ binds[0].buffer = sum;
+ binds[0].buffer_length = MD5_DIGEST_LEN;
+ ok = fetch_mysql(binds, 1, SEL_SUM, 1, NULL);
+ break;
+ }
+#endif
+#ifdef USE_SQLITE
+ case DB_TYPE_SQLITE: {
+ sqlite3_stmt *stmt = statements[SEL_SUM].sqlite;
+ sqlite3_bind_int(stmt, 1, disk_id);
+ sqlite3_bind_int64(stmt, 2, st_p->st_ino);
+ sqlite3_bind_int64(stmt, 3, st_p->st_size);
+ sqlite3_bind_int64(stmt, 4, st_p->st_mtime);
+ if (!db_lax)
+ sqlite3_bind_int64(stmt, 5, st_p->st_ctime);
+ if (sqlite3_step(stmt) == SQLITE_ROW) {
+ int len = sqlite3_column_bytes(stmt, 0);
+ if (len > MAX_DIGEST_LEN)
+ len = MAX_DIGEST_LEN;
+ memcpy(sum, sqlite3_column_blob(stmt, 0), len);
+ ok = 1;
+ }
+ sqlite3_reset(stmt);
+ break;
+ }
+#endif
+ }
+
+ if (DEBUG_GTE(DB, 2)) {
+ if (ok) {
+ rprintf(FCLIENT, "[%s] Found DB checksum for %s,%s,%d: %s\n",
+ who_am_i(), big_num(st_p->st_dev),
+ big_num(st_p->st_ino), md_num, sum_as_hex(md_num, sum, 0));
+ } else {
+ rprintf(FCLIENT, "[%s] No DB checksum for %s,%s,%d\n",
+ who_am_i(), big_num(st_p->st_dev),
+ big_num(st_p->st_ino), md_num);
+ }
+ }
+
+ return ok;
+}
+
+int db_get_both_checksums(const STRUCT_STAT *st_p, int *right_sum_cnt, int *wrong_sum_cnt, char **sum4, char **sum5)
+{
+ static char dbsum[MD5_DIGEST_LEN*2];
+ int rows, j, sum_type[2];
+ int64 dbsize[2], dbmtime[2], dbctime[2];
+ unsigned int disk_id = get_disk_id(st_p->st_dev);
+
+ if (disk_id == 0)
+ return 0;
+
+ switch (use_db) {
+#ifdef USE_MYSQL
+ case DB_TYPE_MYSQL: {
+ MYSQL_BIND binds[5];
+
+ bind_disk_id = disk_id;
+ bind_ino = st_p->st_ino;
+
+ binds[0].buffer_type = MYSQL_TYPE_BLOB;
+ binds[0].buffer = dbsum;
+ binds[0].buffer_length = MD5_DIGEST_LEN;
+ binds[1].buffer_type = MYSQL_TYPE_LONG;
+ binds[1].buffer = (char*)sum_type;
+ binds[2].buffer_type = MYSQL_TYPE_LONGLONG;
+ binds[2].buffer = (char*)dbsize;
+ binds[3].buffer_type = MYSQL_TYPE_LONGLONG;
+ binds[3].buffer = (char*)dbmtime;
+ binds[4].buffer_type = MYSQL_TYPE_LONGLONG;
+ binds[4].buffer = (char*)dbctime;
+ rows = fetch_mysql(binds, 5, SEL_SUM, 2, NULL);
+ break;
+ }
+#endif
+#ifdef USE_SQLITE
+ case DB_TYPE_SQLITE: {
+ sqlite3_stmt *stmt = statements[SEL_SUM].sqlite;
+ sqlite3_bind_int(stmt, 1, disk_id);
+ sqlite3_bind_int64(stmt, 2, st_p->st_ino);
+ for (j = 0; j < 2; j++) {
+ int len;
+ if (sqlite3_step(stmt) != SQLITE_ROW)
+ break;
+ len = sqlite3_column_bytes(stmt, 0);
+ if (len > MD5_DIGEST_LEN)
+ len = MD5_DIGEST_LEN;
+ memcpy(dbsum + MD5_DIGEST_LEN*j, sqlite3_column_blob(stmt, 0), len);
+ sum_type[j] = sqlite3_column_int(stmt, 1);
+ dbsize[j] = sqlite3_column_int(stmt, 2);
+ dbmtime[j] = sqlite3_column_int64(stmt, 3);
+ dbctime[j] = sqlite3_column_int64(stmt, 4);
+ }
+ sqlite3_reset(stmt);
+ rows = j;
+ break;
+ }
+#endif
+ default:
+ return 0;
+ }
+
+ if (sum4)
+ *sum4 = NULL;
+ if (sum5)
+ *sum5 = NULL;
+ *right_sum_cnt = *wrong_sum_cnt = 0;
+ for (j = 0; j < rows; j++) {
+ if (DEBUG_GTE(DB, 3)) {
+ rprintf(FCLIENT, "DB checksum for %s,%s,%d: %s\n",
+ big_num(st_p->st_dev), big_num(st_p->st_ino), sum_type[j],
+ sum_as_hex(sum_type[j], dbsum + MD5_DIGEST_LEN*j, 0));
+ }
+
+ if (sum_type[j] == 4) {
+ if (!sum4)
+ continue;
+ *sum4 = dbsum + MD5_DIGEST_LEN*j;
+ } else {
+ if (!sum5)
+ continue;
+ *sum5 = dbsum + MD5_DIGEST_LEN*j;
+ }
+ if (st_p->st_size == dbsize[j] && st_p->st_mtime == dbmtime[j] && (db_lax || st_p->st_ctime == dbctime[j]))
+ ++*right_sum_cnt;
+ else
+ ++*wrong_sum_cnt;
+ }
+
+ return rows;
+}
+
+int db_set_checksum(int mdnum, const STRUCT_STAT *st_p, const char *sum)
+{
+ unsigned int disk_id;
+ const char *errmsg = NULL;
+ int rc = 0;
+
+ if (am_receiver || (am_generator && same_db)) {
+ /* Forward the setting to a single process. The receiver always
+ * forward to the generator, and the generator will forward to
+ * the receiver ONLY if this is a local transfer. */
+ char data[MSG_CHECKSUM_LEN];
+ SIVAL64(data, 0, st_p->st_dev);
+ SIVAL64(data, 8, st_p->st_ino);
+ SIVAL64(data, 16, st_p->st_size);
+ SIVAL64(data, 24, st_p->st_mtime);
+ SIVAL64(data, 32, st_p->st_ctime);
+#if MSG_CHECKSUM_LONGS != 5
+#error Fix the setting of checksum long values
+#endif
+ SIVAL(data, MSG_CHECKSUM_LONGS*8, mdnum);
+ memcpy(data + MSG_CHECKSUM_LONGS*8 + 4, sum, MAX_DIGEST_LEN);
+ return send_msg(MSG_CHECKSUM, data, sizeof data, 0);
+ }
+
+ if ((disk_id = get_disk_id(st_p->st_dev)) == 0)
+ return 0;
+
+ switch (use_db) {
+#ifdef USE_MYSQL
+ case DB_TYPE_MYSQL:
+ if (transaction_state == 0) {
+ if (!run_sql("BEGIN"))
+ return 0;
+ transaction_state = 1;
+ }
+
+ bind_disk_id = disk_id;
+ bind_ino = st_p->st_ino;
+ bind_mdnum = mdnum;
+ bind_size = st_p->st_size;
+ bind_mtime = st_p->st_mtime;
+ bind_ctime = st_p->st_ctime;
+ memcpy(bind_sum, sum, MD5_DIGEST_LEN);
+ if (exec_mysql(REP_SUM) == NULL)
+ errmsg = mysql_error(dbh.mysql);
+ break;
+#endif
+#ifdef USE_SQLITE
+ case DB_TYPE_SQLITE: {
+ sqlite3_stmt *stmt = statements[REP_SUM].sqlite;
+ int lock_failures = 0;
+
+ if (transaction_state == 0) {
+ if (!run_sql("BEGIN"))
+ return 0;
+ transaction_state = 1;
+ }
+
+ sqlite3_bind_int(stmt, 1, disk_id);
+ sqlite3_bind_int64(stmt, 2, st_p->st_ino);
+ sqlite3_bind_int(stmt, 3, mdnum);
+ sqlite3_bind_int64(stmt, 4, st_p->st_size);
+ sqlite3_bind_int64(stmt, 5, st_p->st_mtime);
+ sqlite3_bind_int64(stmt, 6, st_p->st_ctime);
+ sqlite3_bind_blob(stmt, 7, sum, MD5_DIGEST_LEN, SQLITE_TRANSIENT);
+ while (1) {
+ rc = sqlite3_step(stmt);
+ if (rc != SQLITE_BUSY && rc != SQLITE_LOCKED)
+ break;
+ if (++lock_failures > MAX_LOCK_FAILURES)
+ break;
+ sqlite3_reset(stmt);
+ msleep(LOCK_FAIL_MSLEEP);
+ }
+ if (rc != SQLITE_DONE)
+ errmsg = sqlite3_errmsg(dbh.sqlite);
+ sqlite3_reset(stmt);
+ break;
+ }
+#endif
+ }
+
+ if (!errmsg) {
+ if (DEBUG_GTE(DB, 2)) {
+ rprintf(FCLIENT, "[%s] Set DB checksum for %s,%s,%d: %s\n",
+ who_am_i(), big_num(st_p->st_dev), big_num(st_p->st_ino),
+ md_num, sum_as_hex(md_num, sum, 0));
+ }
+ } else {
+ rprintf(log_code, "[%s] Failed to set checksum for %s,%s,%d: %s (%d) -- closing DB\n",
+ who_am_i(), big_num(st_p->st_dev), big_num(st_p->st_ino),
+ md_num, errmsg, rc);
+ db_disconnect(False);
+ }
+
+ return errmsg ? 0 : 1;
+}
+
+/* For a delayed-update copy, we set the checksum on the file when it was
+ * inside the partial-dir. Since renaming the file changes its ctime, we need
+ * to update the ctime to its new value (we can skip this in db_lax mode). */
+int db_update_ctime(int mdnum, const STRUCT_STAT *st_p)
+{
+ unsigned int disk_id = get_disk_id(st_p->st_dev);
+
+ if (disk_id == 0)
+ return 0;
+
+ switch (use_db) {
+#ifdef USE_MYSQL
+ case DB_TYPE_MYSQL:
+ bind_ctime = st_p->st_ctime;
+ bind_disk_id = disk_id;
+ bind_ino = st_p->st_ino;
+ bind_mdnum = mdnum;
+ bind_size = st_p->st_size;
+ bind_mtime = st_p->st_mtime;
+ return exec_mysql(UPD_CTIME) != NULL;
+#endif
+#ifdef USE_SQLITE
+ case DB_TYPE_SQLITE: {
+ int rc;
+
+ sqlite3_stmt *stmt = statements[UPD_CTIME].sqlite;
+ if (stmt == NULL)
+ return 0;
+ sqlite3_bind_int64(stmt, 1, st_p->st_ctime);
+ sqlite3_bind_int(stmt, 2, disk_id);
+ sqlite3_bind_int64(stmt, 3, st_p->st_ino);
+ sqlite3_bind_int(stmt, 4, mdnum);
+ sqlite3_bind_int64(stmt, 5, st_p->st_size);
+ sqlite3_bind_int64(stmt, 6, st_p->st_mtime);
+ rc = sqlite3_step(stmt);
+ sqlite3_reset(stmt);
+ return rc == SQLITE_DONE;
+ }
+#endif
+ }
+
+ return 0;
+}
+
+int db_clean_init(void)
+{
+ switch (use_db) {
+#ifdef USE_MYSQL
+ case DB_TYPE_MYSQL: {
+ MYSQL_BIND binds[MAX_BIND_CNT];
+ char *sql;
+
+ mysql_query(dbh.mysql,
+ "CREATE TEMPORARY TABLE inode_present ("
+ " disk_id integer unsigned NOT NULL,"
+ " ino bigint unsigned NOT NULL,"
+ " present tinyint NOT NULL default '1',"
+ " PRIMARY KEY (disk_id,ino)"
+ ") ENGINE=MEMORY"
+ );
+
+ sql="INSERT IGNORE INTO inode_present"
+ " SET disk_id = ?, ino = ?, present = 1";
+ memset(binds, 0, sizeof binds);
+ binds[0].buffer_type = MYSQL_TYPE_LONG;
+ binds[0].buffer = &bind_disk_id;
+ binds[1].buffer_type = MYSQL_TYPE_LONGLONG;
+ binds[1].buffer = &bind_ino;
+ if (!prepare_mysql(INS_PRESENT, binds, 2, sql))
+ exit_cleanup(RERR_SYNTAX);
+
+ sql="DELETE m.*"
+ " FROM inode_map AS m"
+ " LEFT JOIN inode_present USING(disk_id, ino)"
+ " JOIN disk AS d ON(m.disk_id = d.disk_id)"
+ " WHERE host = ? AND devno != 0 AND present IS NULL";
+ memset(binds, 0, sizeof binds);
+ binds[0].buffer_type = MYSQL_TYPE_STRING;
+ binds[0].buffer = &bind_thishost;
+ binds[0].buffer_length = bind_thishost_len;
+ if (!prepare_mysql(DEL_SUMS, binds, 1, sql))
+ exit_cleanup(RERR_SYNTAX);
+
+ return 1;
+ }
+#endif
+#ifdef USE_SQLITE
+ case DB_TYPE_SQLITE: {
+ char *sql;
+ sql="ATTACH DATABASE '' AS aux1;"; /* Private temp DB, probably in-memory */
+ if (!run_sql(sql))
+ exit_cleanup(RERR_IPC);
+
+ sql="CREATE TABLE aux1.inode_present ("
+ " disk_id integer NOT NULL,"
+ " ino bigint NOT NULL,"
+ " present tinyint NOT NULL default '1',"
+ " PRIMARY KEY (disk_id,ino)"
+ ")";
+ if (!run_sql(sql))
+ exit_cleanup(RERR_IPC);
+
+ sql="INSERT OR IGNORE INTO aux1.inode_present"
+ " (disk_id, ino, present)"
+ " VALUES (?, ?, 1)";
+ if (!prepare_sqlite(INS_PRESENT, sql))
+ exit_cleanup(RERR_IPC);
+
+ sql="DELETE FROM inode_map"
+ " WHERE ROWID IN ("
+ " SELECT m.ROWID"
+ " FROM inode_map AS m"
+ " LEFT JOIN aux1.inode_present USING(disk_id, ino)"
+ " JOIN disk AS d ON(m.disk_id = d.disk_id)"
+ " WHERE host = ? AND devno != 0 AND present IS NULL"
+ " )";
+ if (!prepare_sqlite(DEL_SUMS, sql))
+ exit_cleanup(RERR_IPC);
+
+ transaction_state = -1; /* bug work-around -- force transaction off when cleaning XXX */
+
+ return 1;
+ }
+#endif
+ }
+
+ return 0;
+}
+
+int db_note_present(int disk_id, int64 ino)
+{
+ switch (use_db) {
+#ifdef USE_MYSQL
+ case DB_TYPE_MYSQL:
+ bind_disk_id = disk_id;
+ bind_ino = ino;
+ return exec_mysql(INS_PRESENT) != NULL;
+#endif
+#ifdef USE_SQLITE
+ case DB_TYPE_SQLITE: {
+ int rc;
+ sqlite3_stmt *stmt = statements[INS_PRESENT].sqlite;
+ sqlite3_bind_int(stmt, 1, disk_id);
+ sqlite3_bind_int64(stmt, 2, ino);
+ rc = sqlite3_step(stmt);
+ sqlite3_reset(stmt);
+ return rc == SQLITE_DONE;
+ }
+#endif
+ }
+
+ return 0;
+}
+
+/* This function requires the user to have populated all disk_id+inode pairs
+ * into the inode_present table. */
+int db_clean_inodes(void)
+{
+ int del_cnt = 0;
+
+ switch (use_db) {
+#ifdef USE_MYSQL
+ case DB_TYPE_MYSQL: {
+ MYSQL_STMT *stmt = exec_mysql(DEL_SUMS);
+ if (stmt != NULL)
+ del_cnt = mysql_affected_rows(dbh.mysql);
+ break;
+ }
+#endif
+#ifdef USE_SQLITE
+ case DB_TYPE_SQLITE: {
+ int rc;
+ sqlite3_stmt *stmt = statements[DEL_SUMS].sqlite;
+ sqlite3_bind_text(stmt, 1, bind_thishost, bind_thishost_len, SQLITE_STATIC);
+ rc = sqlite3_step(stmt);
+ if (rc == SQLITE_DONE)
+ del_cnt = sqlite3_changes(dbh.sqlite);
+ sqlite3_reset(stmt);
+ break;
+ }
+#endif
+ }
+
+ return del_cnt;
+}
+
+static int abs_path(char *buf, int bufsiz, const char *curdir, const char *dir)
+{
+ if (*dir == '/')
+ strlcpy(buf, dir, bufsiz);
+ else
+ snprintf(buf, bufsiz, "%s/%s", curdir, dir);
+
+ return clean_fname(buf, CFN_DROP_TRAILING_DOT_DIR | CFN_COLLAPSE_DOT_DOT_DIRS);
+}
+
+static struct name_list *new_name(const char *basename, const char *filename)
+{
+ struct name_list *n;
+ int blen = strlen(basename);
+ int slen = filename ? (int)strlen(filename) : -1;
+ int len = blen + 1 + slen;
+
+ if (len >= MAXPATHLEN) {
+ if (filename)
+ rprintf(FERROR, "Filename too long: %s/%s\n", basename, filename);
+ else
+ rprintf(FERROR, "Filename too long: %s\n", basename);
+ return NULL;
+ }
+
+ if (!(n = (struct name_list *)malloc(sizeof (struct name_list) + len)))
+ out_of_memory("new_name");
+
+ memcpy(n->name, basename, blen);
+ if (filename) {
+ n->name[blen] = '/';
+ memcpy(n->name + 1 + blen, filename, slen);
+ }
+ n->name[len] = '\0';
+ n->next = NULL;
+
+ return n;
+}
+
+static int name_compare(const void *n1, const void *n2)
+{
+ struct name_list *p1 = *(struct name_list **)n1;
+ struct name_list *p2 = *(struct name_list **)n2;
+ return strcmp(p1->name, p2->name);
+}
+
+static struct name_list *get_sorted_names(const char *dir)
+{
+ struct name_list *add, **sortbuf, *names = NULL, *prior_name = NULL;
+ struct dirent *di;
+ int cnt = 0;
+ DIR *d;
+
+ if (!(d = opendir("."))) {
+ rprintf(FERROR, "Unable to opendir %s: %s\n", dir, strerror(errno));
+ return NULL;
+ }
+ while ((di = readdir(d)) != NULL) {
+ char *dname = d_name(di);
+ if (dname[0] == '.' && (dname[1] == '\0' || (dname[1] == '.' && dname[2] == '\0')))
+ continue;
+ if (!(add = new_name(dname, NULL)))
+ continue;
+ if (prior_name)
+ prior_name->next = add;
+ else
+ names = add;
+ prior_name = add;
+ cnt++;
+ }
+ closedir(d);
+
+ if (cnt) {
+ int j;
+
+ if (!(sortbuf = new_array(struct name_list *, cnt)))
+ out_of_memory("get_sorted_names");
+ for (j = 0; j < cnt; j++) {
+ sortbuf[j] = names;
+ names = names->next;
+ }
+
+ qsort(sortbuf, cnt, PTR_SIZE, name_compare);
+
+ names = prior_name = NULL;
+ for (j = 0; j < cnt; j++) {
+ add = sortbuf[j];
+ if (prior_name)
+ prior_name->next = add;
+ else
+ names = add;
+ prior_name = add;
+ }
+
+ if (prior_name)
+ prior_name->next = NULL;
+ free(sortbuf);
+ }
+
+ return names;
+}
+
+static inline int sums_ne(const char *sum1, const char *sum2)
+{
+ return memcmp(sum1, sum2, MD5_DIGEST_LEN) != 0;
+}
+
+/* Returns 1 if there is a checksum change, else 0. */
+static int mention_file(const char *dir, const char *name, int right_cnt, int wrong_cnt,
+ const char *dbsum4, const char *dbsum5, const char *sum4, const char *sum5)
+{
+ char *info_str = wrong_cnt && !right_cnt ? "!i " : " ";
+ char *md4_str = !db_do_md4 ? NULL : !dbsum4 ? "+4 " : !sum4 ? "?4 " : sums_ne(sum4, dbsum4) ? "!4 " : " ";
+ char *md5_str = !db_do_md5 ? NULL : !dbsum5 ? "+5 " : !sum5 ? "?5 " : sums_ne(sum5, dbsum5) ? "!5 " : " ";
+ int chg = *info_str != ' ' || (md4_str && *md4_str != ' ') || (md5_str && *md5_str != ' ');
+ if (chg || db_output_unchanged) {
+ if (db_output_info) {
+ fputs(info_str, stdout);
+ if (md4_str)
+ fputs(md4_str, stdout);
+ if (md5_str)
+ fputs(md5_str, stdout);
+ }
+ if (db_output_sum) {
+ if (db_do_md4)
+ printf("%s ", sum_as_hex(4, sum4, 0));
+ if (db_do_md5)
+ printf("%s ", sum_as_hex(5, sum5, 0));
+ }
+ if (db_output_name) {
+ if (db_output_sum)
+ putchar(' '); /* We want 2 spaces, like md5sum. */
+ if (*dir != '.' || dir[1]) {
+ fputs(dir, stdout);
+ putchar('/');
+ }
+ puts(name);
+ }
+ }
+
+ return chg;
+}
+
+NORETURN void run_dbonly(const char **args)
+{
+ char start_dir[MAXPATHLEN], dirbuf[MAXPATHLEN];
+ int need_sum_cnt, start_dir_len;
+ struct name_list *prior_dir;
+ struct name_list *names;
+ int exit_code = 0;
+
+ checksum_type = 5;
+
+ need_sum_cnt = db_do_md4 + db_do_md5;
+
+ if (!db_read_config(FERROR, db_config) || !db_connect(1))
+ exit_cleanup(RERR_FILEIO);
+
+ if (db_clean)
+ db_clean_init();
+
+ if (getcwd(start_dir, sizeof start_dir - 1) == NULL) {
+ rsyserr(FERROR, errno, "getcwd()");
+ exit_cleanup(RERR_FILESELECT);
+ }
+ start_dir_len = strlen(start_dir);
+
+ if (args) {
+ prior_dir = NULL;
+ while (*args) {
+ struct name_list *add;
+ if (abs_path(dirbuf, sizeof dirbuf, start_dir, *args++) <= 0)
+ continue;
+ if (!(add = new_name(dirbuf, NULL)))
+ continue;
+ if (prior_dir)
+ prior_dir->next = add;
+ else
+ dirs_list = add;
+ prior_dir = add;
+ }
+ } else
+ dirs_list = new_name(start_dir, NULL);
+
+ prior_dir = NULL;
+ while (dirs_list) {
+ struct name_list *subdirs, *prior_subdir, *prior_name;
+ const char *dir = dirs_list->name;
+ const char *reldir = dir;
+
+ if (prior_dir)
+ free((void*)prior_dir);
+ prior_dir = dirs_list;
+ dirs_list = dirs_list->next;
+
+ if (strncmp(reldir, start_dir, start_dir_len) == 0) {
+ if (reldir[start_dir_len] == '\0')
+ reldir = ".";
+ else if (reldir[start_dir_len] == '/')
+ reldir += start_dir_len + 1;
+ }
+ if (db_output_dirs)
+ printf("... %s/ ...\n", reldir);
+
+ if (chdir(dir) < 0) {
+ rprintf(FERROR, "Unable to chdir to %s: %s\n", dir, strerror(errno));
+ continue;
+ }
+ if (!(names = get_sorted_names(dir)))
+ continue;
+
+ subdirs = prior_subdir = prior_name = NULL;
+ while (names) {
+ STRUCT_STAT st;
+ char *dbsum4, *sum4, sumbuf4[MD5_DIGEST_LEN];
+ char *dbsum5, *sum5, sumbuf5[MD5_DIGEST_LEN];
+ int right_sum_cnt, wrong_sum_cnt;
+ const char *name = names->name;
+ unsigned int disk_id;
+
+ if (prior_name)
+ free((void*)prior_name);
+ prior_name = names;
+ names = names->next;
+
+ dbsum4 = dbsum5 = sum4 = sum5 = NULL;
+
+ if (lstat(name, &st) < 0) {
+ rprintf(FERROR, "Failed to lstat(%s): %s\n", name, strerror(errno));
+ continue;
+ }
+ if (S_ISLNK(st.st_mode))
+ continue;
+ if (S_ISDIR(st.st_mode)) {
+ /* add optional excluding of things like /^(CVS|\.svn|\.git|\.bzr)$/; */
+ if (recurse) {
+ struct name_list *add = new_name(dir, name);
+ if (add) {
+ if (prior_subdir)
+ prior_subdir->next = add;
+ else
+ subdirs = add;
+ prior_subdir = add;
+ }
+ }
+ continue;
+ }
+ if (!S_ISREG(st.st_mode))
+ continue;
+
+ if (!(disk_id = get_disk_id(st.st_dev)))
+ continue;
+ if (db_clean) {
+ db_note_present(disk_id, st.st_ino);
+ if (!db_update && !db_check)
+ continue;
+ }
+ db_get_both_checksums(&st, &right_sum_cnt, &wrong_sum_cnt,
+ db_do_md4 ? &dbsum4 : NULL, db_do_md5 ? &dbsum5 : NULL);
+
+ if (!db_check && right_sum_cnt == need_sum_cnt) {
+ mention_file(reldir, name, right_sum_cnt, wrong_sum_cnt, dbsum4, dbsum5, dbsum4, dbsum5);
+ continue;
+ }
+
+ if (db_update || (db_check && right_sum_cnt) || db_output_sum) {
+ uchar *data;
+ int32 remainder;
+ md_context m4, m5;
+ struct map_struct *buf;
+ OFF_T off, len = st.st_size;
+ int fd = do_open(name, O_RDONLY, 0);
+
+ if (fd < 0) {
+ rprintf(FERROR, "ERROR: unable to read %s: %s\n", name, strerror(errno));
+ continue;
+ }
+
+ if (db_do_md4)
+ mdfour_begin(&m4);
+ if (db_do_md5)
+ md5_begin(&m5);
+
+ buf = map_file(fd, len, MAX_MAP_SIZE, CSUM_CHUNK);
+
+ for (off = 0; off + CSUM_CHUNK <= len; off += CSUM_CHUNK) {
+ data = (uchar*)map_ptr(buf, off, CSUM_CHUNK);
+ if (db_do_md4)
+ mdfour_update(&m4, data, CSUM_CHUNK);
+ if (db_do_md5)
+ md5_update(&m5, data, CSUM_CHUNK);
+ }
+
+ remainder = (int32)(len - off);
+ data = (uchar*)map_ptr(buf, off, remainder);
+ if (db_do_md4) {
+ mdfour_update(&m4, data, remainder);
+ mdfour_result(&m4, (uchar*)(sum4 = sumbuf4));
+ }
+ if (db_do_md5) {
+ md5_update(&m5, data, remainder);
+ md5_result(&m5, (uchar*)(sum5 = sumbuf5));
+ }
+
+ close(fd);
+ unmap_file(buf);
+ }
+
+ int chg = mention_file(reldir, name, right_sum_cnt, wrong_sum_cnt, dbsum4, dbsum5, sum4, sum5);
+ if (!chg) {
+ /* Only db_check should get here... */
+ } else if (!db_update) {
+ exit_code = 1;
+ } else {
+ int fail = 0;
+ if (db_do_md4 && !db_set_checksum(4, &st, sum4))
+ fail = 1;
+ if (db_do_md5 && !db_set_checksum(5, &st, sum5))
+ fail = 1;
+ if (fail) {
+ fprintf(stderr, "Failed to set checksum on %s/%s\n", reldir, name);
+ exit_cleanup(RERR_FILEIO);
+ }
+ }
+ }
+ if (prior_name)
+ free((void*)prior_name);
+
+ if (recurse && subdirs) {
+ prior_subdir->next = dirs_list;
+ dirs_list = subdirs;
+ }
+ }
+ if (prior_dir)
+ free((void*)prior_dir);
+
+ if (db_clean) {
+ int rows = db_clean_inodes();
+ if (db_output_msgs)
+ printf("Cleaned out %d old inode%s.\n", rows, rows == 1 ? "" : "s");
+ }
+
+ db_disconnect(True);
+ exit(exit_code);
+}
diff --git a/flist.c b/flist.c
--- a/flist.c
+++ b/flist.c
@@ -54,6 +54,7 @@ extern int preserve_devices;
extern int preserve_specials;
extern int delete_during;
extern int missing_args;
+extern int use_db;
extern int eol_nulls;
extern int relative_paths;
extern int implied_dirs;
@@ -1294,11 +1295,8 @@ struct file_struct *make_file(const char *fname, struct file_list *flist,
extra_len += EXTRA_LEN;
#endif
- if (always_checksum && am_sender && S_ISREG(st.st_mode)) {
- file_checksum(thisname, &st, tmp_sum);
- if (sender_keeps_checksum)
- extra_len += SUM_EXTRA_CNT * EXTRA_LEN;
- }
+ if (sender_keeps_checksum && S_ISREG(st.st_mode))
+ extra_len += SUM_EXTRA_CNT * EXTRA_LEN;
#if EXTRA_ROUNDING > 0
if (extra_len & (EXTRA_ROUNDING * EXTRA_LEN))
@@ -1383,8 +1381,12 @@ struct file_struct *make_file(const char *fname, struct file_list *flist,
return NULL;
}
- if (sender_keeps_checksum && S_ISREG(st.st_mode))
- memcpy(F_SUM(file), tmp_sum, flist_csum_len);
+ if (always_checksum && am_sender && S_ISREG(st.st_mode)) {
+ if (!use_db || !db_get_checksum(&st, tmp_sum))
+ file_checksum(thisname, &st, tmp_sum);
+ if (sender_keeps_checksum)
+ memcpy(F_SUM(file), tmp_sum, flist_csum_len);
+ }
if (unsort_ndx)
F_NDX(file) = stats.num_dirs;
@@ -2045,6 +2047,9 @@ void send_extra_file_list(int f, int at_least)
finish:
if (io_error != save_io_error && protocol_version == 30 && !ignore_errors)
send_msg_int(MSG_IO_ERROR, io_error);
+
+ if (use_db && flist_eof)
+ db_disconnect(True);
}
struct file_list *send_file_list(int f, int argc, char *argv[])
@@ -2068,6 +2073,13 @@ struct file_list *send_file_list(int f, int argc, char *argv[])
| (eol_nulls || reading_remotely ? RL_EOL_NULLS : 0);
int implied_dot_dir = 0;
+ if (use_db) {
+ if (always_checksum)
+ db_connect(0); /* Will reset use_db on error. */
+ else
+ use_db = 0;
+ }
+
rprintf(FLOG, "building file list\n");
if (show_filelist_progress)
start_filelist_progress("building file list");
@@ -2414,6 +2426,9 @@ struct file_list *send_file_list(int f, int argc, char *argv[])
rprintf(FINFO, "[%s] flist_eof=1\n", who_am_i());
}
+ if (use_db && (!inc_recurse || flist_eof))
+ db_disconnect(True);
+
return flist;
}
diff --git a/generator.c b/generator.c
--- a/generator.c
+++ b/generator.c
@@ -59,6 +59,7 @@ extern int ignore_existing;
extern int ignore_non_existing;
extern int want_xattr_optim;
extern int inplace;
+extern int use_db;
extern int append_mode;
extern int make_backups;
extern int csum_length;
@@ -583,7 +584,8 @@ int unchanged_file(char *fn, struct file_struct *file, STRUCT_STAT *st)
of the file time to determine whether to sync */
if (always_checksum > 0 && S_ISREG(st->st_mode)) {
char sum[MAX_DIGEST_LEN];
- file_checksum(fn, st, sum);
+ if (!use_db || !db_get_checksum(st, sum))
+ file_checksum(fn, st, sum);
return memcmp(sum, F_SUM(file), flist_csum_len) == 0;
}
@@ -2228,6 +2230,13 @@ void generate_files(int f_out, const char *local_name)
: "enabled");
}
+ if (use_db) {
+ if (always_checksum || (append_mode != 1 && protocol_version >= 30))
+ db_connect(0); /* Will reset use_db on error. */
+ else
+ use_db = 0;
+ }
+
dflt_perms = (ACCESSPERMS & ~orig_umask);
do {
@@ -2353,6 +2362,9 @@ void generate_files(int f_out, const char *local_name)
wait_for_receiver();
}
+ if (use_db)
+ db_disconnect(True);
+
info_levels[INFO_FLIST] = save_info_flist;
info_levels[INFO_PROGRESS] = save_info_progress;
diff --git a/io.c b/io.c
--- a/io.c
+++ b/io.c
@@ -41,8 +41,10 @@ extern int am_server;
extern int am_sender;
extern int am_receiver;
extern int am_generator;
+extern int local_server;
extern int msgs2stderr;
extern int inc_recurse;
+extern int same_db;
extern int io_error;
extern int eol_nulls;
extern int flist_eof;
@@ -1481,6 +1483,32 @@ static void read_a_msg(void)
if (am_sender)
maybe_send_keepalive(time(NULL), MSK_ALLOW_FLUSH);
break;
+ case MSG_CHECKSUM:
+ /* This receives some checksum info that we want to make a note of
+ * (which allows a single process to do all the writing to the db). */
+ if (msg_bytes != MSG_CHECKSUM_LEN)
+ goto overflow;
+ raw_read_buf(data, MSG_CHECKSUM_LEN);
+ if (am_generator && same_db) {
+ iobuf.in_multiplexed = 1;
+ send_msg(MSG_CHECKSUM, data, MSG_CHECKSUM_LEN, 0);
+ } if (am_receiver || (am_sender && !local_server))
+ goto unexpected;
+ else {
+ /* The received data is a set of numbers followed by the checksum. */
+ STRUCT_STAT st;
+ st.st_dev = IVAL64(data, 0);
+ st.st_ino = IVAL64(data, 8);
+ st.st_size = IVAL64(data, 16);
+ st.st_mtime = IVAL64(data, 24);
+ st.st_ctime = IVAL64(data, 32);
+#if MSG_CHECKSUM_LONGS != 5
+#error Fix the parsing of checksum long values
+#endif
+ iobuf.in_multiplexed = 1;
+ db_set_checksum(IVAL(data, MSG_CHECKSUM_LONGS*8), &st, data + MSG_CHECKSUM_LONGS*8 + 4);
+ }
+ break;
case MSG_DELETED:
if (msg_bytes >= sizeof data)
goto overflow;
@@ -1632,6 +1660,7 @@ static void read_a_msg(void)
* with a duplicate exit message. */
_exit_cleanup(val, __FILE__, 0 - __LINE__);
default:
+ unexpected:
rprintf(FERROR, "unexpected tag %d [%s%s]\n",
tag, who_am_i(), inc_recurse ? "/inc" : "");
exit_cleanup(RERR_STREAMIO);
diff --git a/loadparm.c b/loadparm.c
--- a/loadparm.c
+++ b/loadparm.c
@@ -112,6 +112,7 @@ typedef struct {
char *auth_users;
char *charset;
char *comment;
+ char *db_config;
char *dont_compress;
char *exclude;
char *exclude_from;
@@ -143,6 +144,7 @@ typedef struct {
int syslog_facility;
int timeout;
+ BOOL db_lax;
BOOL fake_super;
BOOL forward_lookup;
BOOL ignore_errors;
@@ -192,6 +194,7 @@ static const all_vars Defaults = {
/* auth_users; */ NULL,
/* charset; */ NULL,
/* comment; */ NULL,
+ /* db_config; */ NULL,
/* dont_compress; */ DEFAULT_DONT_COMPRESS,
/* exclude; */ NULL,
/* exclude_from; */ NULL,
@@ -221,6 +224,7 @@ static const all_vars Defaults = {
/* syslog_facility; */ LOG_DAEMON,
/* timeout; */ 0,
+ /* db_lax; */ False,
/* fake_super; */ False,
/* forward_lookup; */ True,
/* ignore_errors; */ False,
@@ -333,6 +337,8 @@ static struct parm_struct parm_table[] =
{"auth users", P_STRING, P_LOCAL, &Vars.l.auth_users, NULL,0},
{"charset", P_STRING, P_LOCAL, &Vars.l.charset, NULL,0},
{"comment", P_STRING, P_LOCAL, &Vars.l.comment, NULL,0},
+ {"db config", P_STRING, P_LOCAL, &Vars.l.db_config, NULL,0},
+ {"db lax", P_BOOL, P_LOCAL, &Vars.l.db_lax, NULL,0},
{"dont compress", P_STRING, P_LOCAL, &Vars.l.dont_compress, NULL,0},
{"exclude from", P_STRING, P_LOCAL, &Vars.l.exclude_from, NULL,0},
{"exclude", P_STRING, P_LOCAL, &Vars.l.exclude, NULL,0},
@@ -469,6 +475,7 @@ FN_GLOBAL_INTEGER(lp_rsync_port, &Vars.g.rsync_port)
FN_LOCAL_STRING(lp_auth_users, auth_users)
FN_LOCAL_STRING(lp_charset, charset)
FN_LOCAL_STRING(lp_comment, comment)
+FN_LOCAL_STRING(lp_db_config, db_config)
FN_LOCAL_STRING(lp_dont_compress, dont_compress)
FN_LOCAL_STRING(lp_exclude, exclude)
FN_LOCAL_STRING(lp_exclude_from, exclude_from)
@@ -498,6 +505,7 @@ FN_LOCAL_INTEGER(lp_max_verbosity, max_verbosity)
FN_LOCAL_INTEGER(lp_syslog_facility, syslog_facility)
FN_LOCAL_INTEGER(lp_timeout, timeout)
+FN_LOCAL_BOOL(lp_db_lax, db_lax)
FN_LOCAL_BOOL(lp_fake_super, fake_super)
FN_LOCAL_BOOL(lp_forward_lookup, forward_lookup)
FN_LOCAL_BOOL(lp_ignore_errors, ignore_errors)
diff --git a/main.c b/main.c
--- a/main.c
+++ b/main.c
@@ -34,6 +34,7 @@ extern int am_root;
extern int am_server;
extern int am_sender;
extern int am_daemon;
+extern int am_dbadmin;
extern int inc_recurse;
extern int blocking_io;
extern int always_checksum;
@@ -51,6 +52,7 @@ extern int copy_unsafe_links;
extern int keep_dirlinks;
extern int preserve_hard_links;
extern int protocol_version;
+extern int always_checksum;
extern int file_total;
extern int recurse;
extern int xfer_dirs;
@@ -85,6 +87,7 @@ extern char *filesfrom_host;
extern char *partial_dir;
extern char *dest_option;
extern char *rsync_path;
+extern char *db_config;
extern char *shell_cmd;
extern char *batch_name;
extern char *password_file;
@@ -1103,6 +1106,9 @@ void start_server(int f_in, int f_out, int argc, char *argv[])
if (am_daemon && io_timeout && protocol_version >= 31)
send_msg_int(MSG_IO_TIMEOUT, io_timeout);
+ if (db_config)
+ db_read_config(FERROR, db_config);
+
if (am_sender) {
keep_dirlinks = 0; /* Must be disabled on the sender. */
if (need_messages_from_generator)
@@ -1384,6 +1390,9 @@ static int start_client(int argc, char *argv[])
}
}
+ if (db_config)
+ db_read_config(FERROR, db_config);
+
if (daemon_over_rsh < 0)
return start_socket_client(shell_machine, remote_argc, remote_argv, argc, argv);
diff --git a/mkproto.pl b/mkproto.pl
--- a/mkproto.pl
+++ b/mkproto.pl
@@ -13,6 +13,8 @@ if (open(IN, 'proto.h')) {
STRING => 'char *',
);
+@ARGV = grep !m{/rsyncdb\.c$}, @ARGV;
+
$inheader = 0;
$protos = qq|/* This file is automatically generated with "make proto". DO NOT EDIT */\n\n|;
diff --git a/options.c b/options.c
--- a/options.c
+++ b/options.c
@@ -82,6 +82,7 @@ int am_root = 0; /* 0 = normal, 1 = root, 2 = --super, -1 = --fake-super */
int am_server = 0;
int am_sender = 0;
int am_starting_up = 1;
+int am_dbadmin = 0;
int relative_paths = -1;
int implied_dirs = 1;
int missing_args = 0; /* 0 = FERROR_XFER, 1 = ignore, 2 = delete */
@@ -95,6 +96,7 @@ int use_qsort = 0;
char *files_from = NULL;
int filesfrom_fd = -1;
char *filesfrom_host = NULL;
+char *db_config = NULL;
int eol_nulls = 0;
int protect_args = -1;
int human_readable = 1;
@@ -102,6 +104,9 @@ int recurse = 0;
int allow_inc_recurse = 1;
int xfer_dirs = -1;
int am_daemon = 0;
+int db_clean, db_check, db_do_md4, db_do_md5, db_update = 1, db_lax, db_init, db_mounts;
+int db_output_name, db_output_sum, db_output_info, db_output_unchanged, db_output_dirs, db_output_msgs;
+int saw_db_output_opt, saw_db_sum_opt;
int connect_timeout = 0;
int keep_partial = 0;
int safe_symlinks = 0;
@@ -271,6 +276,7 @@ static struct output_struct debug_words[COUNT_DEBUG+1] = {
DEBUG_WORD(CHDIR, W_CLI|W_SRV, "Debug when the current directory changes"),
DEBUG_WORD(CONNECT, W_CLI, "Debug connection events (levels 1-2)"),
DEBUG_WORD(CMD, W_CLI, "Debug commands+options that are issued (levels 1-2)"),
+ DEBUG_WORD(DB, W_SND|W_REC, "Debug DB operations (levels 1-5)"),
DEBUG_WORD(DEL, W_REC, "Debug delete actions (levels 1-3)"),
DEBUG_WORD(DELTASUM, W_SND|W_REC, "Debug delta-transfer checksumming (levels 1-4)"),
DEBUG_WORD(DUP, W_REC, "Debug weeding of duplicate names"),
@@ -573,6 +579,7 @@ static void print_rsync_version(enum logcode f)
char const *links = "no ";
char const *iconv = "no ";
char const *ipv6 = "no ";
+ char const *db = "no ";
STRUCT_STAT *dumstat;
#if SUBPROTOCOL_VERSION != 0
@@ -609,6 +616,11 @@ static void print_rsync_version(enum logcode f)
#ifdef CAN_SET_SYMLINK_TIMES
symtimes = "";
#endif
+#if defined HAVE_MYSQL_MYSQL_H && defined HAVE_LIBMYSQLCLIENT
+ db = "";
+#elif defined HAVE_SQLITE3_H && defined HAVE_LIBSQLITE3
+ db = "";
+#endif
rprintf(f, "%s version %s protocol version %d%s\n",
RSYNC_NAME, RSYNC_VERSION, PROTOCOL_VERSION, subprotocol);
@@ -622,8 +634,8 @@ static void print_rsync_version(enum logcode f)
(int)(sizeof (int64) * 8));
rprintf(f, " %ssocketpairs, %shardlinks, %ssymlinks, %sIPv6, batchfiles, %sinplace,\n",
got_socketpair, hardlinks, links, ipv6, have_inplace);
- rprintf(f, " %sappend, %sACLs, %sxattrs, %siconv, %ssymtimes, %sprealloc\n",
- have_inplace, acls, xattrs, iconv, symtimes, prealloc);
+ rprintf(f, " %sappend, %sACLs, %sxattrs, %siconv, %ssymtimes, %sprealloc, %sdb\n",
+ have_inplace, acls, xattrs, iconv, symtimes, prealloc, db);
#ifdef MAINTAINER_MODE
rprintf(f, "Panic Action: \"%s\"\n", get_panic_action());
@@ -672,6 +684,9 @@ void usage(enum logcode F)
rprintf(F," -q, --quiet suppress non-error messages\n");
rprintf(F," --no-motd suppress daemon-mode MOTD (see manpage caveat)\n");
rprintf(F," -c, --checksum skip based on checksum, not mod-time & size\n");
+ rprintf(F," --db=CONFIG_FILE specify a CONFIG_FILE for DB checksums\n");
+ rprintf(F," --db-only=CONFIG_FILE behave like rsyncdb\n");
+ rprintf(F," --db-lax ignore ctime changes (use with CAUTION)\n");
rprintf(F," -a, --archive archive mode; equals -rlptgoD (no -H,-A,-X)\n");
rprintf(F," --no-OPTION turn off an implied OPTION (e.g. --no-D)\n");
rprintf(F," -r, --recursive recurse into directories\n");
@@ -820,6 +835,7 @@ enum {OPT_VERSION = 1000, OPT_DAEMON, OPT_SENDER, OPT_EXCLUDE, OPT_EXCLUDE_FROM,
OPT_INCLUDE, OPT_INCLUDE_FROM, OPT_MODIFY_WINDOW, OPT_MIN_SIZE, OPT_CHMOD,
OPT_READ_BATCH, OPT_WRITE_BATCH, OPT_ONLY_WRITE_BATCH, OPT_MAX_SIZE,
OPT_NO_D, OPT_APPEND, OPT_NO_ICONV, OPT_INFO, OPT_DEBUG,
+ OPT_NO_DB, OPT_DBONLY,
OPT_USERMAP, OPT_GROUPMAP, OPT_CHOWN, OPT_BWLIMIT,
OPT_SERVER, OPT_REFUSED_BASE = 9000};
@@ -960,6 +976,10 @@ static struct poptOption long_options[] = {
{"checksum", 'c', POPT_ARG_VAL, &always_checksum, 1, 0, 0 },
{"no-checksum", 0, POPT_ARG_VAL, &always_checksum, 0, 0, 0 },
{"no-c", 0, POPT_ARG_VAL, &always_checksum, 0, 0, 0 },
+ {"db", 0, POPT_ARG_STRING, &db_config, 0, 0, 0 },
+ {"no-db", 0, POPT_ARG_NONE, 0, OPT_NO_DB, 0, 0 },
+ {"db-lax", 0, POPT_ARG_VAL, &db_lax, 1, 0, 0 },
+ {"no-db-lax", 0, POPT_ARG_VAL, &db_lax, 0, 0, 0 },
{"block-size", 'B', POPT_ARG_LONG, &block_size, 0, 0, 0 },
{"compare-dest", 0, POPT_ARG_STRING, 0, OPT_COMPARE_DEST, 0, 0 },
{"copy-dest", 0, POPT_ARG_STRING, 0, OPT_COPY_DEST, 0, 0 },
@@ -1050,6 +1070,9 @@ static struct poptOption long_options[] = {
{"dparam", 0, POPT_ARG_STRING, 0, OPT_DAEMON, 0, 0 },
{"detach", 0, POPT_ARG_NONE, 0, OPT_DAEMON, 0, 0 },
{"no-detach", 0, POPT_ARG_NONE, 0, OPT_DAEMON, 0, 0 },
+ /* All the following options switch us into DB-admin option-parsing. */
+ {"db-help", 0, POPT_ARG_NONE, 0, OPT_DBONLY, 0, 0 },
+ {"db-only", 0, POPT_ARG_STRING, 0, OPT_DBONLY, 0, 0 },
{0,0,0,0, 0, 0, 0}
};
@@ -1103,6 +1126,50 @@ static struct poptOption long_daemon_options[] = {
{0,0,0,0, 0, 0, 0}
};
+static void dbonly_usage(enum logcode F)
+{
+ rprintf(F,"Usage: rsyncdb --db=CONFIG_FILE [OPTIONS] [DIRS]\n");
+ rprintf(F,"\n");
+ rprintf(F,"Options:\n");
+ rprintf(F," --db=CONFIG Specify the CONFIG file to read for the DB info.\n");
+ rprintf(F," --db-lax Ignore ctime changes (use with CAUTION).\n");
+ rprintf(F,"-r, --recursive Scan files in subdirs (the default w/o --no-recursive).\n");
+ rprintf(F,"-s, --sums=SUMS List which checksums to update (default: 4,5).\n");
+ rprintf(F,"-o, --output=STR One or more letters of what to output (default is nothing).\n");
+ rprintf(F,"-c, --check Check the checksums (by reading the files) and fix issues.\n");
+ rprintf(F," --clean Note all inodes in the DIRS and remove DB extras.\n");
+ rprintf(F,"-N, --no-update Avoids updating/adding info with --check and/or --clean.\n");
+ rprintf(F," --init Initialize a DB by (re-)creating its tables.\n");
+ rprintf(F," --mounts Scan for mounted filesystems and update the DB.\n");
+ rprintf(F,"-q, --quiet Disable the default non-error output.\n");
+ rprintf(F,"-h, --help Display this help message.\n");
+}
+
+static struct poptOption long_dbonly_options[] = {
+ /* longName, shortName, argInfo, argPtr, value, descrip, argDesc */
+ {"check", 'c', POPT_ARG_NONE, &db_check, 0, 0, 0},
+ {"clean", 0, POPT_ARG_NONE, &db_clean, 0, 0, 0},
+ {"db", 0, POPT_ARG_STRING, &db_config, 0, 0, 0 },
+ {"db-only", 0, POPT_ARG_STRING, &db_config, 0, 0, 0 },
+ {"db-lax", 0, POPT_ARG_VAL, &db_lax, 1, 0, 0 },
+ {"no-db-lax", 0, POPT_ARG_VAL, &db_lax, 0, 0, 0 },
+ {"info", 0, POPT_ARG_STRING, 0, OPT_INFO, 0, 0 },
+ {"debug", 0, POPT_ARG_STRING, 0, OPT_DEBUG, 0, 0 },
+ {"update", 'u', POPT_ARG_VAL, &db_update, 1, 0, 0 },
+ {"no-update", 'N', POPT_ARG_VAL, &db_update, 0, 0, 0 },
+ {"no-u", 0, POPT_ARG_VAL, &db_update, 0, 0, 0 },
+ {"output", 'o', POPT_ARG_STRING, 0, 'o', 0, 0 },
+ {"recursive", 'r', POPT_ARG_VAL, &recurse, 1, 0, 0 },
+ {"no-recursive", 0, POPT_ARG_VAL, &recurse, 0, 0, 0 },
+ {"no-r", 0, POPT_ARG_VAL, &recurse, 0, 0, 0 },
+ {"sums", 's', POPT_ARG_STRING, 0, 's', 0, 0 },
+ {"init", 0, POPT_ARG_NONE, &db_init, 0, 0, 0 },
+ {"mounts", 0, POPT_ARG_NONE, &db_mounts, 0, 0, 0 },
+ {"quiet", 'q', POPT_ARG_NONE, &quiet, 0, 0, 0 },
+ {"help", 'h', POPT_ARG_NONE, 0, 'h', 0, 0 },
+ {"db-help", 0, POPT_ARG_NONE, 0, 'h', 0, 0 },
+ {0,0,0,0, 0, 0, 0}
+};
static char err_buf[200];
@@ -1281,6 +1348,101 @@ static void create_refuse_error(int which)
}
}
+static NORETURN void parse_dbonly_args(int argc, const char **argv)
+{
+ poptContext pc = poptGetContext(RSYNC_NAME, argc, argv, long_dbonly_options, 0);
+ const char *arg;
+ int opt;
+
+ recurse = 1;
+ am_dbadmin = 1;
+
+ while ((opt = poptGetNextOpt(pc)) != -1) {
+ const char *cp;
+ switch (opt) {
+ case 'o':
+ for (cp = poptGetOptArg(pc); *cp; cp++) {
+ switch (toLower(cp)) {
+ case 'n':
+ db_output_name = 1;
+ break;
+ case 's':
+ case 'c':
+ db_output_sum = db_output_name = 1;
+ break;
+ case 'i':
+ db_output_info = db_output_name = 1;
+ break;
+ case 'u':
+ db_output_unchanged = db_output_name = 1;
+ break;
+ case 'd':
+ db_output_dirs = 1;
+ break;
+ }
+ }
+ saw_db_output_opt = 1;
+ break;
+
+ case 's':
+ for (cp = poptGetOptArg(pc); *cp; cp++) {
+ switch (*cp) {
+ case '4':
+ db_do_md4 = 1;
+ break;
+ case '5':
+ db_do_md5 = 1;
+ break;
+ }
+ }
+ saw_db_sum_opt = 1;
+ break;
+
+ case 'h':
+ dbonly_usage(FINFO);
+ exit_cleanup(0);
+
+ case OPT_INFO:
+ arg = poptGetOptArg(pc);
+ parse_output_words(info_words, info_levels, arg, USER_PRIORITY);
+ break;
+
+ case OPT_DEBUG:
+ arg = poptGetOptArg(pc);
+ parse_output_words(debug_words, debug_levels, arg, USER_PRIORITY);
+ break;
+
+ default:
+ rprintf(FERROR,
+ "rsyncdb: %s: %s\n",
+ poptBadOption(pc, POPT_BADOPTION_NOALIAS),
+ poptStrerror(opt));
+ goto dbonly_usage;
+ }
+ }
+
+ if (!db_config) {
+ rprintf(FERROR, "You must specify the --db=FILE option.\n");
+ dbonly_usage:
+ rprintf(FERROR,
+ "(Type \"rsyncdb --help\" for assistance.)\n");
+ exit_cleanup(RERR_SYNTAX);
+ }
+
+ if (db_check)
+ db_output_info = 1;
+ if (!saw_db_output_opt && !quiet)
+ db_output_dirs = db_output_name = 1;
+ if (!quiet)
+ db_output_msgs = 1;
+ if (!saw_db_sum_opt)
+ db_do_md5 = 1;
+
+ am_starting_up = 0;
+ run_dbonly(poptGetArgs(pc));
+ exit(42); /* NOT REACHED */
+}
+
/* This is used to make sure that --daemon & --server cannot be aliased to
* something else. These options have always disabled popt aliases for the
* parsing of a daemon or server command-line, but we have to make sure that
@@ -1315,10 +1477,18 @@ int parse_arguments(int *argc_p, const char ***argv_p)
int opt;
int orig_protect_args = protect_args;
+ arg = *argv + strlen(*argv);
+ if (arg - *argv > 2 && strcmp(arg-2, "db") == 0) {
+ parse_dbonly_args(argc, argv);
+ /* NOT REACHED */
+ }
+
if (ref && *ref)
set_refuse_options(ref);
if (am_daemon) {
set_refuse_options("log-file*");
+ set_refuse_options("db");
+ set_refuse_options("db-lax");
#ifdef ICONV_OPTION
if (!*lp_charset(module_id))
set_refuse_options("iconv");
@@ -1444,6 +1614,12 @@ int parse_arguments(int *argc_p, const char ***argv_p)
am_daemon = 1;
return 1;
+ case OPT_DBONLY:
+ protect_args = 0;
+ poptFreeContext(pc);
+ parse_dbonly_args(argc, argv);
+ break; /* NOT REACHED */
+
case OPT_MODIFY_WINDOW:
/* The value has already been set by popt, but
* we need to remember that we're using a
@@ -1518,6 +1694,10 @@ int parse_arguments(int *argc_p, const char ***argv_p)
preserve_devices = preserve_specials = 0;
break;
+ case OPT_NO_DB:
+ db_config = NULL;
+ break;
+
case 'h':
human_readable++;
break;
diff --git a/pipe.c b/pipe.c
--- a/pipe.c
+++ b/pipe.c
@@ -27,11 +27,16 @@ extern int am_server;
extern int blocking_io;
extern int filesfrom_fd;
extern int munge_symlinks;
+extern int always_checksum;
+extern int use_db;
+extern char *db_config;
extern char *logfile_name;
extern int remote_option_cnt;
extern const char **remote_options;
extern struct chmod_mode_struct *chmod_modes;
+int same_db = 0;
+
/**
* Create a child connected to us via its stdin/stdout.
*
@@ -142,13 +147,22 @@ pid_t local_child(int argc, char **argv, int *f_in, int *f_out,
}
if (remote_option_cnt) {
+ const char *db_config_save = db_config;
int rc = remote_option_cnt + 1;
const char **rv = remote_options;
if (!parse_arguments(&rc, &rv)) {
option_error();
exit_cleanup(RERR_SYNTAX);
}
- }
+ if (db_config == db_config_save)
+ same_db = db_config != NULL;
+ else if (!db_config || !db_config_save || strcmp(db_config, db_config_save) != 0) {
+ use_db = 0;
+ if (db_config)
+ db_read_config(FERROR, db_config);
+ }
+ } else if (use_db)
+ same_db = 1;
if (dup2(to_child_pipe[0], STDIN_FILENO) < 0 ||
close(to_child_pipe[1]) < 0 ||
diff --git a/receiver.c b/receiver.c
--- a/receiver.c
+++ b/receiver.c
@@ -24,6 +24,8 @@
extern int dry_run;
extern int do_xfers;
+extern int use_db;
+extern int db_lax;
extern int am_root;
extern int am_server;
extern int inc_recurse;
@@ -429,6 +431,11 @@ static void handle_delayed_updates(char *local_name)
"rename failed for %s (from %s)",
full_fname(fname), partialptr);
} else {
+ if (use_db && !db_lax) {
+ STRUCT_STAT st;
+ if (do_lstat(fname, &st) == 0)
+ db_update_ctime(5, &st);
+ }
if (remove_source_files
|| (preserve_hard_links && F_IS_HLINKED(file)))
send_msg_int(MSG_SUCCESS, ndx);
@@ -535,6 +542,9 @@ int recv_files(int f_in, int f_out, char *local_name)
if (delay_updates)
delayed_bits = bitbag_create(cur_flist->used + 1);
+ if (use_db && (append_mode == 1 || protocol_version < 30))
+ use_db = 0; /* We can't note finished md5 values */
+
while (1) {
cleanup_disable();
@@ -863,6 +873,8 @@ int recv_files(int f_in, int f_out, char *local_name)
do_unlink(partialptr);
handle_partial_dir(partialptr, PDIR_DELETE);
}
+ if (use_db && do_lstat(fname, &st) == 0)
+ db_set_checksum(5, &st, sender_file_sum);
} else if (keep_partial && partialptr) {
if (!handle_partial_dir(partialptr, PDIR_CREATE)) {
rprintf(FERROR,
@@ -876,6 +888,8 @@ int recv_files(int f_in, int f_out, char *local_name)
recv_ok = -1;
else if (delay_updates && recv_ok) {
bitbag_set_bit(delayed_bits, ndx);
+ if (use_db && do_lstat(partialptr, &st) == 0)
+ db_set_checksum(5, &st, sender_file_sum);
recv_ok = 2;
} else
partialptr = NULL;
diff --git a/rsync.c b/rsync.c
--- a/rsync.c
+++ b/rsync.c
@@ -39,6 +39,7 @@ extern int am_daemon;
extern int am_sender;
extern int am_receiver;
extern int am_generator;
+extern int am_dbadmin;
extern int am_starting_up;
extern int allow_8bit_chars;
extern int protocol_version;
@@ -748,6 +749,8 @@ struct file_list *flist_for_ndx(int ndx, const char *fatal_error_loc)
const char *who_am_i(void)
{
+ if (am_dbadmin)
+ return "rsyncdb";
if (am_starting_up)
return am_server ? "server" : "client";
return am_sender ? "sender"
diff --git a/rsync.h b/rsync.h
--- a/rsync.h
+++ b/rsync.h
@@ -242,12 +242,16 @@ enum msgcode {
MSG_IO_ERROR=22,/* the sending side had an I/O error */
MSG_IO_TIMEOUT=33,/* tell client about a daemon's timeout value */
MSG_NOOP=42, /* a do-nothing message (legacy protocol-30 only) */
+ MSG_CHECKSUM=55,/* sent via rcvr -> gen pipe and local-host-only gen -> sender */
MSG_ERROR_EXIT=86, /* synchronize an error exit (siblings and protocol >= 31) */
MSG_SUCCESS=100,/* successfully updated indicated flist index */
MSG_DELETED=101,/* successfully deleted a file on receiving side */
MSG_NO_SEND=102,/* sender failed to open a file we wanted */
};
+#define MSG_CHECKSUM_LONGS 5
+#define MSG_CHECKSUM_LEN (MSG_CHECKSUM_LONGS*8 + 4 + MAX_DIGEST_LEN)
+
#define NDX_DONE -1
#define NDX_FLIST_EOF -2
#define NDX_DEL_STATS -3
@@ -1268,7 +1272,8 @@ extern short info_levels[], debug_levels[];
#define DEBUG_CHDIR (DEBUG_BIND+1)
#define DEBUG_CONNECT (DEBUG_CHDIR+1)
#define DEBUG_CMD (DEBUG_CONNECT+1)
-#define DEBUG_DEL (DEBUG_CMD+1)
+#define DEBUG_DB (DEBUG_CMD+1)
+#define DEBUG_DEL (DEBUG_DB+1)
#define DEBUG_DELTASUM (DEBUG_DEL+1)
#define DEBUG_DUP (DEBUG_DELTASUM+1)
#define DEBUG_EXIT (DEBUG_DUP+1)
diff --git a/rsync.yo b/rsync.yo
--- a/rsync.yo
+++ b/rsync.yo
@@ -340,6 +340,9 @@ to the detailed description below for a complete description. verb(
-q, --quiet suppress non-error messages
--no-motd suppress daemon-mode MOTD (see caveat)
-c, --checksum skip based on checksum, not mod-time & size
+ --db=CONFIG_FILE specify a CONFIG_FILE for DB checksums
+ --db-only=CONFIG_FILE Behave like rsyncdb (see that manpage).
+ --db-lax Ignore ctime changes (use with CAUTION).
-a, --archive archive mode; equals -rlptgoD (no -H,-A,-X)
--no-OPTION turn off an implied OPTION (e.g. --no-D)
-r, --recursive recurse into directories
@@ -650,6 +653,67 @@ option's before-the-transfer "Does this file need to be updated?" check.
For protocol 30 and beyond (first supported in 3.0.0), the checksum used is
MD5. For older protocols, the checksum used is MD4.
+dit(bf(--db=CONFIG_FILE)) This option specifies a CONFIG_FILE to read
+that holds connection details for a database of checksum information.
+When combined with the bf(--checksum) (bf(-c)) option, rsync will try to
+use cached checksum information from the DB, and will update it if it is
+missing.
+
+The currently supported DB choices are MySQL and SQLite. For example, a
+MySQL configuration might look like this:
+
+verb( dbtype: mysql
+ dbhost: 127.0.0.1
+ dbname: rsyncdb
+ dbuser: rsyncuser
+ dbpass: somepass
+ port: 3306
+ thishost: hostname )
+
+And a SQLite configuration might look like this:
+
+verb( dbtype: SQLite
+ dbname: /var/cache/rsync/sum.db
+ transaction: 1)
+
+Both the bf(--db) and bf(--db-lax) options only affect the side where the
+option is used. To affect the remote side of a remote-shell connection,
+use the bf(--remote-option) (bf(-M)) option. For example, to specify the
+same options on both sides, you could specify something like this:
+
+verb( rsync -avc {-M,}--db=/etc/rsyncdb.conf src/ host:dest/ )
+
+For a local copy, this option affects both the source and the destination.
+If you wish a local copy to enable this option just for the destination
+files, specify bf(-M--db=CONFIG) (the same for bf(-M--db-lax)). If you wish
+a local copy to enable this option just for the source files, combine
+bf(--db=CONFIG) with bf(-M--no-db) (similarly use bf(-M--no-db-lax)).
+
+See the perl script "rsyncdb" in the support directory of the source code
+(which may also be installed in /usr/bin) for a way to create the tables,
+populate the mounted-disk information, check files against their checksums,
+and update both the MD4 and MD5 checksums for files at the same time (since
+an rsync copy will only update one or the other).
+
+You can use a single MySQL DB for all your hosts if you give each one
+their own "thishost" name and setup their device-mapping data. Or feel
+free to use separate databases, separate servers, etc. See the rsync
+daemon's "db config" parameter for how to configure a daemon to use a DB
+(since a client cannot control this parameter on a daemon).
+
+dit(bf(--db-lax)) This option can be used to modify the inode-matching
+algorithm used by bf(--db) to one that ignores the ctime. This can be very
+DANGEROUS unless your files are known to ALWAYS be updated in a safe manner.
+If unsure, don't use it.
+
+The reason you might want to use it is that the ctime (inode change time) is
+changed by an added hard-link, or the file being moving around. To use this
+option safely you must be CERTAIN that either rsync w/--db is the only program
+adding files into the cached hierarchies, OR that all new files will have new
+modify times (never a historical mtime that might match an orphaned inode).
+So, for certain applications, such as mirrors of new tar releases, this option
+can save a lot of unneeded checksum re-computation due to ctime changes.
+
dit(bf(-a, --archive)) This is equivalent to bf(-rlptgoD). It is a quick
way of saying you want recursion and want to preserve almost
everything (with -H being a notable omission).
diff --git a/rsyncd.conf.yo b/rsyncd.conf.yo
--- a/rsyncd.conf.yo
+++ b/rsyncd.conf.yo
@@ -326,6 +326,22 @@ is daemon. This setting has no effect if the "log file" setting is a
non-empty string (either set in the per-modules settings, or inherited
from the global settings).
+dit(bf(db config)) This parameter specifies a config file to read that
+holds connection details for a database of checksum information.
+
+The config file will be read-in prior to any chroot restrictions, but
+the connection occurs from inside the chroot. This means that you
+should use a socket connection (e.g. 127.0.0.1 rather than localhost)
+for a MySQL config from inside a chroot. For SQLite, the DB file must
+be placed inside the chroot (though it can be placed outside the
+transfer dir if you configured an inside-chroot path).
+
+See the bf(--db=CONFIG_FILE) option for full details.
+
+dit(bf(db lax)) This parameter specifies that a "db config" setup should use
+lax (no ctime) lookups. See the rsync manpage's section -n bf(--db-lax) for
+some warnings about using this setting.
+
dit(bf(syslog tag)) This parameter allows you to specify the syslog
tag to use when logging messages from the rsync daemon. The default is
"rsyncd". This setting has no effect if the "log file" setting is a
diff --git a/rsyncdb-mountinfo b/rsyncdb-mountinfo
new file mode 100755
--- /dev/null
+++ b/rsyncdb-mountinfo
@@ -0,0 +1,60 @@
+#!/usr/bin/perl
+
+# This script outputs data for rsyncdb --mounts. It must output a complete
+# list of the mounts for the current host in a strict format -- 2 fields
+# with a Tab between: $MOUNT_UNIQ\t$PATH
+#
+# The list of mounts MUST NOT contain any entry that has the same devnum
+# (st_dev) as any other entry in the list (as checked via its PATH).
+#
+# MOUNT_UNIQ is a unique string that identifies the mount on this host.
+# This cannot be the devnum (st_dev) because that can vary depending on the
+# mount order or be reused for different mounts if they are not mounted at
+# the same time. By default the value is "Mount of $devname", which should
+# be adequate for situations that don't want removable media in the DB
+# (though you may need to take steps to weed-out removable media from the
+# list to ensure that such inodes stay out of the DB).
+#
+# You can override the MOUNT_UNIQ value by putting a .rsyndb_mount_uniq
+# file in the root directory of any mount, at which point it is up to you
+# to make sure that the value stays unique (note that all sequences of
+# whitespace are transformed into a single space, and leading/trailing
+# whitespace is removed).
+#
+# MOUNT_UNIQ may never contain a Tab but it would be legal for PATH to have
+# a Tab (just really weird). Neither may have a CR or LF in it.
+#
+# The maximum size for MOUNT_UNIQ is 256 characters.
+#
+# If this script doesn't meet your needs, feel free to edit it and choose
+# some other method of finding a unique value for each mount. If you come
+# up with a good idiom that might be useful to others, please share it back
+# to me.
+
+use strict;
+use warnings;
+
+my $MOUNT_FILE = '/etc/mtab';
+my $VALID_DEVICE_REGEX = qr{^/dev};
+
+my %hash;
+
+open MOUNTS, $MOUNT_FILE or die "Unable to open $MOUNT_FILE: $!\n";
+while (<MOUNTS>) {
+ my ($devname, $path) = (split)[0,1];
+ next unless $devname =~ /$VALID_DEVICE_REGEX/;
+
+ my ($devno) = (stat($path))[0];
+ next unless defined $devno; # Skip if mount is invalid.
+ next if $hash{$devno}++; # SKip if we've seen this devno earlier.
+
+ my $mount_uniq = "Mount of $devname";
+ if (open UNIQ, '<', "$path/.rsyndb_mount_uniq") {
+ $mount_uniq = <UNIQ>;
+ close UNIQ;
+ $mount_uniq =~ s/\s+/ /g; # This ensures no tab, CR, nor LF.
+ $mount_uniq =~ s/^ | $//g; # .. and no leading or trailing whitespace.
+ }
+ print $mount_uniq, "\t", $path, "\n";
+}
+close MOUNTS;
diff --git a/rsyncdb.yo b/rsyncdb.yo
new file mode 100644
--- /dev/null
+++ b/rsyncdb.yo
@@ -0,0 +1,186 @@
+mailto(rsync-bugs@samba.org)
+manpage(rsync)(1)(23 Jun 2013)()()
+manpagename(rsyncdb)(Maintain an rsync checksum DB)
+manpagesynopsis()
+
+verb(rsyncdb --db=CONFIG [OPTION...] [DIR...])
+
+manpagedescription()
+
+Rsyncdb can maintain a checksum-caching DB that rsync can use to make its
+bf(--checksum) option more optimal. You must specify a config file via
+the bf(--db=CONFIG_FILE) option in order for rsyncdb to know what DB to
+manipulate. See the rsync manpage's bf(--db) option for full details on
+the file's format.
+
+You can specify one or more directory args for rsyncdb to scan. If no
+DIR args are specified, the current directory is assumed to be the spot
+to start scanning.
+
+Note that the rsyncdb program is usually just a symlink to the rsync program.
+You can force rsync to behave as rsyncdb either by having a symlink (or
+hardlink) name that ends with "db" or by bf(starting) the rsync args with
+bf(--db-only=CONFIG) (and that option works just like bf(--db=CONFIG) to
+a program named rsyncdb).
+
+manpagesection(EXAMPLES)
+
+The following command will update checksum information in the database
+described in the /etc/db.conf file:
+
+verb( rsyncdb --db=/etc/db.conf -o n --clean /dir1 /dir2)
+
+It scans 2 directory hierarchies (/dir1 & /dir2) and cleans out any
+checksums whose inodes are no longer found in those directories (so that
+directory args are presumed to be complete for this host's DB contents).
+
+The following command will scan all the files in the /dir2 directory (without
+recursive scanning, due to the bf(--no-r) option) and check them against
+the DB:
+
+verb( rsyncdb --db=/etc/db.conf --check --no-r /dir2)
+
+Any errors found are output as well as being fixed in the DB. (See
+bf(--no-update) for how to check without updating.)
+
+The following command will output MD5 sums for all the files found in the
+directories mentioned, even if they are unchanged (due to the
+bf(--output=u) option):
+
+verb( rsyncdb --db=/etc/db.conf -rous /dir* >/tmp/md5sums.txt)
+
+This is just like running md5sum, only faster. Unlike md5sum, you can't
+specify a single file, so use bf(--no-r) and grep the output if you just
+want to see a single file's value.
+
+The following command initializes a new DB, and is required for any new DB:
+
+verb( rsyncdb --db=/etc/db.conf --init --mounts)
+
+The bf(--init) option should only be used once (unless you want to
+destroy existing data). The bf(--mounts) option may need to be used
+periodically.
+
+manpagesection(OPTIONS SUMMARY)
+
+Rsyncdb accepts the following options: verb(
+ --db=CONFIG Specify the CONFIG file to read for the DB info.
+ --db-lax Ignore ctime changes (use with CAUTION).
+ --no-recursive Avoid the default --recursive (-r) scanning behavior.
+ -s, --sums=SUMS List which checksums to update (default: md5).
+ -o, --output=STR One or more letters of what to output (default is nothing).
+ -c, --check Check the checksums (by reading the files) and fix any
+ issues. Enables --output=i.
+ --clean Note all inodes in the DIRS and remove DB extras.
+ -N, --no-update Avoids updating/adding info with --check and/or --clean.
+ --init Initialize a DB by (re-)creating its tables.
+ --mounts Scan for mounted filesystems and update the DB.
+ -q, --quiet Disables the default non-error output.
+ -h, --help Display this help message.)
+
+manpageoptions()
+
+Rsyncdb accepts both long (double-dash + word) and short (single-dash + letter)
+options. The full list of the available options are described below. If an
+option can be specified in more than one way, the choices are comma-separated.
+Some options only have a long variant, not a short. If the option takes a
+parameter, the parameter is only listed after the long variant, even though it
+must also be specified for the short. When specifying a parameter, you can
+either use the form --option=param or replace the '=' with whitespace. The
+parameter may need to be quoted in some manner for it to survive the shell's
+command-line parsing.
+
+startdit()
+dit(bf(--db=CONFIG_FILE)) This tells rsyncdb what DB-config file to read
+for the DB setup. This is the same as the option in rsync, so refer to
+that manpage for full details.
+
+dit(bf(--db-lax)) This option works just like it does in rsync, so refer to
+that manpage for full details.
+
+dit(bf(--no-recursive, --no-r)) This disables the default recursive
+directory scan that is performed on the listed directory args. The
+options bf(--recursive) and bf(-r) are also accepted, if someone wants
+to override an earlier bf(--no-r) override.
+
+dit(bf(--sums=SUMS, -s)) Only output/update the listed checksum types. By
+default we deal with just the newer md5 checksums (i.e. bf(--sums=5)).
+
+Note that this option does NOT affect the order that checksums are output
+if "-o s" is enabled, so bf(-s5,4) is the same as bf(-s4,5).
+
+dit(bf(--output=STR, -o)) The output option lets you specify one or more
+letters indicating what information should be output. The default is to
+output "d" and "n" if bf(--output) is not specified.
+
+The following letters are accepted:
+
+quote(itemization(
+ it() bf(d) outputs "... dir_name ..." lines for each directory in our scan.
+ it() bf(n) outputs the names of files with changes (implied by all but "d").
+ it() bf(s) outputs checksum info for changes (implies bf(n)).
+ it() bf(u) outputs unchanged files too (implies bf(n)).
+ it() bf(i) outputs prefixed change info. The output strings are:
+ quote(itemization(
+ it() bf(!i) indicates that the time and/or size is wrong.
+ it() bf(+4) indicates the MD4 sum is missing.
+ it() bf(+5) indicates the MD5 sum is missing.
+ it() bf(!4) indicates the MD4 sum is wrong.
+ it() bf(!5) indicates the MD5 sum is wrong.
+ it() bf(?4) indicates an unknown MD4 difference. This can happen if we
+ didn't need to read the file; i.e. if the time/size is wrong and no sum
+ info was requested.
+ it() bf(?5) indicates an unknown MD5 difference.
+ ))
+))
+
+dit(bf(--check, -c)) Check the checksums (forcing the reading of all the
+files) and fix any issues that are found. Forces bf(--output=ni) on.
+
+dit(bf(--clean)) Makes a temp-DB of all the inodes that we find in all the
+listed directories and removes any extraneous checksums from the DB. You
+will need to specify all the mounted directories that are present (and
+listed as mounted) in the DB on this host or else the checksums from the
+unvisited directories will be discarded from the DB. If you want to just
+--clean without adding or updating the info of new or changed files,
+specify bf(--no-update) as well.
+
+See the bf(--mount)
+
+dit(bf(--no-update, -N)) Avoids updating/adding info with bf(--check)
+and/or bf(--clean).
+
+dit(bf(--quiet, -q)) Disable the default (non-error) output settings. This
+turns off the messages that bf(--init), bf(--mount), and bf(--clean) output,
+and makes the default for bf(--output) be nothing (though an explicit
+bf(--output) option is not affected).
+
+dit(bf(--init)) Create the tables in the DB. If it is used on an existing
+DB, all the existing tables are dropped and re-created.
+
+This option disables scanning for checksum information, but may be combined
+with bf(--mounts).
+
+dit(bf(--mounts)) Populate the "disk" DB with the available device numbers
+and change any mounted/unmount information for devices. This should be run
+every time a mount-change happens that may affect a directory hierarchy in
+the DB. Rsyncdb will not save any checksums for a device that is not
+listed in the "disk" table.
+
+Some advanced users may want to maintain the disk table themselves in order
+to support mounting a drive in different (or multiple) locations, etc.
+
+This option disables scanning for checksum information, but may be combined
+with bf(--init).
+
+dit(bf(--help, -h)) Display a summary of the options.
+
+enddit()
+
+manpageseealso()
+
+bf(rsync)(1)
+
+manpageauthor()
+
+Rsyncdb was written by Wayne Davison.
diff -Nurp a/config.h.in b/config.h.in
--- a/config.h.in
+++ b/config.h.in
@@ -213,6 +213,9 @@
/* Define to 1 if you have the `inet' library (-linet). */
#undef HAVE_LIBINET
+/* Define to 1 if you have the `mysqlclient' library (-lmysqlclient). */
+#undef HAVE_LIBMYSQLCLIENT
+
/* Define to 1 if you have the `nsl' library (-lnsl). */
#undef HAVE_LIBNSL
@@ -231,6 +234,9 @@
/* Define to 1 if you have the `socket' library (-lsocket). */
#undef HAVE_LIBSOCKET
+/* Define to 1 if you have the `sqlite3' library (-lsqlite3). */
+#undef HAVE_LIBSQLITE3
+
/* Define to 1 if you have the `z' library (-lz). */
#undef HAVE_LIBZ
@@ -296,6 +302,9 @@
/* Define to 1 if you have the `mtrace' function. */
#undef HAVE_MTRACE
+/* Define to 1 if you have the <mysql/mysql.h> header file. */
+#undef HAVE_MYSQL_MYSQL_H
+
/* Define to 1 if you have the <ndir.h> header file, and it defines `DIR'. */
#undef HAVE_NDIR_H
@@ -404,6 +413,15 @@
/* True if you have Solaris xattrs */
#undef HAVE_SOLARIS_XATTRS
+/* Define to 1 if you have the <sqlite3.h> header file. */
+#undef HAVE_SQLITE3_H
+
+/* Define to 1 if you have the `sqlite3_open_v2' function. */
+#undef HAVE_SQLITE3_OPEN_V2
+
+/* Define to 1 if you have the `sqlite3_prepare_v2' function. */
+#undef HAVE_SQLITE3_PREPARE_V2
+
/* Define to 1 if you have the <stdint.h> header file. */
#undef HAVE_STDINT_H
diff -Nurp a/configure.sh b/configure.sh
--- a/configure.sh
+++ b/configure.sh
@@ -625,6 +625,7 @@ ac_includes_default="\
ac_header_list=
ac_subst_vars='LTLIBOBJS
+MYSQL_CONFIG
STUNNEL4
STUNNEL
MAKE_MAN
@@ -721,6 +722,8 @@ enable_iconv_open
enable_iconv
enable_acl_support
enable_xattr_support
+enable_mysql
+enable_sqlite
'
ac_precious_vars='build_alias
host_alias
@@ -1355,6 +1358,8 @@ Optional Features:
--disable-iconv disable rsync's --iconv option
--disable-acl-support disable ACL support
--disable-xattr-support disable extended attributes
+ --disable-mysql disable mysql DB support
+ --disable-sqlite disable sqlite DB support
Optional Packages:
--with-PACKAGE[=ARG] use PACKAGE [ARG=yes]
@@ -5597,6 +5602,7 @@ for ac_header in sys/fcntl.h sys/select.
unistd.h utime.h grp.h compat.h sys/param.h ctype.h sys/wait.h \
sys/ioctl.h sys/filio.h string.h stdlib.h sys/socket.h sys/mode.h \
sys/un.h sys/attr.h mcheck.h arpa/inet.h arpa/nameser.h locale.h \
+ mysql/mysql.h sqlite3.h \
netdb.h malloc.h float.h limits.h iconv.h libcharset.h langinfo.h \
sys/acl.h acl/libacl.h attr/xattr.h sys/xattr.h sys/extattr.h \
popt.h popt/popt.h linux/falloc.h netinet/in_systm.h netinet/ip.h \
@@ -9110,6 +9116,196 @@ $as_echo "$rsync_warn_flag" >&6; }
fi
fi
+{ $as_echo "$as_me:${as_lineno-$LINENO}: checking whether to include mysql DB support" >&5
+$as_echo_n "checking whether to include mysql DB support... " >&6; }
+# Check whether --enable-mysql was given.
+if test "${enable_mysql+set}" = set; then :
+ enableval=$enable_mysql;
+fi
+
+
+if test x"$enable_mysql" = x"no"; then
+ { $as_echo "$as_me:${as_lineno-$LINENO}: result: no" >&5
+$as_echo "no" >&6; }
+else
+ { $as_echo "$as_me:${as_lineno-$LINENO}: result: yes" >&5
+$as_echo "yes" >&6; }
+ # Extract the first word of "mysql_config", so it can be a program name with args.
+set dummy mysql_config; ac_word=$2
+{ $as_echo "$as_me:${as_lineno-$LINENO}: checking for $ac_word" >&5
+$as_echo_n "checking for $ac_word... " >&6; }
+if ${ac_cv_prog_MYSQL_CONFIG+:} false; then :
+ $as_echo_n "(cached) " >&6
+else
+ if test -n "$MYSQL_CONFIG"; then
+ ac_cv_prog_MYSQL_CONFIG="$MYSQL_CONFIG" # Let the user override the test.
+else
+as_save_IFS=$IFS; IFS=$PATH_SEPARATOR
+for as_dir in $PATH
+do
+ IFS=$as_save_IFS
+ test -z "$as_dir" && as_dir=.
+ for ac_exec_ext in '' $ac_executable_extensions; do
+ if as_fn_executable_p "$as_dir/$ac_word$ac_exec_ext"; then
+ ac_cv_prog_MYSQL_CONFIG="1"
+ $as_echo "$as_me:${as_lineno-$LINENO}: found $as_dir/$ac_word$ac_exec_ext" >&5
+ break 2
+ fi
+done
+ done
+IFS=$as_save_IFS
+
+ test -z "$ac_cv_prog_MYSQL_CONFIG" && ac_cv_prog_MYSQL_CONFIG="0"
+fi
+fi
+MYSQL_CONFIG=$ac_cv_prog_MYSQL_CONFIG
+if test -n "$MYSQL_CONFIG"; then
+ { $as_echo "$as_me:${as_lineno-$LINENO}: result: $MYSQL_CONFIG" >&5
+$as_echo "$MYSQL_CONFIG" >&6; }
+else
+ { $as_echo "$as_me:${as_lineno-$LINENO}: result: no" >&5
+$as_echo "no" >&6; }
+fi
+
+
+ if test x$MYSQL_CONFIG = x1; then
+ { $as_echo "$as_me:${as_lineno-$LINENO}: checking for mysql version >= 4" >&5
+$as_echo_n "checking for mysql version >= 4... " >&6; }
+ mysql_version=`mysql_config --version`
+ mysql_major_version=`echo $mysql_version | sed 's/\..*//'`
+ if test $mysql_major_version -lt 4; then
+ { $as_echo "$as_me:${as_lineno-$LINENO}: result: no.. skipping MySQL" >&5
+$as_echo "no.. skipping MySQL" >&6; }
+ else
+ { $as_echo "$as_me:${as_lineno-$LINENO}: result: yes" >&5
+$as_echo "yes" >&6; }
+
+ MYSQL_CFLAGS=`mysql_config --cflags`
+ MYSQL_LIBS=`mysql_config --libs`
+
+ CPPFLAGS="$CPPFLAGS $MYSQL_CFLAGS"
+ LIBS="$MYSQL_LIBS $LIBS"
+
+ { $as_echo "$as_me:${as_lineno-$LINENO}: checking for mysql_init in -lmysqlclient" >&5
+$as_echo_n "checking for mysql_init in -lmysqlclient... " >&6; }
+if ${ac_cv_lib_mysqlclient_mysql_init+:} false; then :
+ $as_echo_n "(cached) " >&6
+else
+ ac_check_lib_save_LIBS=$LIBS
+LIBS="-lmysqlclient $LIBS"
+cat confdefs.h - <<_ACEOF >conftest.$ac_ext
+/* end confdefs.h. */
+
+/* Override any GCC internal prototype to avoid an error.
+ Use char because int might match the return type of a GCC
+ builtin and then its argument prototype would still apply. */
+#ifdef __cplusplus
+extern "C"
+#endif
+char mysql_init ();
+int
+main ()
+{
+return mysql_init ();
+ ;
+ return 0;
+}
+_ACEOF
+if ac_fn_c_try_link "$LINENO"; then :
+ ac_cv_lib_mysqlclient_mysql_init=yes
+else
+ ac_cv_lib_mysqlclient_mysql_init=no
+fi
+rm -f core conftest.err conftest.$ac_objext \
+ conftest$ac_exeext conftest.$ac_ext
+LIBS=$ac_check_lib_save_LIBS
+fi
+{ $as_echo "$as_me:${as_lineno-$LINENO}: result: $ac_cv_lib_mysqlclient_mysql_init" >&5
+$as_echo "$ac_cv_lib_mysqlclient_mysql_init" >&6; }
+if test "x$ac_cv_lib_mysqlclient_mysql_init" = xyes; then :
+ cat >>confdefs.h <<_ACEOF
+#define HAVE_LIBMYSQLCLIENT 1
+_ACEOF
+
+ LIBS="-lmysqlclient $LIBS"
+
+fi
+
+ fi
+ fi
+fi
+
+{ $as_echo "$as_me:${as_lineno-$LINENO}: checking whether to include sqlite DB support" >&5
+$as_echo_n "checking whether to include sqlite DB support... " >&6; }
+# Check whether --enable-sqlite was given.
+if test "${enable_sqlite+set}" = set; then :
+ enableval=$enable_sqlite;
+fi
+
+
+if test x"$enable_sqlite" = x"no"; then
+ { $as_echo "$as_me:${as_lineno-$LINENO}: result: no" >&5
+$as_echo "no" >&6; }
+else
+ { $as_echo "$as_me:${as_lineno-$LINENO}: checking for sqlite3_open in -lsqlite3" >&5
+$as_echo_n "checking for sqlite3_open in -lsqlite3... " >&6; }
+if ${ac_cv_lib_sqlite3_sqlite3_open+:} false; then :
+ $as_echo_n "(cached) " >&6
+else
+ ac_check_lib_save_LIBS=$LIBS
+LIBS="-lsqlite3 $LIBS"
+cat confdefs.h - <<_ACEOF >conftest.$ac_ext
+/* end confdefs.h. */
+
+/* Override any GCC internal prototype to avoid an error.
+ Use char because int might match the return type of a GCC
+ builtin and then its argument prototype would still apply. */
+#ifdef __cplusplus
+extern "C"
+#endif
+char sqlite3_open ();
+int
+main ()
+{
+return sqlite3_open ();
+ ;
+ return 0;
+}
+_ACEOF
+if ac_fn_c_try_link "$LINENO"; then :
+ ac_cv_lib_sqlite3_sqlite3_open=yes
+else
+ ac_cv_lib_sqlite3_sqlite3_open=no
+fi
+rm -f core conftest.err conftest.$ac_objext \
+ conftest$ac_exeext conftest.$ac_ext
+LIBS=$ac_check_lib_save_LIBS
+fi
+{ $as_echo "$as_me:${as_lineno-$LINENO}: result: $ac_cv_lib_sqlite3_sqlite3_open" >&5
+$as_echo "$ac_cv_lib_sqlite3_sqlite3_open" >&6; }
+if test "x$ac_cv_lib_sqlite3_sqlite3_open" = xyes; then :
+ cat >>confdefs.h <<_ACEOF
+#define HAVE_LIBSQLITE3 1
+_ACEOF
+
+ LIBS="-lsqlite3 $LIBS"
+
+fi
+
+ for ac_func in sqlite3_open_v2 sqlite3_prepare_v2
+do :
+ as_ac_var=`$as_echo "ac_cv_func_$ac_func" | $as_tr_sh`
+ac_fn_c_check_func "$LINENO" "$ac_func" "$as_ac_var"
+if eval test \"x\$"$as_ac_var"\" = x"yes"; then :
+ cat >>confdefs.h <<_ACEOF
+#define `$as_echo "HAVE_$ac_func" | $as_tr_cpp` 1
+_ACEOF
+
+fi
+done
+
+fi
+
case "$CC" in
' checker'*|checker*)
diff -Nurp a/proto.h b/proto.h
--- a/proto.h
+++ b/proto.h
@@ -62,6 +62,18 @@ int daemon_main(void);
void set_allow_inc_recurse(void);
void setup_protocol(int f_out,int f_in);
int claim_connection(char *fname, int max_connections);
+int db_read_config(enum logcode code, const char *config_file);
+int db_connect(int select_many);
+void db_disconnect(BOOL commit);
+unsigned int get_disk_id(int64 devno);
+int db_get_checksum(const STRUCT_STAT *st_p, char *sum);
+int db_get_both_checksums(const STRUCT_STAT *st_p, int *right_sum_cnt, int *wrong_sum_cnt, char **sum4, char **sum5);
+int db_set_checksum(int mdnum, const STRUCT_STAT *st_p, const char *sum);
+int db_update_ctime(int mdnum, const STRUCT_STAT *st_p);
+int db_clean_init(void);
+int db_note_present(int disk_id, int64 ino);
+int db_clean_inodes(void);
+NORETURN void run_dbonly(const char **args);
enum delret delete_item(char *fbuf, uint16 mode, uint16 flags);
uint16 get_del_for_flag(uint16 mode);
void set_filter_dir(const char *dir, unsigned int dirlen);
@@ -201,6 +213,7 @@ int lp_rsync_port(void);
char *lp_auth_users(int module_id);
char *lp_charset(int module_id);
char *lp_comment(int module_id);
+char *lp_db_config(int module_id);
char *lp_dont_compress(int module_id);
char *lp_exclude(int module_id);
char *lp_exclude_from(int module_id);
@@ -228,6 +241,7 @@ int lp_max_connections(int module_id);
int lp_max_verbosity(int module_id);
int lp_syslog_facility(int module_id);
int lp_timeout(int module_id);
+BOOL lp_db_lax(int module_id);
BOOL lp_fake_super(int module_id);
BOOL lp_forward_lookup(int module_id);
BOOL lp_ignore_errors(int module_id);
diff -Nurp a/rsync.1 b/rsync.1
--- a/rsync.1
+++ b/rsync.1
@@ -416,6 +416,9 @@ to the detailed description below for a
\-q, \-\-quiet suppress non\-error messages
\-\-no\-motd suppress daemon\-mode MOTD (see caveat)
\-c, \-\-checksum skip based on checksum, not mod\-time & size
+ \-\-db=CONFIG_FILE specify a CONFIG_FILE for DB checksums
+ \-\-db\-only=CONFIG_FILE Behave like rsyncdb (see that manpage).
+ \-\-db\-lax Ignore ctime changes (use with CAUTION).
\-a, \-\-archive archive mode; equals \-rlptgoD (no \-H,\-A,\-X)
\-\-no\-OPTION turn off an implied OPTION (e.g. \-\-no\-D)
\-r, \-\-recursive recurse into directories
@@ -752,6 +755,78 @@ option\(cq\&s before\-the\-transfer \(dq
For protocol 30 and beyond (first supported in 3.0.0), the checksum used is
MD5. For older protocols, the checksum used is MD4.
.IP
+.IP "\fB\-\-db=CONFIG_FILE\fP"
+This option specifies a CONFIG_FILE to read
+that holds connection details for a database of checksum information.
+When combined with the \fB\-\-checksum\fP (\fB\-c\fP) option, rsync will try to
+use cached checksum information from the DB, and will update it if it is
+missing.
+.IP
+The currently supported DB choices are MySQL and SQLite. For example, a
+MySQL configuration might look like this:
+.IP
+.nf
+ dbtype: mysql
+ dbhost: 127.0.0.1
+ dbname: rsyncdb
+ dbuser: rsyncuser
+ dbpass: somepass
+ port: 3306
+ thishost: hostname
+.fi
+
+.IP
+And a SQLite configuration might look like this:
+.IP
+.nf
+ dbtype: SQLite
+ dbname: /var/cache/rsync/sum.db
+ transaction: 1
+.fi
+
+.IP
+Both the \fB\-\-db\fP and \fB\-\-db\-lax\fP options only affect the side where the
+option is used. To affect the remote side of a remote\-shell connection,
+use the \fB\-\-remote\-option\fP (\fB\-M\fP) option. For example, to specify the
+same options on both sides, you could specify something like this:
+.IP
+.nf
+ rsync \-avc {\-M,}\-\-db=/etc/rsyncdb.conf src/ host:dest/
+.fi
+
+.IP
+For a local copy, this option affects both the source and the destination.
+If you wish a local copy to enable this option just for the destination
+files, specify \fB\-M\-\-db=CONFIG\fP (the same for \fB\-M\-\-db\-lax\fP). If you wish
+a local copy to enable this option just for the source files, combine
+\fB\-\-db=CONFIG\fP with \fB\-M\-\-no\-db\fP (similarly use \fB\-M\-\-no\-db\-lax\fP).
+.IP
+See the perl script \(dq\&rsyncdb\(dq\& in the support directory of the source code
+(which may also be installed in /usr/bin) for a way to create the tables,
+populate the mounted\-disk information, check files against their checksums,
+and update both the MD4 and MD5 checksums for files at the same time (since
+an rsync copy will only update one or the other).
+.IP
+You can use a single MySQL DB for all your hosts if you give each one
+their own \(dq\&thishost\(dq\& name and setup their device\-mapping data. Or feel
+free to use separate databases, separate servers, etc. See the rsync
+daemon\(cq\&s \(dq\&db config\(dq\& parameter for how to configure a daemon to use a DB
+(since a client cannot control this parameter on a daemon).
+.IP
+.IP "\fB\-\-db\-lax\fP"
+This option can be used to modify the inode\-matching
+algorithm used by \fB\-\-db\fP to one that ignores the ctime. This can be very
+DANGEROUS unless your files are known to ALWAYS be updated in a safe manner.
+If unsure, don\(cq\&t use it.
+.IP
+The reason you might want to use it is that the ctime (inode change time) is
+changed by an added hard\-link, or the file being moving around. To use this
+option safely you must be CERTAIN that either rsync w/\-\-db is the only program
+adding files into the cached hierarchies, OR that all new files will have new
+modify times (never a historical mtime that might match an orphaned inode).
+So, for certain applications, such as mirrors of new tar releases, this option
+can save a lot of unneeded checksum re\-computation due to ctime changes.
+.IP
.IP "\fB\-a, \-\-archive\fP"
This is equivalent to \fB\-rlptgoD\fP. It is a quick
way of saying you want recursion and want to preserve almost
diff -Nurp a/rsyncd.conf.5 b/rsyncd.conf.5
--- a/rsyncd.conf.5
+++ b/rsyncd.conf.5
@@ -364,6 +364,24 @@ is daemon. This setting has no effect i
non\-empty string (either set in the per\-modules settings, or inherited
from the global settings).
.IP
+.IP "\fBdb config\fP"
+This parameter specifies a config file to read that
+holds connection details for a database of checksum information.
+.IP
+The config file will be read\-in prior to any chroot restrictions, but
+the connection occurs from inside the chroot. This means that you
+should use a socket connection (e.g. 127.0.0.1 rather than localhost)
+for a MySQL config from inside a chroot. For SQLite, the DB file must
+be placed inside the chroot (though it can be placed outside the
+transfer dir if you configured an inside\-chroot path).
+.IP
+See the \fB\-\-db=CONFIG_FILE\fP option for full details.
+.IP
+.IP "\fBdb lax\fP"
+This parameter specifies that a \(dq\&db config\(dq\& setup should use
+lax (no ctime) lookups. See the rsync manpage\(cq\&s section \-n \fB\-\-db\-lax\fP for
+some warnings about using this setting.
+.IP
.IP "\fBsyslog tag\fP"
This parameter allows you to specify the syslog
tag to use when logging messages from the rsync daemon. The default is
diff -Nurp a/rsyncdb.1 b/rsyncdb.1
--- a/rsyncdb.1
+++ b/rsyncdb.1
@@ -0,0 +1,234 @@
+.TH "rsync" "1" "23 Jun 2013" "" ""
+.SH "NAME"
+rsyncdb \- Maintain an rsync checksum DB
+.SH "SYNOPSIS"
+
+.PP
+.nf
+rsyncdb \-\-db=CONFIG [OPTION...] [DIR...]
+.fi
+
+.PP
+.SH "DESCRIPTION"
+
+.PP
+Rsyncdb can maintain a checksum\-caching DB that rsync can use to make its
+\fB\-\-checksum\fP option more optimal. You must specify a config file via
+the \fB\-\-db=CONFIG_FILE\fP option in order for rsyncdb to know what DB to
+manipulate. See the rsync manpage\(cq\&s \fB\-\-db\fP option for full details on
+the file\(cq\&s format.
+.PP
+You can specify one or more directory args for rsyncdb to scan. If no
+DIR args are specified, the current directory is assumed to be the spot
+to start scanning.
+.PP
+Note that the rsyncdb program is usually just a symlink to the rsync program.
+You can force rsync to behave as rsyncdb either by having a symlink (or
+hardlink) name that ends with \(dq\&db\(dq\& or by \fBstarting\fP the rsync args with
+\fB\-\-db\-only=CONFIG\fP (and that option works just like \fB\-\-db=CONFIG\fP to
+a program named rsyncdb).
+.PP
+.SH "EXAMPLES"
+
+.PP
+The following command will update checksum information in the database
+described in the /etc/db.conf file:
+.PP
+.nf
+ rsyncdb \-\-db=/etc/db.conf \-o n \-\-clean /dir1 /dir2
+.fi
+
+.PP
+It scans 2 directory hierarchies (/dir1 & /dir2) and cleans out any
+checksums whose inodes are no longer found in those directories (so that
+directory args are presumed to be complete for this host\(cq\&s DB contents).
+.PP
+The following command will scan all the files in the /dir2 directory (without
+recursive scanning, due to the \fB\-\-no\-r\fP option) and check them against
+the DB:
+.PP
+.nf
+ rsyncdb \-\-db=/etc/db.conf \-\-check \-\-no\-r /dir2
+.fi
+
+.PP
+Any errors found are output as well as being fixed in the DB. (See
+\fB\-\-no\-update\fP for how to check without updating.)
+.PP
+The following command will output MD5 sums for all the files found in the
+directories mentioned, even if they are unchanged (due to the
+\fB\-\-output=u\fP option):
+.PP
+.nf
+ rsyncdb \-\-db=/etc/db.conf \-rous /dir* >/tmp/md5sums.txt
+.fi
+
+.PP
+This is just like running md5sum, only faster. Unlike md5sum, you can\(cq\&t
+specify a single file, so use \fB\-\-no\-r\fP and grep the output if you just
+want to see a single file\(cq\&s value.
+.PP
+The following command initializes a new DB, and is required for any new DB:
+.PP
+.nf
+ rsyncdb \-\-db=/etc/db.conf \-\-init \-\-mounts
+.fi
+
+.PP
+The \fB\-\-init\fP option should only be used once (unless you want to
+destroy existing data). The \fB\-\-mounts\fP option may need to be used
+periodically.
+.PP
+.SH "OPTIONS SUMMARY"
+
+.PP
+Rsyncdb accepts the following options:
+.nf
+
+ \-\-db=CONFIG Specify the CONFIG file to read for the DB info.
+ \-\-db\-lax Ignore ctime changes (use with CAUTION).
+ \-\-no\-recursive Avoid the default \-\-recursive (\-r) scanning behavior.
+ \-s, \-\-sums=SUMS List which checksums to update (default: md5).
+ \-o, \-\-output=STR One or more letters of what to output (default is nothing).
+ \-c, \-\-check Check the checksums (by reading the files) and fix any
+ issues. Enables \-\-output=i.
+ \-\-clean Note all inodes in the DIRS and remove DB extras.
+ \-N, \-\-no\-update Avoids updating/adding info with \-\-check and/or \-\-clean.
+ \-\-init Initialize a DB by (re\-)creating its tables.
+ \-\-mounts Scan for mounted filesystems and update the DB.
+ \-q, \-\-quiet Disables the default non\-error output.
+ \-h, \-\-help Display this help message.
+.fi
+
+.PP
+.SH "OPTIONS"
+
+.PP
+Rsyncdb accepts both long (double\-dash + word) and short (single\-dash + letter)
+options. The full list of the available options are described below. If an
+option can be specified in more than one way, the choices are comma\-separated.
+Some options only have a long variant, not a short. If the option takes a
+parameter, the parameter is only listed after the long variant, even though it
+must also be specified for the short. When specifying a parameter, you can
+either use the form \-\-option=param or replace the \(cq\&=\(cq\& with whitespace. The
+parameter may need to be quoted in some manner for it to survive the shell\(cq\&s
+command\-line parsing.
+.PP
+.IP "\fB\-\-db=CONFIG_FILE\fP"
+This tells rsyncdb what DB\-config file to read
+for the DB setup. This is the same as the option in rsync, so refer to
+that manpage for full details.
+.IP
+.IP "\fB\-\-db\-lax\fP"
+This option works just like it does in rsync, so refer to
+that manpage for full details.
+.IP
+.IP "\fB\-\-no\-recursive, \-\-no\-r\fP"
+This disables the default recursive
+directory scan that is performed on the listed directory args. The
+options \fB\-\-recursive\fP and \fB\-r\fP are also accepted, if someone wants
+to override an earlier \fB\-\-no\-r\fP override.
+.IP
+.IP "\fB\-\-sums=SUMS, \-s\fP"
+Only output/update the listed checksum types. By
+default we deal with just the newer md5 checksums (i.e. \fB\-\-sums=5\fP).
+.IP
+Note that this option does NOT affect the order that checksums are output
+if \(dq\&\-o s\(dq\& is enabled, so \fB\-s5,4\fP is the same as \fB\-s4,5\fP.
+.IP
+.IP "\fB\-\-output=STR, \-o\fP"
+The output option lets you specify one or more
+letters indicating what information should be output. The default is to
+output \(dq\&d\(dq\& and \(dq\&n\(dq\& if \fB\-\-output\fP is not specified.
+.IP
+The following letters are accepted:
+.IP
+.RS
+.IP o
+\fBd\fP outputs \(dq\&... dir_name ...\(dq\& lines for each directory in our scan.
+.IP o
+\fBn\fP outputs the names of files with changes (implied by all but \(dq\&d\(dq\&).
+.IP o
+\fBs\fP outputs checksum info for changes (implies \fBn\fP).
+.IP o
+\fBu\fP outputs unchanged files too (implies \fBn\fP).
+.IP o
+\fBi\fP outputs prefixed change info. The output strings are:
+.RS
+.IP o
+\fB!i\fP indicates that the time and/or size is wrong.
+.IP o
+\fB+4\fP indicates the MD4 sum is missing.
+.IP o
+\fB+5\fP indicates the MD5 sum is missing.
+.IP o
+\fB!4\fP indicates the MD4 sum is wrong.
+.IP o
+\fB!5\fP indicates the MD5 sum is wrong.
+.IP o
+\fB?4\fP indicates an unknown MD4 difference. This can happen if we
+didn\(cq\&t need to read the file; i.e. if the time/size is wrong and no sum
+info was requested.
+.IP o
+\fB?5\fP indicates an unknown MD5 difference.
+.RE
+.RE
+
+.IP
+.IP "\fB\-\-check, \-c\fP"
+Check the checksums (forcing the reading of all the
+files) and fix any issues that are found. Forces \fB\-\-output=ni\fP on.
+.IP
+.IP "\fB\-\-clean\fP"
+Makes a temp\-DB of all the inodes that we find in all the
+listed directories and removes any extraneous checksums from the DB. You
+will need to specify all the mounted directories that are present (and
+listed as mounted) in the DB on this host or else the checksums from the
+unvisited directories will be discarded from the DB. If you want to just
+\-\-clean without adding or updating the info of new or changed files,
+specify \fB\-\-no\-update\fP as well.
+.IP
+See the \fB\-\-mount\fP
+.IP
+.IP "\fB\-\-no\-update, \-N\fP"
+Avoids updating/adding info with \fB\-\-check\fP
+and/or \fB\-\-clean\fP.
+.IP
+.IP "\fB\-\-quiet, \-q\fP"
+Disable the default (non\-error) output settings. This
+turns off the messages that \fB\-\-init\fP, \fB\-\-mount\fP, and \fB\-\-clean\fP output,
+and makes the default for \fB\-\-output\fP be nothing (though an explicit
+\fB\-\-output\fP option is not affected).
+.IP
+.IP "\fB\-\-init\fP"
+Create the tables in the DB. If it is used on an existing
+DB, all the existing tables are dropped and re\-created.
+.IP
+This option disables scanning for checksum information, but may be combined
+with \fB\-\-mounts\fP.
+.IP
+.IP "\fB\-\-mounts\fP"
+Populate the \(dq\&disk\(dq\& DB with the available device numbers
+and change any mounted/unmount information for devices. This should be run
+every time a mount\-change happens that may affect a directory hierarchy in
+the DB. Rsyncdb will not save any checksums for a device that is not
+listed in the \(dq\&disk\(dq\& table.
+.IP
+Some advanced users may want to maintain the disk table themselves in order
+to support mounting a drive in different (or multiple) locations, etc.
+.IP
+This option disables scanning for checksum information, but may be combined
+with \fB\-\-init\fP.
+.IP
+.IP "\fB\-\-help, \-h\fP"
+Display a summary of the options.
+.IP
+.SH "SEE ALSO"
+
+.PP
+\fBrsync\fP(1)
+.PP
+.SH "AUTHOR"
+
+.PP
+Rsyncdb was written by Wayne Davison.