Add new-style compression that skips matching data.

Adding new-style compression that only compresses the literal data that
is sent over the wire and not also matching file data that was not sent.
This new-style compression is compatible with external zlib instances,
and will eventually become the default (once enough time has passed that
all servers support the --new-compress and --old-compress options).

NOTE: if you build rsync with an external zlib (i.e. if you specified
configure --with-included-zlib=no) you will ONLY get support for the
--new-compress option!  A client will treat -z as uncompressed (with a
warning) and a server will exit with an error (unless -zz was used).

6 files changed, 77 insertions, 34 deletions

diff --git a/configure.ac b/configure.ac
index dccb2aaf..c7b28c52 100644
--- a/configure.ac
+++ b/configure.ac
@@ -792,6 +792,7 @@ if test x"$with_included_zlib" = x"yes"; then
     BUILD_ZLIB='$(zlib_OBJS)'
     CFLAGS="$CFLAGS -I$srcdir/zlib"
 else
+    AC_DEFINE(EXTERNAL_ZLIB, 1, [Define to 1 if using external zlib])
     AC_MSG_RESULT(no)
 fi
 
diff --git a/options.c b/options.c
index dc9e62a4..0aa64bf0 100644
--- a/options.c
+++ b/options.c
@@ -32,6 +32,8 @@ extern unsigned int module_dirlen;
 extern filter_rule_list filter_list;
 extern filter_rule_list daemon_filter_list;
 
+#define NOT_SPECIFIED (-42)
+
 int make_backups = 0;
 
 /**
@@ -75,7 +77,7 @@ int protocol_version = PROTOCOL_VERSION;
 int sparse_files = 0;
 int preallocate_files = 0;
 int do_compression = 0;
-int def_compress_level = Z_DEFAULT_COMPRESSION;
+int def_compress_level = NOT_SPECIFIED;
 int am_root = 0; /* 0 = normal, 1 = root, 2 = --super, -1 = --fake-super */
 int am_server = 0;
 int am_sender = 0;
@@ -965,10 +967,12 @@ static struct poptOption long_options[] = {
   {"no-fuzzy",         0,  POPT_ARG_VAL,    &fuzzy_basis, 0, 0, 0 },
   {"no-y",             0,  POPT_ARG_VAL,    &fuzzy_basis, 0, 0, 0 },
   {"compress",        'z', POPT_ARG_NONE,   0, 'z', 0, 0 },
+  {"old-compress",     0,  POPT_ARG_VAL,    &do_compression, 1, 0, 0 },
+  {"new-compress",     0,  POPT_ARG_VAL,    &do_compression, 2, 0, 0 },
   {"no-compress",      0,  POPT_ARG_VAL,    &do_compression, 0, 0, 0 },
   {"no-z",             0,  POPT_ARG_VAL,    &do_compression, 0, 0, 0 },
   {"skip-compress",    0,  POPT_ARG_STRING, &skip_compress, 0, 0, 0 },
-  {"compress-level",   0,  POPT_ARG_INT,    &def_compress_level, 'z', 0, 0 },
+  {"compress-level",   0,  POPT_ARG_INT,    &def_compress_level, 0, 0, 0 },
   {0,                 'P', POPT_ARG_NONE,   0, 'P', 0, 0 },
   {"progress",         0,  POPT_ARG_VAL,    &do_progress, 1, 0, 0 },
   {"no-progress",      0,  POPT_ARG_VAL,    &do_progress, 0, 0, 0 },
@@ -1543,18 +1547,7 @@ int parse_arguments(int *argc_p, const char ***argv_p)
 			break;
 
 		case 'z':
-			if (def_compress_level < Z_DEFAULT_COMPRESSION
-			 || def_compress_level > Z_BEST_COMPRESSION) {
-				snprintf(err_buf, sizeof err_buf,
-					"--compress-level value is invalid: %d\n",
-					def_compress_level);
-				return 0;
-			}
-			do_compression = def_compress_level != Z_NO_COMPRESSION;
-			if (do_compression && refused_compress) {
-				create_refuse_error(refused_compress);
-				return 0;
-			}
+			do_compression++;
 			break;
 
 		case 'M':
@@ -1829,6 +1822,33 @@ int parse_arguments(int *argc_p, const char ***argv_p)
 		exit_cleanup(0);
 	}
 
+	if (do_compression || def_compress_level != NOT_SPECIFIED) {
+		if (def_compress_level == NOT_SPECIFIED)
+			def_compress_level = Z_DEFAULT_COMPRESSION;
+		else if (def_compress_level < Z_DEFAULT_COMPRESSION || def_compress_level > Z_BEST_COMPRESSION) {
+			snprintf(err_buf, sizeof err_buf, "--compress-level value is invalid: %d\n",
+				 def_compress_level);
+			return 0;
+		} else if (def_compress_level == Z_NO_COMPRESSION)
+			do_compression = 0;
+		else if (!do_compression)
+			do_compression = 1;
+		if (do_compression && refused_compress) {
+			create_refuse_error(refused_compress);
+			return 0;
+		}
+#ifdef EXTERNAL_ZLIB
+		if (do_compression == 1) {
+			snprintf(err_buf, sizeof err_buf,
+				"This rsync lacks old-style --compress due to its external zlib.  Try -zz.\n");
+			if (am_server)
+				return 0;
+			fprintf(stderr, "%s" "Continuing without compression.\n\n", err_buf);
+			do_compression = 0;
+		}
+#endif
+	}
+
 #ifdef HAVE_SETVBUF
 	if (outbuf_mode && !am_server) {
 		int mode = *(uchar *)outbuf_mode;
@@ -2451,7 +2471,7 @@ void server_options(char **args, int *argc_p)
 	}
 	if (sparse_files)
 		argstr[x++] = 'S';
-	if (do_compression)
+	if (do_compression == 1)
 		argstr[x++] = 'z';
 
 	set_allow_inc_recurse();
@@ -2747,6 +2767,9 @@ void server_options(char **args, int *argc_p)
 		exit_cleanup(RERR_MALLOC);
 	}
 
+	if (do_compression > 1)
+		args[ac++] = "--new-compress";
+
 	if (remote_option_cnt) {
 		int j;
 		if (ac + remote_option_cnt > MAX_SERVER_ARGS) {
diff --git a/rsync.yo b/rsync.yo
index eafa3854..c99d150c 100644
--- a/rsync.yo
+++ b/rsync.yo
@@ -1867,7 +1867,20 @@ being transmitted -- something that is useful over a slow connection.
 Note that this option typically achieves better compression ratios than can
 be achieved by using a compressing remote shell or a compressing transport
 because it takes advantage of the implicit information in the matching data
-blocks that are not explicitly sent over the connection.
+blocks that are not explicitly sent over the connection.  This matching-data
+compression comes at a cost of CPU, though, and can be disabled by repeating
+the bf(-z) option, but only if both sides are at least version 3.1.1.
+
+Note that if your version of rsync was compiled with an external zlib (instead
+of the zlib that comes packaged with rsync) then it will not support the
+old-style compression, only the new-style (repeated-option) compression.  In
+the future this new-style compression will likely become the default.
+
+The client rsync requests new-style compression on the server via the
+bf(--new-compress) option, so if you see that option rejected it means that
+the server is not new enough to support bf(-zz).  Rsync also accepts the
+bf(--old-compress) option for a future time when new-style compression
+becomes the default.
 
 See the bf(--skip-compress) option for the default list of file suffixes
 that will not be compressed.
diff --git a/testsuite/daemon-gzip-download.test b/testsuite/daemon-gzip-download.test
index 89a112ff..57dd820b 100644
--- a/testsuite/daemon-gzip-download.test
+++ b/testsuite/daemon-gzip-download.test
@@ -31,7 +31,7 @@ hands_setup
 # Build chkdir with a normal rsync and an --exclude.
 $RSYNC -av --exclude=foobar.baz "$fromdir/" "$chkdir/"
 
-checkit "$RSYNC -avvvvz localhost::test-from/ '$todir/'" "$chkdir" "$todir"
+checkit "$RSYNC -avvvvzz localhost::test-from/ '$todir/'" "$chkdir" "$todir"
 
 # The script would have aborted on error, so getting here means we've won.
 exit 0
diff --git a/testsuite/daemon-gzip-upload.test b/testsuite/daemon-gzip-upload.test
index 11c52bad..b2110ea6 100644
--- a/testsuite/daemon-gzip-upload.test
+++ b/testsuite/daemon-gzip-upload.test
@@ -25,7 +25,7 @@ hands_setup
 # Build chkdir with a normal rsync and an --exclude.
 $RSYNC -av --exclude=foobar.baz "$fromdir/" "$chkdir/"
 
-checkit "'$ignore23' $RSYNC -avvvvz '$fromdir/' localhost::test-to/" "$chkdir" "$todir"
+checkit "'$ignore23' $RSYNC -avvvvzz '$fromdir/' localhost::test-to/" "$chkdir" "$todir"
 
 # The script would have aborted on error, so getting here means we've won.
 exit 0
diff --git a/token.c b/token.c
index bd4d52fe..8cc55329 100644
--- a/token.c
+++ b/token.c
@@ -23,10 +23,6 @@
 #include "itypes.h"
 #include <zlib.h>
 
-#ifndef Z_INSERT_ONLY
-#define Z_INSERT_ONLY Z_SYNC_FLUSH
-#endif
-
 extern int do_compression;
 extern int protocol_version;
 extern int module_id;
@@ -406,9 +402,10 @@ send_deflated_token(int f, int32 token, struct map_struct *buf, OFF_T offset,
 	if (token == -1) {
 		/* end of file - clean up */
 		write_byte(f, END_FLAG);
-	} else if (token != -2) {
+	} else if (token != -2 && do_compression == 1) {
 		/* Add the data in the current block to the compressor's
 		 * history and hash table. */
+#ifndef EXTERNAL_ZLIB
 		do {
 			/* Break up long sections in the same way that
 			 * see_deflate_token() does. */
@@ -418,17 +415,20 @@ send_deflated_token(int f, int32 token, struct map_struct *buf, OFF_T offset,
 			tx_strm.avail_in = n1;
 			if (protocol_version >= 31) /* Newer protocols avoid a data-duplicating bug */
 				offset += n1;
-			do {
-				tx_strm.next_out = (Bytef *) obuf;
-				tx_strm.avail_out = AVAIL_OUT_SIZE(CHUNK_SIZE);
-				r = deflate(&tx_strm, Z_INSERT_ONLY);
-				if (r != Z_OK) {
-					rprintf(FERROR, "deflate on token returned %d (%d bytes left)\n",
-						r, tx_strm.avail_in);
-					exit_cleanup(RERR_STREAMIO);
-				}
-			} while (tx_strm.avail_in != 0);
+			tx_strm.next_out = (Bytef *) obuf;
+			tx_strm.avail_out = AVAIL_OUT_SIZE(CHUNK_SIZE);
+			r = deflate(&tx_strm, Z_INSERT_ONLY);
+			if (r != Z_OK || tx_strm.avail_in != 0) {
+				rprintf(FERROR, "deflate on token returned %d (%d bytes left)\n",
+					r, tx_strm.avail_in);
+				exit_cleanup(RERR_STREAMIO);
+			}
 		} while (toklen > 0);
+#else
+		toklen++;
+		rprintf(FERROR, "Impossible error in external-zlib code (1).\n");
+		exit_cleanup(RERR_STREAMIO);
+#endif
 	}
 }
 
@@ -579,6 +579,7 @@ static int32 recv_deflated_token(int f, char **data)
  */
 static void see_deflate_token(char *buf, int32 len)
 {
+#ifndef EXTERNAL_ZLIB
 	int r;
 	int32 blklen;
 	unsigned char hdr[5];
@@ -616,6 +617,11 @@ static void see_deflate_token(char *buf, int32 len)
 			exit_cleanup(RERR_STREAMIO);
 		}
 	} while (len || rx_strm.avail_out == 0);
+#else
+	buf++; len++;
+	rprintf(FERROR, "Impossible error in external-zlib code (2).\n");
+	exit_cleanup(RERR_STREAMIO);
+#endif
 }
 
 /**
@@ -655,6 +661,6 @@ int32 recv_token(int f, char **data)
  */
 void see_token(char *data, int32 toklen)
 {
-	if (do_compression)
+	if (do_compression == 1)
 		see_deflate_token(data, toklen);
 }