Merge branch 'danielsilverstone/all-repos-bare' of git://git.baserock.org/baserock/lorry

Includes a fixup of quoting uses of $SRCDIR, so it may include
paths with spaces.

19 files changed, 409 insertions, 88 deletions

diff --git a/README b/README
index 45b803a..a1355a4 100644
--- a/README
+++ b/README
@@ -5,9 +5,9 @@ Lorry is a tool to take upstream source code (in various formats,
 though preferably in version control) and converts it into a git
 repository.
 
-If you want to try this, use `--pull-only` and/or `--gitorious-base-url`
+If you want to try this, use `--pull-only` and/or `--mirror-base-url-push`
 so that you do not accidentally overwrite important stuff for Baserock.
-(If you don't have direct commit access to Baserock on Gitorious.org,
+(If you don't have direct commit access to Baserock on git.baserock.org
 then you're not dangerous.)
 
 See the manual page for instructions on using.
@@ -25,8 +25,8 @@ You can find a lot of lorries to crib ideas from at:
 Implementation
 --------------
 
-Lorry relies on git-svn, git-cvsimport, and bzr fast-export for the
-conversions. You need to have them installed.
+Lorry relies on git-svn, git-cvsimport, hg-fast-export, perl (for tarballs) and
+bzr fast-export for the conversions. You need to have them installed.
 
 Lorry file specification
 ------------------------
@@ -193,12 +193,14 @@ often have the folder name as the first component.
     }
 }
 
-NOTE: tarball imports are unlikely to give the same sha.
+NOTE: tarball imports are unlikely to give the same commit SHA1 but the tree
+SHA1 inside (which is what is used for artifact cache IDs) should remain
+stable.
 
 Legal stuff
 -----------
 
-Copyright (C) 2011  Codethink Limited
+Copyright (C) 2011, 2012  Codethink Limited
  
 This program is free software; you can redistribute it and/or modify
 it under the terms of the GNU General Public License as published by
diff --git a/lorry b/lorry
index 29df6dd..5fea080 100755
--- a/lorry
+++ b/lorry
@@ -80,6 +80,13 @@ class Lorry(cliapp.Application):
         self.settings.string(['bundle-dest'],
                              'put created bundles in BUNDLES',
                              metavar='BUNDLES')
+        self.settings.choice(['tarball'], ['first', 'never', 'always'],
+                             'create tarballs of git repositories.'
+                             'first will only tar if there is not already '
+                             'a tarball in TARBALLS (default: first)')
+        self.settings.string(['tarball-dest'],
+                             'put created tarballs in TARBALLS',
+                             metavar='TARBALLS')
 
     def process_args(self, args):
         status = 0
@@ -133,6 +140,36 @@ class Lorry(cliapp.Application):
             expr = '1,/^[0-9a-f]\{40\}/{ /^[0-9a-f]\{40\}/!{/^[^#]/d}}'
             self.run_program(['sed', '-i', '-e', expr, path], cwd=gitdir)
 
+    def make_tarball(self, name, gitdir):
+        if self.settings['tarball'] == 'never': return
+        tarballname = "%s/%s" % (self.settings['mirror-base-url-fetch'],
+                                 name)
+        path = os.path.join(self.settings['tarball-dest'],
+                            quote_url(tarballname)) + '.tar'
+        if os.path.exists(os.path.join(gitdir, '.git')):
+            gitdir = os.path.join(gitdir, '.git')
+        if not os.path.exists(path) or self.settings['tarball'] == 'always':
+            self.progress('.. building tarball %s' % tarballname)
+            args = ['tar', 'cf', path]
+            if os.path.exists(os.path.join(gitdir, 'config')):
+                os.rename(os.path.join(gitdir, 'config'),
+                          os.path.join(gitdir, 'config.lorrytmp'))
+                with open(os.path.join(gitdir, 'config'), 'w') as fh:
+                    fh.write("""[core]
+	repositoryformatversion = 0
+	filemode = true
+	bare = true
+""")
+            for entry in ['HEAD', 'objects', 'refs',
+                          'info', 'packed-refs', 'config', 'branches',
+                          'description']:
+                if os.path.exists(os.path.join(gitdir, entry)):
+                    args += [entry]
+            self.run_program(args, cwd=gitdir)
+            if os.path.exists(os.path.join(gitdir, 'config.lorrytmp')):
+                os.rename(os.path.join(gitdir, 'config.lorrytmp'),
+                          os.path.join(gitdir, 'config'))
+
     def gitify(self, name, spec):
         self.progress('Getting %s' % name)
         table = {
@@ -152,12 +189,19 @@ class Lorry(cliapp.Application):
         gitdir = os.path.join(dirname, 'git')
         backupdir = self.backup_gitdir(name, dirname, gitdir)
         try:
+            self.needs_aggressive = False
             table[vcstype](name, dirname, gitdir, spec)
             if self.settings['repack']:
                 self.progress('.. repacking %s git repository' % name)
-                self.run_program(['git', 'repack', '-a', '-d', '--depth=250',
-                                  '--window=250'], cwd=gitdir)
+                self.run_program(['git', 'config',
+                                  'pack.windowMemory', '128M'], cwd=gitdir)
+                args = ['git', 'gc']
+                if self.needs_aggressive:
+                    args += ['--aggressive']
+                self.run_program(args, cwd=gitdir)
+
             self.bundle(name, gitdir)
+            self.make_tarball(name, gitdir)
         except:
             if backupdir is not None:
                 faildir = self.save_failgit(name, dirname, gitdir)
@@ -238,13 +282,12 @@ class Lorry(cliapp.Application):
 
     def mirror_git(self, project_name, dirname, gitdir, spec):
         if not os.path.exists(gitdir):
-            self.progress('.. doing initial clone')
-            self.run_program(['git', 'clone', '--mirror', spec['url'], gitdir])
-        else:
-            self.progress('.. updating existing clone')
-            self.run_program(['git', 'fetch', spec['url'],
-                              '+refs/heads/*:refs/heads/*',
-                              '+refs/tags/*:refs/tags/*'], cwd=gitdir)
+            self.progress('.. initialising git dir')
+            self.run_program(['git', 'init', '--bare', gitdir])
+        self.progress('.. updating existing clone')
+        self.run_program(['git', 'fetch', spec['url'],
+                          '+refs/heads/*:refs/heads/*',
+                          '+refs/tags/*:refs/tags/*'], cwd=gitdir)
 
     def gitify_bzr(self, project_name, dirname, gitdir, spec):
         bzrdir = os.path.join(dirname, 'bzr')
@@ -256,7 +299,8 @@ class Lorry(cliapp.Application):
         if not os.path.exists(gitdir):
             self.progress('.. creating git repo')
             os.mkdir(gitdir)
-            self.run_program(['git', 'init', gitdir])
+            self.run_program(['git', 'init', '--bare', gitdir])
+            self.needs_aggressive = True
 
         # branches are the listed branches, plus the branch specified in url
         if 'branches' in spec:
@@ -279,7 +323,7 @@ class Lorry(cliapp.Application):
                                  cwd=branchdir)
 
         exports = {}
-        bzrmarks = os.path.join(gitdir, '.git', 'marks.bzr')
+        bzrmarks = os.path.join(gitdir, 'marks.bzr')
         for branch, address in branches.iteritems():
             branchdir = os.path.join(bzrdir, branch)
             self.progress('.. fast-exporting branch %s from bzr' % branch)
@@ -292,7 +336,7 @@ class Lorry(cliapp.Application):
                 cmdline.append('--export-marks=' + bzrmarks)
             self.run_program(cmdline)
 
-        gitmarks = os.path.join(gitdir, '.git', 'marks.git')
+        gitmarks = os.path.join(gitdir, 'marks.git')
         for branch, address in branches.iteritems():
             self.progress('.. fast-importing branch %s into git' % branch)
             with open(exports[branch], 'rb') as exportfile:
@@ -311,7 +355,7 @@ class Lorry(cliapp.Application):
     def gitify_svn(self, project_name, dirname, gitdir, spec):
         if not os.path.exists(gitdir):
             self.progress('.. doing initial clone')
-            os.mkdir(gitdir)
+            self.needs_aggressive = True
             layout = spec["layout"]
             # if standard layour specified, fill in the defaults
             if layout == "standard":
@@ -323,8 +367,12 @@ class Lorry(cliapp.Application):
             # fetching the root of the repository
             # git-svn will convert branch, trunk and tag paths to allow this,
             # but it is simpler to disable it and do it manually
-            self.run_program(['git', 'svn', 'init', spec['url'], gitdir,
+            self.run_program(['git', 'svn', 'init', spec['url'], gitdir + "-tmp",
                               '--svn-remote=svn', '--no-minimize-url'])
+            os.rename(os.path.join(gitdir + "-tmp", '.git'), gitdir)
+            os.rmdir(gitdir + "-tmp")
+            self.run_program(['git', 'config', 'core.bare', 'true'],
+                             cwd=gitdir)
             self.run_program(['git', 'config', 'svn-remote.svn.fetch',
                               layout["trunk"]+':refs/heads/master'], 
                              cwd=gitdir)
@@ -340,6 +388,7 @@ class Lorry(cliapp.Application):
         self.run_program(['git', 'svn', 'fetch'], cwd=gitdir)
 
     def gitify_cvs(self, project_name, dirname, gitdir, spec):
+        self.needs_aggressive = True
         self.run_program(['git', 'cvsimport', '-a', '-d', spec['url'],
                           '-C', gitdir, spec['module']])
 
@@ -353,37 +402,32 @@ class Lorry(cliapp.Application):
             self.run_program(['hg', 'clone', '--quiet', spec['url'], hgdir])
 
         if not os.path.exists(gitdir):
-            self.run_program(['git', 'init', gitdir])
+            self.needs_aggressive = True
+            self.run_program(['git', 'init', '--bare', gitdir])
         
         self.progress('.. fast-exporting into git')
         self.run_program(['hg-fast-export', '--quiet', '-r', '../hg'], 
                          cwd=gitdir)
 
     def gitify_tarball(self, project_name, dirname, gitdir, spec):
-        tardest = os.path.join(dirname, 'tarball')
+        url = spec['url']
+        url_path = urllib2.urlparse.urlparse(url)[2]
+        basename = os.path.basename(url_path)
+        tardest = os.path.join(dirname, basename)
+        self.progress('.. checking if we need to fetch %s' % basename)
         if not os.path.exists(tardest):
+            self.progress('.. attempting to fetch.')
             with open(tardest, 'w') as tarfile:
                 urlfile = urllib2.urlopen(spec['url'])
                 tarfile.write(urlfile.read())
                 urlfile.close()
-
+        else:
+            self.progress('.. no need to run, nothing to do')
         if not os.path.exists(gitdir):
-            self.run_program(['git', 'init', gitdir])
-        cmdline = ['tar', '--extract', '--file', tardest]
-        # compression is handled in long form, so use gzip instead of z
-        try:
-            cmdline += ['--' + spec['compression']]
-        except KeyError:
-            pass
-        # tarballs often have a directory on top, strip = 1 will remove it
-        try:
-            cmdline += ['--strip-components', str(spec['strip'])]
-        except KeyError:
-            pass
+            self.run_program(['git', 'init', '--bare', gitdir])
+        cmdline = ["%s.tar-importer" % __file__, tardest]
         self.run_program(cmdline, cwd=gitdir)
-        self.run_program(['git', 'add', '.'], cwd=gitdir)
-        self.run_program(['git', 'commit', '-m', 'Tarball conversion'],
-                         cwd=gitdir)
+        self.needs_aggressive = True
             
 
     def push_to_mirror_server(self, name, gitdir,
diff --git a/lorry.tar-importer b/lorry.tar-importer
new file mode 100755
index 0000000..eb17ef2
--- /dev/null
+++ b/lorry.tar-importer
@@ -0,0 +1,205 @@
+#!/usr/bin/perl
+
+## Note: Modified for Baserock lorry.
+
+
+## tar archive frontend for git-fast-import
+##
+## For example:
+##
+##  mkdir project; cd project; git init
+##  perl import-tars.perl *.tar.bz2
+##  git whatchanged import-tars
+##
+## Use --metainfo to specify the extension for a meta data file, where
+## import-tars can read the commit message and optionally author and
+## committer information.
+##
+##  echo 'This is the commit message' > myfile.tar.bz2.msg
+##  perl import-tars.perl --metainfo=msg myfile.tar.bz2
+
+use strict;
+use Getopt::Long;
+
+my $metaext = '';
+
+die "usage: import-tars [--metainfo=extension] *.tar.{gz,bz2,lzma,xz,Z}\n"
+	unless GetOptions('metainfo=s' => \$metaext) && @ARGV;
+
+my $branch_name = 'master';
+my $branch_ref = "refs/heads/$branch_name";
+my $old_sha = `git show-ref $branch_ref 2>/dev/null`;
+my $author_name = $ENV{'GIT_AUTHOR_NAME'} || 'Lorry Tar Creator';
+my $author_email = $ENV{'GIT_AUTHOR_EMAIL'} || 'lorry-tar-importer@baserock.org';
+my $committer_name = $ENV{'GIT_COMMITTER_NAME'} || `git config --get user.name`;
+my $committer_email = $ENV{'GIT_COMMITTER_EMAIL'} || `git config --get user.email`;
+
+chomp($committer_name, $committer_email, $old_sha);
+
+if ($old_sha ne '') {
+   $old_sha = ($old_sha =~ /^([a-f0-9]+)/)[0];
+}
+open(FI, '|-', 'git', 'fast-import', '--quiet')
+	or die "Unable to start git fast-import: $!\n";
+foreach my $tar_file (@ARGV)
+{
+	my $commit_time = time;
+	$tar_file =~ m,([^/]+)$,;
+	my $tar_name = $1;
+
+	if ($tar_name =~ s/\.(tar\.gz|tgz)$//) {
+		open(I, '-|', 'gunzip', '-c', $tar_file)
+			or die "Unable to gunzip -c $tar_file: $!\n";
+	} elsif ($tar_name =~ s/\.(tar\.bz2|tbz2)$//) {
+		open(I, '-|', 'bunzip2', '-c', $tar_file)
+			or die "Unable to bunzip2 -c $tar_file: $!\n";
+	} elsif ($tar_name =~ s/\.tar\.Z$//) {
+		open(I, '-|', 'uncompress', '-c', $tar_file)
+			or die "Unable to uncompress -c $tar_file: $!\n";
+	} elsif ($tar_name =~ s/\.(tar\.(lzma|xz)|(tlz|txz))$//) {
+		open(I, '-|', 'xz', '-dc', $tar_file)
+			or die "Unable to xz -dc $tar_file: $!\n";
+	} elsif ($tar_name =~ s/\.tar$//) {
+		open(I, $tar_file) or die "Unable to open $tar_file: $!\n";
+	} else {
+		die "Unrecognized compression format: $tar_file\n";
+	}
+
+	my $author_time = 0;
+	my $next_mark = 1;
+	my $have_top_dir = 1;
+	my ($top_dir, %files);
+
+	while (read(I, $_, 512) == 512) {
+		my ($name, $mode, $uid, $gid, $size, $mtime,
+			$chksum, $typeflag, $linkname, $magic,
+			$version, $uname, $gname, $devmajor, $devminor,
+			$prefix) = unpack 'Z100 Z8 Z8 Z8 Z12 Z12
+			Z8 Z1 Z100 Z6
+			Z2 Z32 Z32 Z8 Z8 Z*', $_;
+		last unless length($name);
+		if ($name eq '././@LongLink') {
+			# GNU tar extension
+			if (read(I, $_, 512) != 512) {
+				die ('Short archive');
+			}
+			$name = unpack 'Z257', $_;
+			next unless $name;
+
+			my $dummy;
+			if (read(I, $_, 512) != 512) {
+				die ('Short archive');
+			}
+			($dummy, $mode, $uid, $gid, $size, $mtime,
+			$chksum, $typeflag, $linkname, $magic,
+			$version, $uname, $gname, $devmajor, $devminor,
+			$prefix) = unpack 'Z100 Z8 Z8 Z8 Z12 Z12
+			Z8 Z1 Z100 Z6
+			Z2 Z32 Z32 Z8 Z8 Z*', $_;
+		}
+		next if $name =~ m{/\z};
+		$mode = oct $mode;
+		$size = oct $size;
+		$mtime = oct $mtime;
+		next if $typeflag == 5; # directory
+
+		print FI "blob\n", "mark :$next_mark\n";
+		if ($typeflag == 2) { # symbolic link
+			print FI "data ", length($linkname), "\n", $linkname;
+			$mode = 0120000;
+		} else {
+			print FI "data $size\n";
+			while ($size > 0 && read(I, $_, 512) == 512) {
+				print FI substr($_, 0, $size);
+				$size -= 512;
+			}
+		}
+		print FI "\n";
+
+		my $path;
+		if ($prefix) {
+			$path = "$prefix/$name";
+		} else {
+			$path = "$name";
+		}
+		$files{$path} = [$next_mark++, $mode];
+
+		$author_time = $mtime if $mtime > $author_time;
+		$path =~ m,^([^/]+)/,;
+		$top_dir = $1 unless $top_dir;
+		$have_top_dir = 0 if $top_dir ne $1;
+	}
+
+	my $commit_msg = "Imported from $tar_file.";
+	my $this_committer_name = $committer_name;
+	my $this_committer_email = $committer_email;
+	my $this_author_name = $author_name;
+	my $this_author_email = $author_email;
+	if ($metaext ne '') {
+		# Optionally read a commit message from <filename.tar>.msg
+		# Add a line on the form "Committer: name <e-mail>" to override
+		# the committer and "Author: name <e-mail>" to override the
+		# author for this tar ball.
+		if (open MSG, '<', "${tar_file}.${metaext}") {
+			my $header_done = 0;
+			$commit_msg = '';
+			while (<MSG>) {
+				if (!$header_done && /^Committer:\s+([^<>]*)\s+<(.*)>\s*$/i) {
+					$this_committer_name = $1;
+					$this_committer_email = $2;
+				} elsif (!$header_done && /^Author:\s+([^<>]*)\s+<(.*)>\s*$/i) {
+					$this_author_name = $1;
+					$this_author_email = $2;
+				} elsif (!$header_done && /^$/) { # empty line ends header.
+					$header_done = 1;
+				} else {
+					$commit_msg .= $_;
+					$header_done = 1;
+				}
+			}
+			close MSG;
+		}
+	}
+
+	print FI <<EOF;
+commit $branch_ref
+author $this_author_name <$this_author_email> $author_time +0000
+committer $this_committer_name <$this_committer_email> $commit_time +0000
+data <<END_OF_COMMIT_MESSAGE
+$commit_msg
+END_OF_COMMIT_MESSAGE
+EOF
+
+        if ($old_sha ne '') {
+           print FI <<EOF;
+from $old_sha
+EOF
+        }
+
+        print FI <<EOF;
+deleteall
+EOF
+
+	foreach my $path (keys %files)
+	{
+		my ($mark, $mode) = @{$files{$path}};
+		$path =~ s,^([^/]+)/,, if $have_top_dir;
+		$mode = $mode & 0111 ? 0755 : 0644 unless $mode == 0120000;
+		printf FI "M %o :%i %s\n", $mode, $mark, $path;
+	}
+	print FI "\n";
+
+
+	print FI <<EOF;
+tag $tar_name
+from $branch_ref
+tagger $author_name <$author_email> $author_time +0000
+data <<END_OF_TAG_MESSAGE
+Package $tar_name
+END_OF_TAG_MESSAGE
+
+EOF
+
+	close I;
+}
+close FI;
diff --git a/setup.py b/setup.py
index cc5bde5..63bd431 100644
--- a/setup.py
+++ b/setup.py
@@ -26,8 +26,6 @@ import os
 import shutil
 import subprocess
 
-import morphlib
-
 
 class GenerateManpage(build):
 
@@ -46,6 +44,7 @@ class Clean(clean):
     clean_files = [
         '.coverage',
         'build',
+        'lorry.1'
     ]
     clean_globs = [
         '*/*.py[co]',
@@ -86,7 +85,7 @@ FIXME
       author='Baserock',
       author_email='baserock-dev@baserock.org',
       url='http://wiki.baserock.org/',
-      scripts=['lorry'],
+      scripts=['lorry', 'lorry.tar-importer'],
       data_files=[('share/man/man1', glob.glob('*.[1-8]'))],
       cmdclass={
           'build': GenerateManpage,
diff --git a/test-lorry b/test-lorry
new file mode 100755
index 0000000..67f051d
--- /dev/null
+++ b/test-lorry
@@ -0,0 +1,3 @@
+#!/bin/sh
+
+exec "${SRCDIR}/lorry" --no-default-configs --tarball=never "$@"
diff --git a/tests/bzr-single-commit.script b/tests/bzr-single-commit.script
index a870558..9b38552 100755
--- a/tests/bzr-single-commit.script
+++ b/tests/bzr-single-commit.script
@@ -24,7 +24,7 @@ set -e
 logfile="$DATADIR/bzr-test-repo.log"
 workdir="$DATADIR/work-dir"
 
-./lorry --pull-only --log="$logfile" --working-area="$workdir" \
+"${SRCDIR}/test-lorry" --pull-only --log="$logfile" --working-area="$workdir" \
   "$DATADIR/bzr-test-repo.lorry" > /dev/null 2> /dev/null
 
 # verify that the git repository was set up correctly
diff --git a/tests/cvs-single-commit.script b/tests/cvs-single-commit.script
index ad4eca8..4ce2587 100755
--- a/tests/cvs-single-commit.script
+++ b/tests/cvs-single-commit.script
@@ -28,7 +28,7 @@ export USER=root
 export LOGNAME=$USER
 export USERNAME=$USER
 
-./lorry --pull-only --log="$logfile" --working-area="$workdir" \
+"${SRCDIR}/test-lorry" --pull-only --log="$logfile" --working-area="$workdir" \
   "$DATADIR/cvs-test-repo.lorry" > /dev/null 2> /dev/null
 
 # verify that the git repository was created successfully
diff --git a/tests/git-backup-on-error.script b/tests/git-backup-on-error.script
index 134367c..fbb3368 100755
--- a/tests/git-backup-on-error.script
+++ b/tests/git-backup-on-error.script
@@ -30,16 +30,17 @@ normalize() {
     DATETIMESPEC='[0-9]*-[0-9]*-[0-9]*-[0-9]*:[0-9]*:[0-9]*'
     sed -r -e "s|git-pre-update-$DATETIMESPEC|git-pre-update-DATETIME|g" \
            -e "s|git-post-fail-$DATETIMESPEC|git-post-fail-DATETIME|g" \
+           -e '/hooks\/.*\.sample/d' \
            -e "s|$DATADIR|DATADIR|g" "$@"
 }
 
 # mirror some history
-./lorry --pull-only --log="$logfile" --working-area="$workdir" --bundle=never \
+"${SRCDIR}/test-lorry" --pull-only --log="$logfile" --working-area="$workdir" --bundle=never \
   "$DATADIR/git-backup-test-repo.lorry" | normalize 
 
 # make upstream disappear to cause errors
 rm -rf "$repo"
-if ./lorry --pull-only --log="$logfile" --working-area="$workdir" \
+if "${SRCDIR}/test-lorry" --pull-only --log="$logfile" --working-area="$workdir" \
            "$DATADIR/git-backup-test-repo.lorry" --bundle=never 2>/dev/null | \
            normalize
 then
diff --git a/tests/git-backup-on-error.stdout b/tests/git-backup-on-error.stdout
index 195970e..b284701 100644
--- a/tests/git-backup-on-error.stdout
+++ b/tests/git-backup-on-error.stdout
@@ -8,16 +8,6 @@ DATADIR/work-dir/git-backup-test-repo/git-post-fail-DATETIME/branches
 DATADIR/work-dir/git-backup-test-repo/git-post-fail-DATETIME/config
 DATADIR/work-dir/git-backup-test-repo/git-post-fail-DATETIME/description
 DATADIR/work-dir/git-backup-test-repo/git-post-fail-DATETIME/hooks
-DATADIR/work-dir/git-backup-test-repo/git-post-fail-DATETIME/hooks/applypatch-msg.sample
-DATADIR/work-dir/git-backup-test-repo/git-post-fail-DATETIME/hooks/commit-msg.sample
-DATADIR/work-dir/git-backup-test-repo/git-post-fail-DATETIME/hooks/post-commit.sample
-DATADIR/work-dir/git-backup-test-repo/git-post-fail-DATETIME/hooks/post-receive.sample
-DATADIR/work-dir/git-backup-test-repo/git-post-fail-DATETIME/hooks/post-update.sample
-DATADIR/work-dir/git-backup-test-repo/git-post-fail-DATETIME/hooks/pre-applypatch.sample
-DATADIR/work-dir/git-backup-test-repo/git-post-fail-DATETIME/hooks/pre-commit.sample
-DATADIR/work-dir/git-backup-test-repo/git-post-fail-DATETIME/hooks/pre-rebase.sample
-DATADIR/work-dir/git-backup-test-repo/git-post-fail-DATETIME/hooks/prepare-commit-msg.sample
-DATADIR/work-dir/git-backup-test-repo/git-post-fail-DATETIME/hooks/update.sample
 DATADIR/work-dir/git-backup-test-repo/git-post-fail-DATETIME/info
 DATADIR/work-dir/git-backup-test-repo/git-post-fail-DATETIME/info/exclude
 DATADIR/work-dir/git-backup-test-repo/git-post-fail-DATETIME/info/refs
@@ -27,21 +17,12 @@ DATADIR/work-dir/git-backup-test-repo/git-post-fail-DATETIME/refs
 DATADIR/work-dir/git-backup-test-repo/git-post-fail-DATETIME/refs/heads
 DATADIR/work-dir/git-backup-test-repo/git-post-fail-DATETIME/refs/tags
 DATADIR/work-dir/git-backup-test-repo/git-pre-update-DATETIME
+DATADIR/work-dir/git-backup-test-repo/git-pre-update-DATETIME/FETCH_HEAD
 DATADIR/work-dir/git-backup-test-repo/git-pre-update-DATETIME/HEAD
 DATADIR/work-dir/git-backup-test-repo/git-pre-update-DATETIME/branches
 DATADIR/work-dir/git-backup-test-repo/git-pre-update-DATETIME/config
 DATADIR/work-dir/git-backup-test-repo/git-pre-update-DATETIME/description
 DATADIR/work-dir/git-backup-test-repo/git-pre-update-DATETIME/hooks
-DATADIR/work-dir/git-backup-test-repo/git-pre-update-DATETIME/hooks/applypatch-msg.sample
-DATADIR/work-dir/git-backup-test-repo/git-pre-update-DATETIME/hooks/commit-msg.sample
-DATADIR/work-dir/git-backup-test-repo/git-pre-update-DATETIME/hooks/post-commit.sample
-DATADIR/work-dir/git-backup-test-repo/git-pre-update-DATETIME/hooks/post-receive.sample
-DATADIR/work-dir/git-backup-test-repo/git-pre-update-DATETIME/hooks/post-update.sample
-DATADIR/work-dir/git-backup-test-repo/git-pre-update-DATETIME/hooks/pre-applypatch.sample
-DATADIR/work-dir/git-backup-test-repo/git-pre-update-DATETIME/hooks/pre-commit.sample
-DATADIR/work-dir/git-backup-test-repo/git-pre-update-DATETIME/hooks/pre-rebase.sample
-DATADIR/work-dir/git-backup-test-repo/git-pre-update-DATETIME/hooks/prepare-commit-msg.sample
-DATADIR/work-dir/git-backup-test-repo/git-pre-update-DATETIME/hooks/update.sample
 DATADIR/work-dir/git-backup-test-repo/git-pre-update-DATETIME/info
 DATADIR/work-dir/git-backup-test-repo/git-pre-update-DATETIME/info/exclude
 DATADIR/work-dir/git-backup-test-repo/git-pre-update-DATETIME/info/refs
@@ -50,21 +31,12 @@ DATADIR/work-dir/git-backup-test-repo/git-pre-update-DATETIME/packed-refs
 DATADIR/work-dir/git-backup-test-repo/git-pre-update-DATETIME/refs
 DATADIR/work-dir/git-backup-test-repo/git-pre-update-DATETIME/refs/heads
 DATADIR/work-dir/git-backup-test-repo/git-pre-update-DATETIME/refs/tags
+DATADIR/work-dir/git-backup-test-repo/git/FETCH_HEAD
 DATADIR/work-dir/git-backup-test-repo/git/HEAD
 DATADIR/work-dir/git-backup-test-repo/git/branches
 DATADIR/work-dir/git-backup-test-repo/git/config
 DATADIR/work-dir/git-backup-test-repo/git/description
 DATADIR/work-dir/git-backup-test-repo/git/hooks
-DATADIR/work-dir/git-backup-test-repo/git/hooks/applypatch-msg.sample
-DATADIR/work-dir/git-backup-test-repo/git/hooks/commit-msg.sample
-DATADIR/work-dir/git-backup-test-repo/git/hooks/post-commit.sample
-DATADIR/work-dir/git-backup-test-repo/git/hooks/post-receive.sample
-DATADIR/work-dir/git-backup-test-repo/git/hooks/post-update.sample
-DATADIR/work-dir/git-backup-test-repo/git/hooks/pre-applypatch.sample
-DATADIR/work-dir/git-backup-test-repo/git/hooks/pre-commit.sample
-DATADIR/work-dir/git-backup-test-repo/git/hooks/pre-rebase.sample
-DATADIR/work-dir/git-backup-test-repo/git/hooks/prepare-commit-msg.sample
-DATADIR/work-dir/git-backup-test-repo/git/hooks/update.sample
 DATADIR/work-dir/git-backup-test-repo/git/info
 DATADIR/work-dir/git-backup-test-repo/git/info/exclude
 DATADIR/work-dir/git-backup-test-repo/git/info/refs
diff --git a/tests/git-single-commit.script b/tests/git-single-commit.script
index 4a6e8b3..7a980b2 100755
--- a/tests/git-single-commit.script
+++ b/tests/git-single-commit.script
@@ -24,7 +24,7 @@ set -e
 logfile="$DATADIR/git-test-repo.log"
 workdir="$DATADIR/work-dir"
 
-./lorry --pull-only --log="$logfile" --working-area="$workdir" \
+"${SRCDIR}/test-lorry" --pull-only --log="$logfile" --working-area="$workdir" \
   "$DATADIR/git-test-repo.lorry" > /dev/null 2> /dev/null
 
 # verify that the git repository was set up correctly
diff --git a/tests/hg-single-commit.script b/tests/hg-single-commit.script
index c370190..05343fb 100755
--- a/tests/hg-single-commit.script
+++ b/tests/hg-single-commit.script
@@ -24,7 +24,7 @@ set -e
 logfile="$DATADIR/hg-test-repo.log"
 workdir="$DATADIR/work-dir"
 
-./lorry --verbose --pull-only --log="$logfile" --working-area="$workdir" \
+"${SRCDIR}/test-lorry" --verbose --pull-only --log="$logfile" --working-area="$workdir" \
   "$DATADIR/hg-test-repo.lorry" > /dev/null 2> /dev/null
 
 # verify that the git repository was created correctly
diff --git a/tests/make-tarball.script b/tests/make-tarball.script
new file mode 100755
index 0000000..95eacaa
--- /dev/null
+++ b/tests/make-tarball.script
@@ -0,0 +1,32 @@
+#!/bin/sh
+#
+# Test that we can create a tarball of the git trees.
+#
+# Copyright (C) 2012  Codethink Limited
+# 
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation; version 2 of the License.
+# 
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+# 
+# You should have received a copy of the GNU General Public License along
+# with this program; if not, write to the Free Software Foundation, Inc.,
+# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
+
+
+set -e
+
+logfile="$DATADIR/make-tarball.log"
+workdir="$DATADIR/work-dir"
+
+"${SRCDIR}/test-lorry" --pull-only --log="$logfile" --working-area="$workdir" \
+  --tarball=first \
+  "$DATADIR/make-tarball-repo.lorry" > /dev/null 2> /dev/null
+
+# verify that we can see the tarball generated of the git tree
+
+test -r "${workdir}/make-tarball-repo-bzip2/git/"*"make_tarball_repo_bzip2.tar"
diff --git a/tests/make-tarball.setup b/tests/make-tarball.setup
new file mode 100755
index 0000000..2032610
--- /dev/null
+++ b/tests/make-tarball.setup
@@ -0,0 +1,59 @@
+#!/bin/sh
+#
+# Creates gzip/bzip2/lzma tarballs, each with a single file.
+#
+# Copyright (C) 2012  Codethink Limited
+# 
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation; version 2 of the License.
+# 
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+# 
+# You should have received a copy of the GNU General Public License along
+# with this program; if not, write to the Free Software Foundation, Inc.,
+# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
+
+
+set -e
+
+# create the original "repository"
+repo="$DATADIR/make-tarball-repo"
+mkdir "$repo"
+echo "first line" > "$repo/test.txt"
+
+# create the tarballs
+cd "$DATADIR"
+tar -czf make-tarball-repo.tar.gz "`basename $repo`"
+tar -cjf make-tarball-repo.tar.bz2 "`basename $repo`"
+tar -cf make-tarball-repo.tar.lzma "`basename $repo`" --lzma
+
+# create the .lorry file for the tarball "repositories"
+cat <<EOF > $DATADIR/make-tarball-repo.lorry
+{
+  "make-tarball-repo-gzip": {
+    "type": "tarball",
+    "compression": "gzip",
+    "strip": 1,
+    "url": "file://$DATADIR/make-tarball-repo.tar.gz"
+  },
+  "make-tarball-repo-bzip2": {
+    "type": "tarball",
+    "compression": "bzip2",
+    "strip": 1,
+    "url": "file://$DATADIR/make-tarball-repo.tar.bz2"
+  },
+  "make-tarball-repo-lzma": {
+    "type": "tarball",
+    "compression": "lzma",
+    "strip": 1,
+    "url": "file://$DATADIR/make-tarball-repo.tar.lzma"
+  }
+}
+EOF
+
+# create the working directory
+test -d "$DATADIR/work-dir" || mkdir "$DATADIR/work-dir"
diff --git a/tests/no-pushspec-pushall.script b/tests/no-pushspec-pushall.script
index cc56d8a..a649122 100755
--- a/tests/no-pushspec-pushall.script
+++ b/tests/no-pushspec-pushall.script
@@ -37,7 +37,7 @@ mirror_path="$DATADIR"/git-mirror
 mkdir -p "$mirror_path"
 git init --quiet --bare "$mirror_path"/no-pushspec.git
 
-./lorry --log="$logfile" --working-area="$workdir" \
+"${SRCDIR}/test-lorry" --log="$logfile" --working-area="$workdir" \
         --mirror-base-url-push=file://"$mirror_path" \
         --mirror-base-url-fetch=file://"$mirror_path" \
         "$lorryfile"
diff --git a/tests/pushspecs-only.script b/tests/pushspecs-only.script
index 787cc38..80aa8de 100755
--- a/tests/pushspecs-only.script
+++ b/tests/pushspecs-only.script
@@ -41,7 +41,7 @@ mirror_path="$DATADIR"/git-mirror
 mkdir -p "$mirror_path"
 git init --quiet --bare "$mirror_path"/pushspecs.git
 
-./lorry --log="$logfile" --working-area="$workdir" \
+"${SRCDIR}/test-lorry" --log="$logfile" --working-area="$workdir" \
         --mirror-base-url-push=file://"$mirror_path" \
         --mirror-base-url-fetch=file://"$mirror_path" \
         "$lorryfile"
diff --git a/tests/svn-single-commit.script b/tests/svn-single-commit.script
index f1c769c..32a9cca 100755
--- a/tests/svn-single-commit.script
+++ b/tests/svn-single-commit.script
@@ -23,7 +23,7 @@ set -e
 logfile="$DATADIR/svn-test-repo.log"
 workdir="$DATADIR/work-dir"
 
-./lorry --pull-only --log="$logfile" --working-area="$workdir" \
+"${SRCDIR}/test-lorry" --pull-only --log="$logfile" --working-area="$workdir" \
   "$DATADIR/svn-test-repo.lorry" > /dev/null 2> /dev/null
 
 # verify that the git repository was created successfully
diff --git a/tests/tar-single-commit.script b/tests/tar-single-commit.script
index 623adee..1562446 100755
--- a/tests/tar-single-commit.script
+++ b/tests/tar-single-commit.script
@@ -20,10 +20,10 @@
 
 set -e
 
-logfile="$DATADIR/svn-test-repo.log"
+logfile="$DATADIR/tar-single-commit.log"
 workdir="$DATADIR/work-dir"
 
-./lorry --pull-only --log="$logfile" --working-area="$workdir" \
+"${SRCDIR}/test-lorry" --pull-only --log="$logfile" --working-area="$workdir" \
   "$DATADIR/tar-test-repo.lorry" > /dev/null 2> /dev/null
 
 # verify that the git repositories were created successfully
@@ -39,5 +39,5 @@ for FORMAT in "gzip" "bzip2" "lzma"; do
   git cat-file blob master:test.txt
 
   # list the commit messages
-  git log --pretty='%s' master
+  git log --pretty='%s' master | sed -e"s,${DATADIR},DATADIR,"
 done
diff --git a/tests/tar-single-commit.setup b/tests/tar-single-commit.setup
index c99450f..bfc99f8 100755
--- a/tests/tar-single-commit.setup
+++ b/tests/tar-single-commit.setup
@@ -29,7 +29,8 @@ echo "first line" > "$repo/test.txt"
 cd "$DATADIR"
 tar -czf tar-test-repo.tar.gz "`basename $repo`"
 tar -cjf tar-test-repo.tar.bz2 "`basename $repo`"
-tar -cf tar-test-repo.tar.lzma "`basename $repo`" --lzma
+tar -cf tar-test-repo.tar "`basename $repo`"
+xz -z tar-test-repo.tar
 
 # create the .lorry file for the tarball "repositories"
 cat <<EOF > $DATADIR/tar-test-repo.lorry
@@ -50,7 +51,7 @@ cat <<EOF > $DATADIR/tar-test-repo.lorry
     "type": "tarball",
     "compression": "lzma",
     "strip": 1,
-    "url": "file://$DATADIR/tar-test-repo.tar.lzma"
+    "url": "file://$DATADIR/tar-test-repo.tar.xz"
   }
 }
 EOF
diff --git a/tests/tar-single-commit.stdout b/tests/tar-single-commit.stdout
index 04cec41..32fcdd5 100644
--- a/tests/tar-single-commit.stdout
+++ b/tests/tar-single-commit.stdout
@@ -1,12 +1,15 @@
 gzip
 refs/heads/master
+refs/tags/tar-test-repo
 first line
-Tarball conversion
+Imported from DATADIR/work-dir/tar-test-repo-gzip/tar-test-repo.tar.gz.
 bzip2
 refs/heads/master
+refs/tags/tar-test-repo
 first line
-Tarball conversion
+Imported from DATADIR/work-dir/tar-test-repo-bzip2/tar-test-repo.tar.bz2.
 lzma
 refs/heads/master
+refs/tags/tar-test-repo
 first line
-Tarball conversion
+Imported from DATADIR/work-dir/tar-test-repo-lzma/tar-test-repo.tar.xz.