Fix up Lorry to expect repositories to be bare.

This patch makes Lorry always create bare repositories where it can (Note that
it cannot for CVS imports) and to create tarballs of bare repositories (if not
disabled) which will be more efficient than bundles for creation and cloning.

We may be able to disable bundles later.

19 files changed, 214 insertions, 92 deletions

diff --git a/README b/README
index 45b803a..a1355a4 100644
--- a/README
+++ b/README
@@ -5,9 +5,9 @@ Lorry is a tool to take upstream source code (in various formats,
 though preferably in version control) and converts it into a git
 repository.
 
-If you want to try this, use `--pull-only` and/or `--gitorious-base-url`
+If you want to try this, use `--pull-only` and/or `--mirror-base-url-push`
 so that you do not accidentally overwrite important stuff for Baserock.
-(If you don't have direct commit access to Baserock on Gitorious.org,
+(If you don't have direct commit access to Baserock on git.baserock.org
 then you're not dangerous.)
 
 See the manual page for instructions on using.
@@ -25,8 +25,8 @@ You can find a lot of lorries to crib ideas from at:
 Implementation
 --------------
 
-Lorry relies on git-svn, git-cvsimport, and bzr fast-export for the
-conversions. You need to have them installed.
+Lorry relies on git-svn, git-cvsimport, hg-fast-export, perl (for tarballs) and
+bzr fast-export for the conversions. You need to have them installed.
 
 Lorry file specification
 ------------------------
@@ -193,12 +193,14 @@ often have the folder name as the first component.
     }
 }
 
-NOTE: tarball imports are unlikely to give the same sha.
+NOTE: tarball imports are unlikely to give the same commit SHA1 but the tree
+SHA1 inside (which is what is used for artifact cache IDs) should remain
+stable.
 
 Legal stuff
 -----------
 
-Copyright (C) 2011  Codethink Limited
+Copyright (C) 2011, 2012  Codethink Limited
  
 This program is free software; you can redistribute it and/or modify
 it under the terms of the GNU General Public License as published by
diff --git a/lorry b/lorry
index 29df6dd..39fed88 100755
--- a/lorry
+++ b/lorry
@@ -80,6 +80,13 @@ class Lorry(cliapp.Application):
         self.settings.string(['bundle-dest'],
                              'put created bundles in BUNDLES',
                              metavar='BUNDLES')
+        self.settings.choice(['tarball'], ['first', 'never', 'always'],
+                             'create tarballs of git repositories.'
+                             'first will only tar if there is not already '
+                             'a tarball in TARBALLS (default: first)')
+        self.settings.string(['tarball-dest'],
+                             'put created tarballs in TARBALLS',
+                             metavar='TARBALLS')
 
     def process_args(self, args):
         status = 0
@@ -133,6 +140,36 @@ class Lorry(cliapp.Application):
             expr = '1,/^[0-9a-f]\{40\}/{ /^[0-9a-f]\{40\}/!{/^[^#]/d}}'
             self.run_program(['sed', '-i', '-e', expr, path], cwd=gitdir)
 
+    def make_tarball(self, name, gitdir):
+        if self.settings['tarball'] == 'never': return
+        tarballname = "%s/%s" % (self.settings['mirror-base-url-fetch'],
+                                 name)
+        path = os.path.join(self.settings['tarball-dest'],
+                            quote_url(tarballname)) + '.tar'
+        if os.path.exists(os.path.join(gitdir, '.git')):
+            gitdir = os.path.join(gitdir, '.git')
+        if not os.path.exists(path) or self.settings['tarball'] == 'always':
+            self.progress('.. building tarball %s' % tarballname)
+            args = ['tar', 'cf', path]
+            if os.path.exists(os.path.join(gitdir, 'config')):
+                os.rename(os.path.join(gitdir, 'config'),
+                          os.path.join(gitdir, 'config.lorrytmp'))
+                with open(os.path.join(gitdir, 'config'), 'w') as fh:
+                    fh.write("""[core]
+	repositoryformatversion = 0
+	filemode = true
+	bare = true
+""")
+            for entry in ['HEAD', 'objects', 'refs',
+                          'info', 'packed-refs', 'config', 'branches',
+                          'description']:
+                if os.path.exists(os.path.join(gitdir, entry)):
+                    args += [entry]
+            self.run_program(args, cwd=gitdir)
+            if os.path.exists(os.path.join(gitdir, 'config.lorrytmp')):
+                os.rename(os.path.join(gitdir, 'config.lorrytmp'),
+                          os.path.join(gitdir, 'config'))
+
     def gitify(self, name, spec):
         self.progress('Getting %s' % name)
         table = {
@@ -158,6 +195,7 @@ class Lorry(cliapp.Application):
                 self.run_program(['git', 'repack', '-a', '-d', '--depth=250',
                                   '--window=250'], cwd=gitdir)
             self.bundle(name, gitdir)
+            self.make_tarball(name, gitdir)
         except:
             if backupdir is not None:
                 faildir = self.save_failgit(name, dirname, gitdir)
@@ -238,13 +276,12 @@ class Lorry(cliapp.Application):
 
     def mirror_git(self, project_name, dirname, gitdir, spec):
         if not os.path.exists(gitdir):
-            self.progress('.. doing initial clone')
-            self.run_program(['git', 'clone', '--mirror', spec['url'], gitdir])
-        else:
-            self.progress('.. updating existing clone')
-            self.run_program(['git', 'fetch', spec['url'],
-                              '+refs/heads/*:refs/heads/*',
-                              '+refs/tags/*:refs/tags/*'], cwd=gitdir)
+            self.progress('.. initialising git dir')
+            self.run_program(['git', 'init', '--bare', gitdir])
+        self.progress('.. updating existing clone')
+        self.run_program(['git', 'fetch', spec['url'],
+                          '+refs/heads/*:refs/heads/*',
+                          '+refs/tags/*:refs/tags/*'], cwd=gitdir)
 
     def gitify_bzr(self, project_name, dirname, gitdir, spec):
         bzrdir = os.path.join(dirname, 'bzr')
@@ -256,7 +293,7 @@ class Lorry(cliapp.Application):
         if not os.path.exists(gitdir):
             self.progress('.. creating git repo')
             os.mkdir(gitdir)
-            self.run_program(['git', 'init', gitdir])
+            self.run_program(['git', 'init', '--bare', gitdir])
 
         # branches are the listed branches, plus the branch specified in url
         if 'branches' in spec:
@@ -279,7 +316,7 @@ class Lorry(cliapp.Application):
                                  cwd=branchdir)
 
         exports = {}
-        bzrmarks = os.path.join(gitdir, '.git', 'marks.bzr')
+        bzrmarks = os.path.join(gitdir, 'marks.bzr')
         for branch, address in branches.iteritems():
             branchdir = os.path.join(bzrdir, branch)
             self.progress('.. fast-exporting branch %s from bzr' % branch)
@@ -292,7 +329,7 @@ class Lorry(cliapp.Application):
                 cmdline.append('--export-marks=' + bzrmarks)
             self.run_program(cmdline)
 
-        gitmarks = os.path.join(gitdir, '.git', 'marks.git')
+        gitmarks = os.path.join(gitdir, 'marks.git')
         for branch, address in branches.iteritems():
             self.progress('.. fast-importing branch %s into git' % branch)
             with open(exports[branch], 'rb') as exportfile:
@@ -323,8 +360,12 @@ class Lorry(cliapp.Application):
             # fetching the root of the repository
             # git-svn will convert branch, trunk and tag paths to allow this,
             # but it is simpler to disable it and do it manually
-            self.run_program(['git', 'svn', 'init', spec['url'], gitdir,
+            self.run_program(['git', 'svn', 'init', spec['url'], gitdir + "-tmp",
                               '--svn-remote=svn', '--no-minimize-url'])
+            os.rename(os.path.join(gitdir + "-tmp", '.git'), gitdir)
+            os.rmdir(gitdir + "-tmp")
+            self.run_program(['git', 'config', 'core.bare', 'true'],
+                             cwd=gitdir)
             self.run_program(['git', 'config', 'svn-remote.svn.fetch',
                               layout["trunk"]+':refs/heads/master'], 
                              cwd=gitdir)
@@ -353,37 +394,30 @@ class Lorry(cliapp.Application):
             self.run_program(['hg', 'clone', '--quiet', spec['url'], hgdir])
 
         if not os.path.exists(gitdir):
-            self.run_program(['git', 'init', gitdir])
+            self.run_program(['git', 'init', '--bare', gitdir])
         
         self.progress('.. fast-exporting into git')
         self.run_program(['hg-fast-export', '--quiet', '-r', '../hg'], 
                          cwd=gitdir)
 
     def gitify_tarball(self, project_name, dirname, gitdir, spec):
-        tardest = os.path.join(dirname, 'tarball')
+        url = spec['url']
+        url_path = urllib2.urlparse.urlparse(url)[2]
+        basename = os.path.basename(url_path)
+        tardest = os.path.join(dirname, basename)
+        self.progress('.. checking if we need to fetch %s' % basename)
         if not os.path.exists(tardest):
+            self.progress('.. attempting to fetch.')
             with open(tardest, 'w') as tarfile:
                 urlfile = urllib2.urlopen(spec['url'])
                 tarfile.write(urlfile.read())
                 urlfile.close()
-
+        else:
+            self.progress('.. no need to run, nothing to do')
         if not os.path.exists(gitdir):
-            self.run_program(['git', 'init', gitdir])
-        cmdline = ['tar', '--extract', '--file', tardest]
-        # compression is handled in long form, so use gzip instead of z
-        try:
-            cmdline += ['--' + spec['compression']]
-        except KeyError:
-            pass
-        # tarballs often have a directory on top, strip = 1 will remove it
-        try:
-            cmdline += ['--strip-components', str(spec['strip'])]
-        except KeyError:
-            pass
+            self.run_program(['git', 'init', '--bare', gitdir])
+        cmdline = ["%s.tar-importer" % __file__, tardest]
         self.run_program(cmdline, cwd=gitdir)
-        self.run_program(['git', 'add', '.'], cwd=gitdir)
-        self.run_program(['git', 'commit', '-m', 'Tarball conversion'],
-                         cwd=gitdir)
             
 
     def push_to_mirror_server(self, name, gitdir,
diff --git a/import-tars.perl b/lorry.tar-importer
index 95438e1..eb17ef2 100644..100755
--- a/import-tars.perl
+++ b/lorry.tar-importer
@@ -1,5 +1,8 @@
 #!/usr/bin/perl
 
+## Note: Modified for Baserock lorry.
+
+
 ## tar archive frontend for git-fast-import
 ##
 ## For example:
@@ -23,15 +26,19 @@ my $metaext = '';
 die "usage: import-tars [--metainfo=extension] *.tar.{gz,bz2,lzma,xz,Z}\n"
 	unless GetOptions('metainfo=s' => \$metaext) && @ARGV;
 
-my $branch_name = 'import-tars';
+my $branch_name = 'master';
 my $branch_ref = "refs/heads/$branch_name";
-my $author_name = $ENV{'GIT_AUTHOR_NAME'} || 'T Ar Creator';
-my $author_email = $ENV{'GIT_AUTHOR_EMAIL'} || 'tar@example.com';
+my $old_sha = `git show-ref $branch_ref 2>/dev/null`;
+my $author_name = $ENV{'GIT_AUTHOR_NAME'} || 'Lorry Tar Creator';
+my $author_email = $ENV{'GIT_AUTHOR_EMAIL'} || 'lorry-tar-importer@baserock.org';
 my $committer_name = $ENV{'GIT_COMMITTER_NAME'} || `git config --get user.name`;
 my $committer_email = $ENV{'GIT_COMMITTER_EMAIL'} || `git config --get user.email`;
 
-chomp($committer_name, $committer_email);
+chomp($committer_name, $committer_email, $old_sha);
 
+if ($old_sha ne '') {
+   $old_sha = ($old_sha =~ /^([a-f0-9]+)/)[0];
+}
 open(FI, '|-', 'git', 'fast-import', '--quiet')
 	or die "Unable to start git fast-import: $!\n";
 foreach my $tar_file (@ARGV)
@@ -161,7 +168,15 @@ committer $this_committer_name <$this_committer_email> $commit_time +0000
 data <<END_OF_COMMIT_MESSAGE
 $commit_msg
 END_OF_COMMIT_MESSAGE
+EOF
 
+        if ($old_sha ne '') {
+           print FI <<EOF;
+from $old_sha
+EOF
+        }
+
+        print FI <<EOF;
 deleteall
 EOF
 
@@ -174,6 +189,7 @@ EOF
 	}
 	print FI "\n";
 
+
 	print FI <<EOF;
 tag $tar_name
 from $branch_ref
diff --git a/setup.py b/setup.py
index cc5bde5..63bd431 100644
--- a/setup.py
+++ b/setup.py
@@ -26,8 +26,6 @@ import os
 import shutil
 import subprocess
 
-import morphlib
-
 
 class GenerateManpage(build):
 
@@ -46,6 +44,7 @@ class Clean(clean):
     clean_files = [
         '.coverage',
         'build',
+        'lorry.1'
     ]
     clean_globs = [
         '*/*.py[co]',
@@ -86,7 +85,7 @@ FIXME
       author='Baserock',
       author_email='baserock-dev@baserock.org',
       url='http://wiki.baserock.org/',
-      scripts=['lorry'],
+      scripts=['lorry', 'lorry.tar-importer'],
       data_files=[('share/man/man1', glob.glob('*.[1-8]'))],
       cmdclass={
           'build': GenerateManpage,
diff --git a/test-lorry b/test-lorry
new file mode 100755
index 0000000..c693507
--- /dev/null
+++ b/test-lorry
@@ -0,0 +1,3 @@
+#!/bin/sh
+
+exec ${SRCDIR}/lorry --no-default-configs --tarball=never "$@"
diff --git a/tests/bzr-single-commit.script b/tests/bzr-single-commit.script
index a870558..51fdb6d 100755
--- a/tests/bzr-single-commit.script
+++ b/tests/bzr-single-commit.script
@@ -24,7 +24,7 @@ set -e
 logfile="$DATADIR/bzr-test-repo.log"
 workdir="$DATADIR/work-dir"
 
-./lorry --pull-only --log="$logfile" --working-area="$workdir" \
+${SRCDIR}/test-lorry --pull-only --log="$logfile" --working-area="$workdir" \
   "$DATADIR/bzr-test-repo.lorry" > /dev/null 2> /dev/null
 
 # verify that the git repository was set up correctly
diff --git a/tests/cvs-single-commit.script b/tests/cvs-single-commit.script
index ad4eca8..e133fae 100755
--- a/tests/cvs-single-commit.script
+++ b/tests/cvs-single-commit.script
@@ -28,7 +28,7 @@ export USER=root
 export LOGNAME=$USER
 export USERNAME=$USER
 
-./lorry --pull-only --log="$logfile" --working-area="$workdir" \
+${SRCDIR}/test-lorry --pull-only --log="$logfile" --working-area="$workdir" \
   "$DATADIR/cvs-test-repo.lorry" > /dev/null 2> /dev/null
 
 # verify that the git repository was created successfully
diff --git a/tests/git-backup-on-error.script b/tests/git-backup-on-error.script
index 134367c..8be1a83 100755
--- a/tests/git-backup-on-error.script
+++ b/tests/git-backup-on-error.script
@@ -30,16 +30,17 @@ normalize() {
     DATETIMESPEC='[0-9]*-[0-9]*-[0-9]*-[0-9]*:[0-9]*:[0-9]*'
     sed -r -e "s|git-pre-update-$DATETIMESPEC|git-pre-update-DATETIME|g" \
            -e "s|git-post-fail-$DATETIMESPEC|git-post-fail-DATETIME|g" \
+           -e '/hooks\/.*\.sample/d' \
            -e "s|$DATADIR|DATADIR|g" "$@"
 }
 
 # mirror some history
-./lorry --pull-only --log="$logfile" --working-area="$workdir" --bundle=never \
+${SRCDIR}/test-lorry --pull-only --log="$logfile" --working-area="$workdir" --bundle=never \
   "$DATADIR/git-backup-test-repo.lorry" | normalize 
 
 # make upstream disappear to cause errors
 rm -rf "$repo"
-if ./lorry --pull-only --log="$logfile" --working-area="$workdir" \
+if ${SRCDIR}/test-lorry --pull-only --log="$logfile" --working-area="$workdir" \
            "$DATADIR/git-backup-test-repo.lorry" --bundle=never 2>/dev/null | \
            normalize
 then
diff --git a/tests/git-backup-on-error.stdout b/tests/git-backup-on-error.stdout
index 195970e..7e18431 100644
--- a/tests/git-backup-on-error.stdout
+++ b/tests/git-backup-on-error.stdout
@@ -8,68 +8,40 @@ DATADIR/work-dir/git-backup-test-repo/git-post-fail-DATETIME/branches
 DATADIR/work-dir/git-backup-test-repo/git-post-fail-DATETIME/config
 DATADIR/work-dir/git-backup-test-repo/git-post-fail-DATETIME/description
 DATADIR/work-dir/git-backup-test-repo/git-post-fail-DATETIME/hooks
-DATADIR/work-dir/git-backup-test-repo/git-post-fail-DATETIME/hooks/applypatch-msg.sample
-DATADIR/work-dir/git-backup-test-repo/git-post-fail-DATETIME/hooks/commit-msg.sample
-DATADIR/work-dir/git-backup-test-repo/git-post-fail-DATETIME/hooks/post-commit.sample
-DATADIR/work-dir/git-backup-test-repo/git-post-fail-DATETIME/hooks/post-receive.sample
-DATADIR/work-dir/git-backup-test-repo/git-post-fail-DATETIME/hooks/post-update.sample
-DATADIR/work-dir/git-backup-test-repo/git-post-fail-DATETIME/hooks/pre-applypatch.sample
-DATADIR/work-dir/git-backup-test-repo/git-post-fail-DATETIME/hooks/pre-commit.sample
-DATADIR/work-dir/git-backup-test-repo/git-post-fail-DATETIME/hooks/pre-rebase.sample
-DATADIR/work-dir/git-backup-test-repo/git-post-fail-DATETIME/hooks/prepare-commit-msg.sample
-DATADIR/work-dir/git-backup-test-repo/git-post-fail-DATETIME/hooks/update.sample
 DATADIR/work-dir/git-backup-test-repo/git-post-fail-DATETIME/info
 DATADIR/work-dir/git-backup-test-repo/git-post-fail-DATETIME/info/exclude
 DATADIR/work-dir/git-backup-test-repo/git-post-fail-DATETIME/info/refs
 DATADIR/work-dir/git-backup-test-repo/git-post-fail-DATETIME/objects
-DATADIR/work-dir/git-backup-test-repo/git-post-fail-DATETIME/packed-refs
 DATADIR/work-dir/git-backup-test-repo/git-post-fail-DATETIME/refs
 DATADIR/work-dir/git-backup-test-repo/git-post-fail-DATETIME/refs/heads
+DATADIR/work-dir/git-backup-test-repo/git-post-fail-DATETIME/refs/heads/master
 DATADIR/work-dir/git-backup-test-repo/git-post-fail-DATETIME/refs/tags
 DATADIR/work-dir/git-backup-test-repo/git-pre-update-DATETIME
+DATADIR/work-dir/git-backup-test-repo/git-pre-update-DATETIME/FETCH_HEAD
 DATADIR/work-dir/git-backup-test-repo/git-pre-update-DATETIME/HEAD
 DATADIR/work-dir/git-backup-test-repo/git-pre-update-DATETIME/branches
 DATADIR/work-dir/git-backup-test-repo/git-pre-update-DATETIME/config
 DATADIR/work-dir/git-backup-test-repo/git-pre-update-DATETIME/description
 DATADIR/work-dir/git-backup-test-repo/git-pre-update-DATETIME/hooks
-DATADIR/work-dir/git-backup-test-repo/git-pre-update-DATETIME/hooks/applypatch-msg.sample
-DATADIR/work-dir/git-backup-test-repo/git-pre-update-DATETIME/hooks/commit-msg.sample
-DATADIR/work-dir/git-backup-test-repo/git-pre-update-DATETIME/hooks/post-commit.sample
-DATADIR/work-dir/git-backup-test-repo/git-pre-update-DATETIME/hooks/post-receive.sample
-DATADIR/work-dir/git-backup-test-repo/git-pre-update-DATETIME/hooks/post-update.sample
-DATADIR/work-dir/git-backup-test-repo/git-pre-update-DATETIME/hooks/pre-applypatch.sample
-DATADIR/work-dir/git-backup-test-repo/git-pre-update-DATETIME/hooks/pre-commit.sample
-DATADIR/work-dir/git-backup-test-repo/git-pre-update-DATETIME/hooks/pre-rebase.sample
-DATADIR/work-dir/git-backup-test-repo/git-pre-update-DATETIME/hooks/prepare-commit-msg.sample
-DATADIR/work-dir/git-backup-test-repo/git-pre-update-DATETIME/hooks/update.sample
 DATADIR/work-dir/git-backup-test-repo/git-pre-update-DATETIME/info
 DATADIR/work-dir/git-backup-test-repo/git-pre-update-DATETIME/info/exclude
 DATADIR/work-dir/git-backup-test-repo/git-pre-update-DATETIME/info/refs
 DATADIR/work-dir/git-backup-test-repo/git-pre-update-DATETIME/objects
-DATADIR/work-dir/git-backup-test-repo/git-pre-update-DATETIME/packed-refs
 DATADIR/work-dir/git-backup-test-repo/git-pre-update-DATETIME/refs
 DATADIR/work-dir/git-backup-test-repo/git-pre-update-DATETIME/refs/heads
+DATADIR/work-dir/git-backup-test-repo/git-pre-update-DATETIME/refs/heads/master
 DATADIR/work-dir/git-backup-test-repo/git-pre-update-DATETIME/refs/tags
+DATADIR/work-dir/git-backup-test-repo/git/FETCH_HEAD
 DATADIR/work-dir/git-backup-test-repo/git/HEAD
 DATADIR/work-dir/git-backup-test-repo/git/branches
 DATADIR/work-dir/git-backup-test-repo/git/config
 DATADIR/work-dir/git-backup-test-repo/git/description
 DATADIR/work-dir/git-backup-test-repo/git/hooks
-DATADIR/work-dir/git-backup-test-repo/git/hooks/applypatch-msg.sample
-DATADIR/work-dir/git-backup-test-repo/git/hooks/commit-msg.sample
-DATADIR/work-dir/git-backup-test-repo/git/hooks/post-commit.sample
-DATADIR/work-dir/git-backup-test-repo/git/hooks/post-receive.sample
-DATADIR/work-dir/git-backup-test-repo/git/hooks/post-update.sample
-DATADIR/work-dir/git-backup-test-repo/git/hooks/pre-applypatch.sample
-DATADIR/work-dir/git-backup-test-repo/git/hooks/pre-commit.sample
-DATADIR/work-dir/git-backup-test-repo/git/hooks/pre-rebase.sample
-DATADIR/work-dir/git-backup-test-repo/git/hooks/prepare-commit-msg.sample
-DATADIR/work-dir/git-backup-test-repo/git/hooks/update.sample
 DATADIR/work-dir/git-backup-test-repo/git/info
 DATADIR/work-dir/git-backup-test-repo/git/info/exclude
 DATADIR/work-dir/git-backup-test-repo/git/info/refs
 DATADIR/work-dir/git-backup-test-repo/git/objects
-DATADIR/work-dir/git-backup-test-repo/git/packed-refs
 DATADIR/work-dir/git-backup-test-repo/git/refs
 DATADIR/work-dir/git-backup-test-repo/git/refs/heads
+DATADIR/work-dir/git-backup-test-repo/git/refs/heads/master
 DATADIR/work-dir/git-backup-test-repo/git/refs/tags
diff --git a/tests/git-single-commit.script b/tests/git-single-commit.script
index 4a6e8b3..b3d8e43 100755
--- a/tests/git-single-commit.script
+++ b/tests/git-single-commit.script
@@ -24,7 +24,7 @@ set -e
 logfile="$DATADIR/git-test-repo.log"
 workdir="$DATADIR/work-dir"
 
-./lorry --pull-only --log="$logfile" --working-area="$workdir" \
+${SRCDIR}/test-lorry --pull-only --log="$logfile" --working-area="$workdir" \
   "$DATADIR/git-test-repo.lorry" > /dev/null 2> /dev/null
 
 # verify that the git repository was set up correctly
diff --git a/tests/hg-single-commit.script b/tests/hg-single-commit.script
index c370190..585fd33 100755
--- a/tests/hg-single-commit.script
+++ b/tests/hg-single-commit.script
@@ -24,7 +24,7 @@ set -e
 logfile="$DATADIR/hg-test-repo.log"
 workdir="$DATADIR/work-dir"
 
-./lorry --verbose --pull-only --log="$logfile" --working-area="$workdir" \
+${SRCDIR}/test-lorry --verbose --pull-only --log="$logfile" --working-area="$workdir" \
   "$DATADIR/hg-test-repo.lorry" > /dev/null 2> /dev/null
 
 # verify that the git repository was created correctly
diff --git a/tests/make-tarball.script b/tests/make-tarball.script
new file mode 100755
index 0000000..8be7b77
--- /dev/null
+++ b/tests/make-tarball.script
@@ -0,0 +1,32 @@
+#!/bin/sh
+#
+# Test that we can create a tarball of the git trees.
+#
+# Copyright (C) 2012  Codethink Limited
+# 
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation; version 2 of the License.
+# 
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+# 
+# You should have received a copy of the GNU General Public License along
+# with this program; if not, write to the Free Software Foundation, Inc.,
+# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
+
+
+set -e
+
+logfile="$DATADIR/make-tarball.log"
+workdir="$DATADIR/work-dir"
+
+${SRCDIR}/test-lorry --pull-only --log="$logfile" --working-area="$workdir" \
+  --tarball=first \
+  "$DATADIR/make-tarball-repo.lorry" > /dev/null 2> /dev/null
+
+# verify that we can see the tarball generated of the git tree
+
+test -r "${workdir}/make-tarball-repo-bzip2/git/"*"make_tarball_repo_bzip2.tar"
diff --git a/tests/make-tarball.setup b/tests/make-tarball.setup
new file mode 100755
index 0000000..2032610
--- /dev/null
+++ b/tests/make-tarball.setup
@@ -0,0 +1,59 @@
+#!/bin/sh
+#
+# Creates gzip/bzip2/lzma tarballs, each with a single file.
+#
+# Copyright (C) 2012  Codethink Limited
+# 
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation; version 2 of the License.
+# 
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+# 
+# You should have received a copy of the GNU General Public License along
+# with this program; if not, write to the Free Software Foundation, Inc.,
+# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
+
+
+set -e
+
+# create the original "repository"
+repo="$DATADIR/make-tarball-repo"
+mkdir "$repo"
+echo "first line" > "$repo/test.txt"
+
+# create the tarballs
+cd "$DATADIR"
+tar -czf make-tarball-repo.tar.gz "`basename $repo`"
+tar -cjf make-tarball-repo.tar.bz2 "`basename $repo`"
+tar -cf make-tarball-repo.tar.lzma "`basename $repo`" --lzma
+
+# create the .lorry file for the tarball "repositories"
+cat <<EOF > $DATADIR/make-tarball-repo.lorry
+{
+  "make-tarball-repo-gzip": {
+    "type": "tarball",
+    "compression": "gzip",
+    "strip": 1,
+    "url": "file://$DATADIR/make-tarball-repo.tar.gz"
+  },
+  "make-tarball-repo-bzip2": {
+    "type": "tarball",
+    "compression": "bzip2",
+    "strip": 1,
+    "url": "file://$DATADIR/make-tarball-repo.tar.bz2"
+  },
+  "make-tarball-repo-lzma": {
+    "type": "tarball",
+    "compression": "lzma",
+    "strip": 1,
+    "url": "file://$DATADIR/make-tarball-repo.tar.lzma"
+  }
+}
+EOF
+
+# create the working directory
+test -d "$DATADIR/work-dir" || mkdir "$DATADIR/work-dir"
diff --git a/tests/no-pushspec-pushall.script b/tests/no-pushspec-pushall.script
index cc56d8a..7de3a58 100755
--- a/tests/no-pushspec-pushall.script
+++ b/tests/no-pushspec-pushall.script
@@ -37,7 +37,7 @@ mirror_path="$DATADIR"/git-mirror
 mkdir -p "$mirror_path"
 git init --quiet --bare "$mirror_path"/no-pushspec.git
 
-./lorry --log="$logfile" --working-area="$workdir" \
+${SRCDIR}/test-lorry --log="$logfile" --working-area="$workdir" \
         --mirror-base-url-push=file://"$mirror_path" \
         --mirror-base-url-fetch=file://"$mirror_path" \
         "$lorryfile"
diff --git a/tests/pushspecs-only.script b/tests/pushspecs-only.script
index 787cc38..d7f4c2f 100755
--- a/tests/pushspecs-only.script
+++ b/tests/pushspecs-only.script
@@ -41,7 +41,7 @@ mirror_path="$DATADIR"/git-mirror
 mkdir -p "$mirror_path"
 git init --quiet --bare "$mirror_path"/pushspecs.git
 
-./lorry --log="$logfile" --working-area="$workdir" \
+${SRCDIR}/test-lorry --log="$logfile" --working-area="$workdir" \
         --mirror-base-url-push=file://"$mirror_path" \
         --mirror-base-url-fetch=file://"$mirror_path" \
         "$lorryfile"
diff --git a/tests/svn-single-commit.script b/tests/svn-single-commit.script
index f1c769c..cff91b0 100755
--- a/tests/svn-single-commit.script
+++ b/tests/svn-single-commit.script
@@ -23,7 +23,7 @@ set -e
 logfile="$DATADIR/svn-test-repo.log"
 workdir="$DATADIR/work-dir"
 
-./lorry --pull-only --log="$logfile" --working-area="$workdir" \
+${SRCDIR}/test-lorry --pull-only --log="$logfile" --working-area="$workdir" \
   "$DATADIR/svn-test-repo.lorry" > /dev/null 2> /dev/null
 
 # verify that the git repository was created successfully
diff --git a/tests/tar-single-commit.script b/tests/tar-single-commit.script
index 623adee..f1d92c5 100755
--- a/tests/tar-single-commit.script
+++ b/tests/tar-single-commit.script
@@ -20,10 +20,10 @@
 
 set -e
 
-logfile="$DATADIR/svn-test-repo.log"
+logfile="$DATADIR/tar-single-commit.log"
 workdir="$DATADIR/work-dir"
 
-./lorry --pull-only --log="$logfile" --working-area="$workdir" \
+${SRCDIR}/test-lorry --pull-only --log="$logfile" --working-area="$workdir" \
   "$DATADIR/tar-test-repo.lorry" > /dev/null 2> /dev/null
 
 # verify that the git repositories were created successfully
@@ -39,5 +39,5 @@ for FORMAT in "gzip" "bzip2" "lzma"; do
   git cat-file blob master:test.txt
 
   # list the commit messages
-  git log --pretty='%s' master
+  git log --pretty='%s' master | sed -e"s,${DATADIR},DATADIR,"
 done
diff --git a/tests/tar-single-commit.setup b/tests/tar-single-commit.setup
index c99450f..bfc99f8 100755
--- a/tests/tar-single-commit.setup
+++ b/tests/tar-single-commit.setup
@@ -29,7 +29,8 @@ echo "first line" > "$repo/test.txt"
 cd "$DATADIR"
 tar -czf tar-test-repo.tar.gz "`basename $repo`"
 tar -cjf tar-test-repo.tar.bz2 "`basename $repo`"
-tar -cf tar-test-repo.tar.lzma "`basename $repo`" --lzma
+tar -cf tar-test-repo.tar "`basename $repo`"
+xz -z tar-test-repo.tar
 
 # create the .lorry file for the tarball "repositories"
 cat <<EOF > $DATADIR/tar-test-repo.lorry
@@ -50,7 +51,7 @@ cat <<EOF > $DATADIR/tar-test-repo.lorry
     "type": "tarball",
     "compression": "lzma",
     "strip": 1,
-    "url": "file://$DATADIR/tar-test-repo.tar.lzma"
+    "url": "file://$DATADIR/tar-test-repo.tar.xz"
   }
 }
 EOF
diff --git a/tests/tar-single-commit.stdout b/tests/tar-single-commit.stdout
index 04cec41..32fcdd5 100644
--- a/tests/tar-single-commit.stdout
+++ b/tests/tar-single-commit.stdout
@@ -1,12 +1,15 @@
 gzip
 refs/heads/master
+refs/tags/tar-test-repo
 first line
-Tarball conversion
+Imported from DATADIR/work-dir/tar-test-repo-gzip/tar-test-repo.tar.gz.
 bzip2
 refs/heads/master
+refs/tags/tar-test-repo
 first line
-Tarball conversion
+Imported from DATADIR/work-dir/tar-test-repo-bzip2/tar-test-repo.tar.bz2.
 lzma
 refs/heads/master
+refs/tags/tar-test-repo
 first line
-Tarball conversion
+Imported from DATADIR/work-dir/tar-test-repo-lzma/tar-test-repo.tar.xz.