Remove original lorry binary

1 files changed, 0 insertions, 511 deletions

diff --git a/lorry b/lorry
deleted file mode 100755
index 2339c9f..0000000
--- a/lorry
+++ /dev/null
@@ -1,511 +0,0 @@
-#!/usr/bin/python
-# Copyright (C) 2013  Codethink Limited
-# 
-# This program is free software; you can redistribute it and/or modify
-# it under the terms of the GNU General Public License as published by
-# the Free Software Foundation; version 2 of the License.
-# 
-# This program is distributed in the hope that it will be useful,
-# but WITHOUT ANY WARRANTY; without even the implied warranty of
-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-# GNU General Public License for more details.
-# 
-# You should have received a copy of the GNU General Public License along
-# with this program; if not, write to the Free Software Foundation, Inc.,
-# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
-
-
-import cliapp
-import json
-import logging
-import os
-import urllib2
-import string
-import sys
-from datetime import datetime
-import shutil
-import traceback
-
-__version__ = '0.0'
-
-lorry_path = os.path.realpath(__file__)
-
-def quote_url(url):
-    ''' Convert URIs to strings that only contain digits, letters, % and _.
-
-    NOTE: When changing the code of this function, make sure to also apply
-    the same to the quote_url() function of morph. Otherwise the git bundles
-    generated by lorry may no longer be found by morph.
-
-    ''' 
-    valid_chars = string.digits + string.letters + '%_'
-    transl = lambda x: x if x in valid_chars else '_'
-    return ''.join([transl(x) for x in url])
-
-
-class Lorry(cliapp.Application):
-
-    def add_settings(self):
-        self.settings.string(['working-area', 'w'],
-                             'use DIR as the working area (for holding '
-                                'intermediate git repositories, etc)',
-                             metavar='DIR', default='workd')
-        self.settings.string(['mirror-base-url-push'],
-                             'base URL to use for pushing to the mirror '
-                             'server (default: %default)',
-                             metavar='URL',
-                             default='ssh://gitano@roadtrain.codethink.co.uk/delta')
-        self.settings.string(['mirror-base-url-fetch'],
-                             'base URL to use for bundle names and for '
-                             'pulling from the mirror server (default: '
-                             '%default)',
-                             metavar='URL',
-                             default='git://git.baserock.org/delta')
-        self.settings.boolean(['pull-only'], 
-                              'only pull from upstreams, do not push to '
-                              'the mirror server')
-        self.settings.boolean(['verbose', 'v'], 
-                              'report what is going on to stdout')
-        self.settings.boolean(['repack'], 
-                              'repack git repositories when an import has '
-                              'been updated (enabled by default)', default=True)
-        self.settings.string(['command-stdout'],
-                             'write the stdout of commands to this file',
-                             metavar='FILE', default=None)
-        self.settings.string(['command-stderr'],
-                             'write the stderr of commands to this file',
-                             metavar='FILE', default=None)
-        self.settings.choice(['bundle'], ['first', 'never', 'always'],
-                             'create bundles of git repositories.'
-                             'first will only bundle if there is not already '
-                             'a bundle in BUNDLES (default: first)')
-        self.settings.string(['bundle-dest'],
-                             'put created bundles in BUNDLES',
-                             metavar='BUNDLES')
-        self.settings.choice(['tarball'], ['first', 'never', 'always'],
-                             'create tarballs of git repositories.'
-                             'first will only tar if there is not already '
-                             'a tarball in TARBALLS (default: first)')
-        self.settings.string(['tarball-dest'],
-                             'put created tarballs in TARBALLS',
-                             metavar='TARBALLS')
-        self.settings.boolean(['keep-multiple-backups'],
-                              'keep multiple (time-stamped) backups (disabled by default)',
-                              default=False)
-
-    def process_args(self, args):
-        status = 0
-        
-        self.settings['working-area'] = os.path.abspath(self.settings['working-area'])
-        if not os.path.exists(self.settings['working-area']): 
-            os.makedirs(self.settings['working-area'])
-
-        for arg in args:
-            self.progress('Processing spec file %s' % arg)
-            with open(arg) as f:
-                specs = json.load(f)
-            for name in sorted(specs.keys()):
-                self.progress('Getting: %s' % name)
-                try:
-                    self.gitify(name, specs[name])
-                except Exception,e:
-                    status += 1
-                    sys.stderr.write(
-                        'Error mirroring:\n%s' % traceback.format_exc())
-                    logging.error(traceback.format_exc())
-        if status > 0 :
-           logging.debug('Total Mirrors failed: %d' %status)
-           status = 1
- 
-        self.progress('Done')
-        #print 'total failed:',status
-        sys.exit(status) 
-
-    def bundle(self, name, gitdir):
-        if self.settings['bundle'] == 'never': return
-        bundlename = "%s/%s" % (self.settings['mirror-base-url-fetch'], name)
-        path = os.path.join(self.settings['bundle-dest'],
-                            quote_url(bundlename)) + '.bndl'
-        if not os.path.exists(path) or self.settings['bundle'] == 'always':
-            self.progress('.. building bundle %s' % bundlename)
-
-            # create the bundle
-            self.run_program(['git', 'bundle', 'create', path, '--branches',
-                              '--tags'], cwd=gitdir)
-
-            # FIXME this is a hack to avoid unrecognized headers in bundles,
-            # which happens with some repositories. See 
-            #
-            #   http://marc.info/?l=git&m=132992959317420&w=2
-            #
-            # for more information. From the bundle's header section, the 
-            # expression below will remove all garbage lines that appear 
-            # between the first line (the bundle format meta comment) and 
-            # the list of refs.
-            expr = '1,/^[0-9a-f]\{40\}/{ /^[0-9a-f]\{40\}/!{/^[^#]/d}}'
-            self.run_program(['sed', '-i', '-e', expr, path], cwd=gitdir)
-
-    def make_tarball(self, name, gitdir):
-        if self.settings['tarball'] == 'never': return
-        tarballname = "%s/%s" % (self.settings['mirror-base-url-fetch'],
-                                 name)
-        path = os.path.join(self.settings['tarball-dest'],
-                            quote_url(tarballname)) + '.tar'
-        if os.path.exists(os.path.join(gitdir, '.git')):
-            gitdir = os.path.join(gitdir, '.git')
-        if not os.path.exists(path) or self.settings['tarball'] == 'always':
-            self.progress('.. building tarball %s' % tarballname)
-            args = ['tar', 'cf', path]
-            if os.path.exists(os.path.join(gitdir, 'config')):
-                os.rename(os.path.join(gitdir, 'config'),
-                          os.path.join(gitdir, 'config.lorrytmp'))
-                with open(os.path.join(gitdir, 'config'), 'w') as fh:
-                    fh.write("""[core]
-	repositoryformatversion = 0
-	filemode = true
-	bare = true
-""")
-            for entry in ['HEAD', 'objects', 'refs',
-                          'info', 'packed-refs', 'config', 'branches',
-                          'description']:
-                if os.path.exists(os.path.join(gitdir, entry)):
-                    args += [entry]
-            self.run_program(args, cwd=gitdir)
-            if os.path.exists(os.path.join(gitdir, 'config.lorrytmp')):
-                os.rename(os.path.join(gitdir, 'config.lorrytmp'),
-                          os.path.join(gitdir, 'config'))
-
-    def gitify(self, name, spec):
-        self.progress('Getting %s' % name)
-        table = {
-            'bzr': self.gitify_bzr,
-            'cvs': self.gitify_cvs,
-            'git': self.mirror_git,
-            'hg':  self.gitify_hg,
-            'svn': self.gitify_svn,
-            'tarball': self.gitify_tarball,
-        }
-        vcstype = spec['type']
-        if vcstype not in table:
-            raise cliapp.AppException('Unknown VCS type %s' % vcstype)
-        dirname = self.dirname(name)
-        if not os.path.exists(dirname):
-            os.mkdir(dirname)
-        gitdir = os.path.join(dirname, 'git')
-
-        time = datetime.now().strftime('%F-%T')
-
-        pre_update_name = 'git-pre-update'
-        pre_update_backup_suffix = (pre_update_name + '-' + time
-            if self.settings['keep-multiple-backups'] else pre_update_name)
-
-        pre_update_backup_dir = os.path.join(dirname, pre_update_backup_suffix)
-
-        post_fail_name = 'git-post-fail'
-        post_fail_backup_suffix = (post_fail_name + '-' + time
-            if self.settings['keep-multiple-backups'] else post_fail_name)
-
-        post_fail_backup_dir = os.path.join(dirname, post_fail_backup_suffix)
-
-        if not self.settings['keep-multiple-backups']:
-            # remove previous backups if they exist
-            if os.path.exists(pre_update_backup_dir):
-                shutil.rmtree(pre_update_backup_dir)
-
-            if os.path.exists(post_fail_backup_dir):
-                shutil.rmtree(post_fail_backup_dir)
-
-        backupdir = self.backup_gitdir(name, gitdir, pre_update_backup_dir)
-
-        try:
-            self.needs_aggressive = False
-            table[vcstype](name, dirname, gitdir, spec)
-            if self.settings['repack']:
-                self.progress('.. repacking %s git repository' % name)
-                self.run_program(['git', 'config',
-                                  'pack.windowMemory', '128M'], cwd=gitdir)
-                args = ['git', 'gc']
-                if self.needs_aggressive:
-                    args += ['--aggressive']
-                self.run_program(args, cwd=gitdir)
-
-            self.bundle(name, gitdir)
-            self.make_tarball(name, gitdir)
-        except:
-            if backupdir is not None:
-                faildir = self.backup_gitdir(name, gitdir, post_fail_backup_dir)
-                self.restore_backup(name, backupdir, gitdir)
-                self.output.write('Mirror of %s failed, state before mirror '
-                                  'is saved at %s and state after mirror is '
-                                  'saved at %s\n' % (name, backupdir, faildir))
-                logging.debug('Mirror of %s failed, state before mirror '
-                              'is saved at %s and state after mirror is '
-                              'saved at %s\n', name, backupdir, faildir)
-            raise
-
-        if not self.settings['pull-only']:
-            if 'refspecs' in spec:
-                self.push_to_mirror_server(name, gitdir, spec['refspecs'])
-            else:
-                self.push_to_mirror_server(name, gitdir)
-
-        if backupdir is not None:
-            self.progress('.. removing %s git repository backup' % name)
-            shutil.rmtree(backupdir)
-
-    def restore_backup(self, name, backupdir, gitdir):
-        self.progress('.. restoring %s good git repository' % name)
-        dotgit = os.path.join(gitdir, '.git')
-        if not os.path.exists(dotgit):
-            dotgit = gitdir
-        shutil.rmtree(dotgit)
-        self.copy_gitdir(backupdir, dotgit)
-
-    def backup_gitdir(self, name, gitdir, backupdir):
-        dotgit = os.path.join(gitdir, '.git')
-        if not os.path.exists(dotgit):
-            dotgit = gitdir
-
-        self.progress('.. backing up %s git repository to %s' % (name, backupdir))
-
-        return self.copy_gitdir(dotgit, backupdir)
-
-    def copy_gitdir(self, source, dest):
-        if not os.path.exists(source):
-            return None
-
-        # copy everything except the objects dir
-        def ignoreobjects(dirname, filenames):
-            if dirname.endswith(source):
-                return ['objects']
-            return []
-        shutil.copytree(source, dest, ignore=ignoreobjects)
-
-        # hardlink the objects
-        sourceobjects = os.path.join(source, 'objects')
-        assert os.path.exists(sourceobjects), "No source objects"
-        objectspath = os.path.join(dest, 'objects')
-        os.mkdir(objectspath)
-        for dirpath, dirnames, filenames in os.walk(sourceobjects):
-            assert dirpath.startswith(sourceobjects)
-            # strip sourceobjects and / from relpath
-            relpath = dirpath[len(sourceobjects)+1:]
-            for dirname in dirnames:
-                os.mkdir(os.path.join(objectspath, relpath, dirname))
-            for filename in filenames:
-                assert os.path.exists(os.path.join(objectspath, relpath))
-                os.link(os.path.join(dirpath, filename),
-                        os.path.join(objectspath, relpath, filename))
-        return dest
-
-    def mirror_git(self, project_name, dirname, gitdir, spec):
-        if not os.path.exists(gitdir):
-            self.progress('.. initialising git dir')
-            self.run_program(['git', 'init', '--bare', gitdir])
-        self.progress('.. updating existing clone')
-        if 'remote.origin.fetch' in self.run_program(['git', 'config', '-l'],
-                                                     cwd=gitdir):
-            self.run_program(['git', 'config', '--unset-all',
-                              'remote.origin.fetch'], cwd=gitdir)
-        self.run_program(['git', 'config', 'remote.origin.url',
-                          spec['url']], cwd=gitdir)
-        self.run_program(['git', 'config', 'remote.origin.mirror', 'true'],
-                         cwd=gitdir)
-        self.run_program(['git', 'config', '--add', 'remote.origin.fetch',
-                          '+refs/heads/*:refs/heads/*'], cwd=gitdir)
-        self.run_program(['git', 'config', '--add', 'remote.origin.fetch',
-                          '+refs/tags/*:refs/tags/*'], cwd=gitdir)
-        try:
-            self.run_program(['git', 'remote', 'update', 'origin', '--prune'],
-                             cwd=gitdir)
-        except:
-            self.run_program(['git', 'remote', 'prune', 'origin'],
-                             cwd=gitdir)
-            self.run_program(['git', 'remote', 'update', 'origin', '--prune'],
-                             cwd=gitdir)
-
-    def gitify_bzr(self, project_name, dirname, gitdir, spec):
-        bzrdir = os.path.join(dirname, 'bzr')
-        # check if repo exists
-        if not os.path.exists(bzrdir):
-            self.progress('.. creating bzr repository')
-            self.run_program(['bzr', 'init-repo', '--no-trees', bzrdir])
-
-        if not os.path.exists(gitdir):
-            self.progress('.. creating git repo')
-            os.mkdir(gitdir)
-            self.run_program(['git', 'init', '--bare', gitdir])
-            self.needs_aggressive = True
-
-        # branches are the listed branches, plus the branch specified in url
-        if 'branches' in spec:
-            branches = spec['branches']
-        else:
-            branches = {}
-        if 'url' in spec:
-            branches['trunk'] = spec['url']
-        logging.debug('all branches: %s' % repr(branches))
-
-        for branch, address in branches.iteritems():
-            branchdir = os.path.join(bzrdir, branch)
-            if not os.path.exists(branchdir):
-                self.progress('.. doing initial bzr branch')
-                self.run_program(['bzr', 'branch', '--quiet', address,
-                                  branchdir])
-            else:
-                self.progress('.. updating bzr branch')
-                self.run_program(['bzr', 'pull', '--quiet', address],
-                                 cwd=branchdir)
-
-        exports = {}
-        bzrmarks = os.path.join(gitdir, 'marks.bzr')
-        for branch, address in branches.iteritems():
-            branchdir = os.path.join(bzrdir, branch)
-            self.progress('.. fast-exporting branch %s from bzr' % branch)
-            exports[branch] = os.path.join(dirname, 'fast-export' + branch)
-            cmdline = ['bzr', 'fast-export', '--git-branch=' + branch, 
-                       branchdir, exports[branch]]
-            if os.path.exists(bzrmarks):
-                cmdline.append('--marks=' + bzrmarks)
-            else:
-                cmdline.append('--export-marks=' + bzrmarks)
-            self.run_program(cmdline)
-
-        gitmarks = os.path.join(gitdir, 'marks.git')
-        for branch, address in branches.iteritems():
-            self.progress('.. fast-importing branch %s into git' % branch)
-            with open(exports[branch], 'rb') as exportfile:
-                cmdline = ['git', 'fast-import', '--export-marks=' + gitmarks]
-                if os.path.exists(gitmarks):
-                    cmdline.append('--import-marks=' + gitmarks)
-                self.run_program(cmdline, stdin=exportfile,
-                                 cwd=gitdir)
-
-        for branch, address in branches.iteritems():
-            branchdir = os.path.join(bzrdir, branch)
-            self.progress('.. removing temporary fast-export file ' + 
-                            exports[branch])
-            os.remove(exports[branch])
-
-    def gitify_svn(self, project_name, dirname, gitdir, spec):
-        if not os.path.exists(gitdir):
-            self.progress('.. doing initial clone')
-            self.needs_aggressive = True
-            layout = spec["layout"]
-            # if standard layour specified, fill in the defaults
-            if layout == "standard":
-                layout = { "trunk": "trunk",
-                           "tags": "tags/*",
-                           "branches": "branches/*" }
-            # init the repo then manually set the refspecs to fetch into local
-            # git-svn can apparently provide better history tracking by
-            # fetching the root of the repository
-            # git-svn will convert branch, trunk and tag paths to allow this,
-            # but it is simpler to disable it and do it manually
-            self.run_program(['git', 'svn', 'init', spec['url'], gitdir + "-tmp",
-                              '--svn-remote=svn', '--no-minimize-url'])
-            os.rename(os.path.join(gitdir + "-tmp", '.git'), gitdir)
-            os.rmdir(gitdir + "-tmp")
-            self.run_program(['git', 'config', 'core.bare', 'true'],
-                             cwd=gitdir)
-            self.run_program(['git', 'config', 'svn-remote.svn.fetch',
-                              layout["trunk"]+':refs/heads/master'], 
-                             cwd=gitdir)
-            self.run_program(['git', 'config', 'svn-remote.svn.branches',
-                              layout["branches"] + ':refs/heads/*'], 
-                             cwd=gitdir)
-            self.run_program(['git', 'config', 'svn-remote.svn.tags',
-                              layout["tags"] + ':refs/tags/*'],
-                             cwd=gitdir)
-        else:
-            self.progress('.. updating existing clone')
-        # update the remote tracking branches
-        self.run_program(['git', 'svn', 'fetch'], cwd=gitdir)
-
-    def gitify_cvs(self, project_name, dirname, gitdir, spec):
-        self.needs_aggressive = True
-        self.run_program(['git', 'cvsimport', '-a', '-d', spec['url'],
-                          '-C', gitdir, spec['module']])
-
-    def gitify_hg(self, project_name, dirname, gitdir, spec):
-        hgdir = os.path.join(dirname, 'hg')
-        if os.path.exists(hgdir):
-            self.progress('.. updating hg branch')
-            self.run_program(['hg', 'pull', '--quiet', '--insecure'], cwd=hgdir)
-        else:
-            self.progress('.. doing initial hg branch')
-            self.run_program(['hg', 'clone', '--quiet', '--insecure', spec['url'], hgdir])
-
-        if not os.path.exists(gitdir):
-            self.needs_aggressive = True
-            self.run_program(['git', 'init', '--bare', gitdir])
-        
-        self.progress('.. fast-exporting into git')
-        self.run_program(['hg-fast-export', '--quiet', '--force', '-r', '../hg'], 
-                         cwd=gitdir)
-
-    def gitify_tarball(self, project_name, dirname, gitdir, spec):
-        url = spec['url']
-        url_path = urllib2.urlparse.urlparse(url)[2]
-        basename = os.path.basename(url_path)
-        tardest = os.path.join(dirname, basename)
-        self.progress('.. checking if we need to fetch %s' % basename)
-        if not os.path.exists(tardest):
-            self.progress('.. attempting to fetch.')
-            with open(tardest, 'w') as tarfile:
-                urlfile = urllib2.urlopen(spec['url'])
-                tarfile.write(urlfile.read())
-                urlfile.close()
-        else:
-            self.progress('.. no need to run, nothing to do')
-            return
-        if not os.path.exists(gitdir):
-            self.run_program(['git', 'init', '--bare', gitdir])
-        cmdline = ["%s.tar-importer" % lorry_path, tardest]
-        self.run_program(cmdline, cwd=gitdir)
-        self.needs_aggressive = True
-            
-
-    def push_to_mirror_server(self, name, gitdir,
-                              pushrefspecs=['refs/heads/*:refs/heads/*',
-                                            'refs/tags/*:refs/tags/*']):
-        pushurl = "%s/%s.git" % (self.settings['mirror-base-url-push'], name)
-        self.progress('.. pushing %s to mirror server %s' % (name, pushurl))
-        self.run_program(['git', 'push', pushurl]+pushrefspecs, cwd=gitdir)
-
-    def run_program(self, argv, **kwargs):
-        if self.settings['command-stdout']:
-            kwargs['stdout'] = open(self.settings['command-stdout'], 'a')
-        if self.settings['command-stderr']:
-            kwargs['stderr'] = open(self.settings['command-stderr'], 'a')
-        if 'stdin' not in kwargs:
-            kwargs['stdin'] = open('/dev/null', 'r')
-        logging.debug('Running: argv=%s kwargs=%s' % 
-                        (repr(argv), repr(kwargs)))
-        exit, out, err = self.runcmd_unchecked(argv, **kwargs)
-        logging.debug('Command: %s\nExit: %s\nStdout:\n%sStderr:\n%s' % 
-                      (argv, exit, self.indent(out or ''),
-                       self.indent(err or '')))
-        if exit != 0:
-            raise Exception('%s failed (exit code %s):\n%s' % 
-                                (' '.join(argv), exit, self.indent(err or '')))
-        return out
-
-    def indent(self, string):
-        return ''.join('    %s\n' % line for line in string.splitlines())
-    
-    def dirname(self, project_name):
-        assert '\0' not in project_name
-        # We escape slashes as underscores.
-        project_name = '_'.join(project_name.split('/'))
-        return os.path.join(self.settings['working-area'], project_name)
-
-    def progress(self, msg):
-        logging.debug(msg)
-        if self.settings['verbose']:
-            self.output.write('%s\n' % msg)
-
-
-if __name__ == '__main__':
-    Lorry(version=__version__).run()
-