summaryrefslogtreecommitdiff
path: root/morphlib/repos.py
diff options
context:
space:
mode:
Diffstat (limited to 'morphlib/repos.py')
-rw-r--r--morphlib/repos.py202
1 files changed, 202 insertions, 0 deletions
diff --git a/morphlib/repos.py b/morphlib/repos.py
new file mode 100644
index 00000000..a2c5967b
--- /dev/null
+++ b/morphlib/repos.py
@@ -0,0 +1,202 @@
+# Copyright (C) 2014 Codethink Limited
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation; version 2 of the License.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License along
+# with this program; if not, write to the Free Software Foundation, Inc.,
+# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
+#
+# =*= License: GPL-2 =*=
+
+import os
+import app
+import re
+from subprocess import call
+from subprocess import check_output
+import string
+import definitions
+import urllib2
+import json
+
+
+def get_repo_url(this):
+ url = this['repo']
+ url = url.replace('upstream:', 'git://git.baserock.org/delta/')
+ url = url.replace('baserock:baserock/',
+ 'git://git.baserock.org/baserock/baserock/')
+ url = url.replace('freedesktop:', 'git://anongit.freedesktop.org/')
+ url = url.replace('github:', 'git://github.com/')
+ url = url.replace('gnome:', 'git://git.gnome.org')
+ if url.endswith('.git'):
+ url = url[:-4]
+ return url
+
+
+def quote_url(url):
+ ''' Convert URIs to strings that only contain digits, letters, % and _.
+
+ NOTE: When changing the code of this function, make sure to also apply
+ the same to the quote_url() function of lorry. Otherwise the git tarballs
+ generated by lorry may no longer be found by morph.
+
+ '''
+ valid_chars = string.digits + string.ascii_letters + '%_'
+ transl = lambda x: x if x in valid_chars else '_'
+ return ''.join([transl(x) for x in url])
+
+
+def get_repo_name(this):
+ return quote_url(get_repo_url(this))
+# return re.split('[:/]', this['repo'])[-1]
+
+
+def get_tree(this):
+ defs = definitions.Definitions()
+
+ if defs.lookup(this, 'repo') == []:
+ return None
+
+ if defs.lookup(this, 'git') == []:
+ this['git'] = (os.path.join(app.settings['gits'],
+ get_repo_name(this)))
+
+ ref = defs.lookup(this, 'ref')
+ if not os.path.exists(this['git']):
+ try:
+ url = (app.settings['cache-server-url'] + 'repo='
+ + get_repo_url(this) + '&ref=' + ref)
+ with urllib2.urlopen(url) as response:
+ tree = json.loads(response.read().decode())['tree']
+ return tree
+ except Exception, e:
+ yapp.log(this, 'WARNING: no tree from cache-server', tree)
+ mirror(this)
+
+ with app.chdir(this['git']), open(os.devnull, "w") as fnull:
+ if call(['git', 'rev-parse', ref + '^{object}'], stdout=fnull,
+ stderr=fnull):
+ # can't resolve this ref. is it upstream?
+ call(['git', 'fetch', 'origin'], stdout=fnull, stderr=fnull)
+
+ try:
+ tree = check_output(['git', 'rev-parse', ref + '^{tree}'],
+ universal_newlines=True)[0:-1]
+ return tree
+
+ except Exception, e:
+ # either we don't have a git dir, or ref is not unique
+ # or ref does not exist
+
+ yapp.log(this, 'ERROR: could not find tree for ref', ref)
+ raise SystemExit
+
+
+def copy_repo(repo, destdir):
+ '''Copies a cached repository into a directory using cp.
+
+ This also fixes up the repository afterwards, so that it can contain
+ code etc. It does not leave any given branch ready for use.
+
+ '''
+
+ # core.bare should be false so that git believes work trees are possible
+ # we do not want the origin remote to behave as a mirror for pulls
+ # we want a traditional refs/heads -> refs/remotes/origin ref mapping
+ # set the origin url to the cached repo so that we can quickly clean up
+ # by packing the refs, we can then edit then en-masse easily
+ call(['cp', '-a', repo, os.path.join(destdir, '.git')])
+ call(['git', 'config', 'core.bare', 'false'])
+ call(['git', 'config', '--unset', 'remote.origin.mirror'])
+ with open(os.devnull, "w") as fnull:
+ call(['git', 'config', 'remote.origin.fetch',
+ '+refs/heads/*:refs/remotes/origin/*'],
+ stdout=fnull,
+ stderr=fnull)
+ call(['git', 'config', 'remote.origin.url', repo])
+ call(['git', 'pack-refs', '--all', '--prune'])
+
+ # turn refs/heads/* into refs/remotes/origin/* in the packed refs
+ # so that the new copy behaves more like a traditional clone.
+ with open(os.path.join(destdir, ".git", "packed-refs"), "r") as ref_fh:
+ pack_lines = ref_fh.read().split("\n")
+ with open(os.path.join(destdir, ".git", "packed-refs"), "w") as ref_fh:
+ ref_fh.write(pack_lines.pop(0) + "\n")
+ for refline in pack_lines:
+ if ' refs/remotes/' in refline:
+ continue
+ if ' refs/heads/' in refline:
+ sha, ref = refline[:40], refline[41:]
+ if ref.startswith("refs/heads/"):
+ ref = "refs/remotes/origin/" + ref[11:]
+ refline = "%s %s" % (sha, ref)
+ ref_fh.write("%s\n" % (refline))
+ # Finally run a remote update to clear up the refs ready for use.
+ with open(os.devnull, "w") as fnull:
+ call(['git', 'remote', 'update', 'origin', '--prune'],
+ stdout=fnull,
+ stderr=fnull)
+
+
+def mirror(this):
+ # try tarball first
+ try:
+ os.makedirs(this['git'])
+ with app.chdir(this['git']):
+ yapp.log(this, 'Try fetching tarball')
+ repo_url = get_repo_url(this)
+ tar_file = quote_url(repo_url) + '.tar'
+ tar_url = os.path.join("http://git.baserock.org/tarballs",
+ tar_file)
+ with open(os.devnull, "w") as fnull:
+ call(['wget', tar_url], stdout=fnull, stderr=fnull)
+ call(['tar', 'xf', tar_file], stdout=fnull, stderr=fnull)
+ os.remove(tar_file)
+ call(['git', 'config', 'remote.origin.url', repo_url],
+ stdout=fnull, stderr=fnull)
+ call(['git', 'config', 'remote.origin.mirror', 'true'],
+ stdout=fnull, stderr=fnull)
+ if call(['git', 'config', 'remote.origin.fetch',
+ '+refs/*:refs/*'],
+ stdout=fnull, stderr=fnull) != 0:
+ raise BaseException('Did not get a valid git repo')
+ call(['git', 'fetch', 'origin'], stdout=fnull, stderr=fnull)
+ except Exception, e:
+ yapp.log(this, 'Using git clone', get_repo_url(this))
+ try:
+ with open(os.devnull, "w") as fnull:
+ call(['git', 'clone', '--mirror', '-n', get_repo_url(this),
+ this['git']], stdout=fnull, stderr=fnull)
+ except Exception, e:
+ yapp.log(this, 'ERROR: failed to clone', get_repo_url(this))
+ raise SystemExit
+
+ yapp.log(this, 'Git repo is mirrored at', this['git'])
+
+
+def fetch(repo):
+ with app.chdir(repo), open(os.devnull, "w") as fnull:
+ call(['git', 'fetch', 'origin'], stdout=fnull, stderr=fnull)
+
+
+def checkout(this):
+ # checkout the required version of this from git
+ with app.chdir(this['build']):
+ yapp.log(this, 'Git checkout')
+ if not this.get('git'):
+ this['git'] = (os.path.join(app.settings['gits'],
+ get_repo_name(this)))
+ if not os.path.exists(this['git']):
+ mirror(this)
+ copy_repo(this['git'], this['build'])
+ with open(os.devnull, "w") as fnull:
+ if call(['git', 'checkout', this['ref']],
+ stdout=fnull, stderr=fnull) != 0:
+ yapp.log(this, 'ERROR: git checkout failed for', this['tree'])
+ raise SystemExit