summaryrefslogtreecommitdiff
diff options
context:
space:
mode:
authorSam Thursfield <sam.thursfield@codethink.co.uk>2014-10-23 15:16:07 +0100
committerSam Thursfield <sam.thursfield@codethink.co.uk>2014-11-10 12:31:03 +0000
commit2a9c29502a76a54aba3cc92c6e617abbe41ba1ee (patch)
tree3232e99d32eb4a7263b84171f4cc18575aa3fa24
parent8f47d014bd8946acc67e6fd291a91028ba6c6a08 (diff)
downloadmorph-2a9c29502a76a54aba3cc92c6e617abbe41ba1ee.tar.gz
Move create_source_pool code into new 'sourceresolver' module
This code is an essential part of 'morph build'. It's quite complex and really shouldn't be mixed in with the base Application class. Given a dedicated class we can store some state in the object and avoid functions with seven parameters, too.
-rw-r--r--morphlib/__init__.py1
-rw-r--r--morphlib/app.py131
-rw-r--r--morphlib/buildcommand.py7
-rw-r--r--morphlib/plugins/list_artifacts_plugin.py6
-rw-r--r--morphlib/sourceresolver.py176
-rw-r--r--without-test-modules4
6 files changed, 189 insertions, 136 deletions
diff --git a/morphlib/__init__.py b/morphlib/__init__.py
index f98c11aa..8319430c 100644
--- a/morphlib/__init__.py
+++ b/morphlib/__init__.py
@@ -79,6 +79,7 @@ import repoaliasresolver
import savefile
import source
import sourcepool
+import sourceresolver
import stagingarea
import stopwatch
import sysbranchdir
diff --git a/morphlib/app.py b/morphlib/app.py
index c4d45aae..3f4171a4 100644
--- a/morphlib/app.py
+++ b/morphlib/app.py
@@ -15,13 +15,11 @@
import cliapp
-import collections
import logging
import os
import sys
import time
import urlparse
-import warnings
import extensions
import morphlib
@@ -290,135 +288,6 @@ class Morph(cliapp.Application):
morphlib.util.sanitise_morphology_path(args[2]))
args = args[3:]
- def create_source_pool(self, lrc, rrc, repo, ref, filename,
- original_ref=None):
- pool = morphlib.sourcepool.SourcePool()
-
- def add_to_pool(reponame, ref, filename, absref, tree, morphology):
- sources = morphlib.source.make_sources(reponame, ref,
- filename, absref,
- tree, morphology)
- for source in sources:
- pool.add(source)
-
- self.traverse_morphs(repo, ref, [filename], lrc, rrc,
- update=not self.settings['no-git-update'],
- visit=add_to_pool,
- definitions_original_ref=original_ref)
- return pool
-
- def resolve_ref(self, lrc, rrc, reponame, ref, update=True):
- '''Resolves commit and tree sha1s of the ref in a repo and returns it.
-
- If update is True then this has the side-effect of updating
- or cloning the repository into the local repo cache.
- '''
- absref = None
-
- if lrc.has_repo(reponame):
- repo = lrc.get_repo(reponame)
- if update and repo.requires_update_for_ref(ref):
- self.status(msg='Updating cached git repository %(reponame)s '
- 'for ref %(ref)s', reponame=reponame, ref=ref)
- repo.update()
- # If the user passed --no-git-update, and the ref is a SHA1 not
- # available locally, this call will raise an exception.
- absref, tree = repo.resolve_ref(ref)
- elif rrc is not None:
- try:
- absref, tree = rrc.resolve_ref(reponame, ref)
- if absref is not None:
- self.status(msg='Resolved %(reponame)s %(ref)s via remote '
- 'repo cache',
- reponame=reponame,
- ref=ref,
- chatty=True)
- except BaseException, e:
- logging.warning('Caught (and ignored) exception: %s' % str(e))
- if absref is None:
- if update:
- self.status(msg='Caching git repository %(reponame)s',
- reponame=reponame)
- repo = lrc.cache_repo(reponame)
- repo.update()
- else:
- repo = lrc.get_repo(reponame)
- absref, tree = repo.resolve_ref(ref)
- return absref, tree
-
- def traverse_morphs(self, definitions_repo, definitions_ref,
- system_filenames, lrc, rrc, update=True,
- visit=lambda rn, rf, fn, arf, m: None,
- definitions_original_ref=None):
- morph_factory = morphlib.morphologyfactory.MorphologyFactory(
- lrc, rrc, self.status)
- definitions_queue = collections.deque(system_filenames)
- chunk_in_definitions_repo_queue = []
- chunk_in_source_repo_queue = []
- resolved_refs = {}
- resolved_morphologies = {}
-
- # Resolve the (repo, ref) pair for the definitions repo, cache result.
- definitions_absref, definitions_tree = self.resolve_ref(
- lrc, rrc, definitions_repo, definitions_ref, update)
-
- if definitions_original_ref:
- definitions_ref = definitions_original_ref
-
- while definitions_queue:
- filename = definitions_queue.popleft()
-
- key = (definitions_repo, definitions_absref, filename)
- if not key in resolved_morphologies:
- resolved_morphologies[key] = morph_factory.get_morphology(*key)
- morphology = resolved_morphologies[key]
-
- visit(definitions_repo, definitions_ref, filename,
- definitions_absref, definitions_tree, morphology)
- if morphology['kind'] == 'cluster':
- raise cliapp.AppException(
- "Cannot build a morphology of type 'cluster'.")
- elif morphology['kind'] == 'system':
- definitions_queue.extend(
- morphlib.util.sanitise_morphology_path(s['morph'])
- for s in morphology['strata'])
- elif morphology['kind'] == 'stratum':
- if morphology['build-depends']:
- definitions_queue.extend(
- morphlib.util.sanitise_morphology_path(s['morph'])
- for s in morphology['build-depends'])
- for c in morphology['chunks']:
- if 'morph' not in c:
- path = morphlib.util.sanitise_morphology_path(
- c.get('morph', c['name']))
- chunk_in_source_repo_queue.append(
- (c['repo'], c['ref'], path))
- continue
- chunk_in_definitions_repo_queue.append(
- (c['repo'], c['ref'], c['morph']))
-
- for repo, ref, filename in chunk_in_definitions_repo_queue:
- if (repo, ref) not in resolved_refs:
- resolved_refs[repo, ref] = self.resolve_ref(
- lrc, rrc, repo, ref, update)
- absref, tree = resolved_refs[repo, ref]
- key = (definitions_repo, definitions_absref, filename)
- if not key in resolved_morphologies:
- resolved_morphologies[key] = morph_factory.get_morphology(*key)
- morphology = resolved_morphologies[key]
- visit(repo, ref, filename, absref, tree, morphology)
-
- for repo, ref, filename in chunk_in_source_repo_queue:
- if (repo, ref) not in resolved_refs:
- resolved_refs[repo, ref] = self.resolve_ref(
- lrc, rrc, repo, ref, update)
- absref, tree = resolved_refs[repo, ref]
- key = (repo, absref, filename)
- if key not in resolved_morphologies:
- resolved_morphologies[key] = morph_factory.get_morphology(*key)
- morphology = resolved_morphologies[key]
- visit(repo, ref, filename, absref, tree, morphology)
-
def cache_repo_and_submodules(self, cache, url, ref, done):
subs_to_process = set()
subs_to_process.add((url, ref))
diff --git a/morphlib/buildcommand.py b/morphlib/buildcommand.py
index 438badb3..a658fc55 100644
--- a/morphlib/buildcommand.py
+++ b/morphlib/buildcommand.py
@@ -94,10 +94,11 @@ class BuildCommand(object):
'''
self.app.status(msg='Creating source pool', chatty=True)
- srcpool = self.app.create_source_pool(
+ srcpool = morphlib.sourceresolver.create_source_pool(
self.lrc, self.rrc, repo_name, ref, filename,
- original_ref=original_ref)
-
+ original_ref=original_ref,
+ update_repos=not self.app.settings['no-git-update'],
+ status_cb=self.app.status)
return srcpool
def validate_sources(self, srcpool):
diff --git a/morphlib/plugins/list_artifacts_plugin.py b/morphlib/plugins/list_artifacts_plugin.py
index 0f14a579..713473ac 100644
--- a/morphlib/plugins/list_artifacts_plugin.py
+++ b/morphlib/plugins/list_artifacts_plugin.py
@@ -84,8 +84,10 @@ class ListArtifactsPlugin(cliapp.Plugin):
self.app.status(
msg='Creating source pool for %s' % system_filename, chatty=True)
- source_pool = self.app.create_source_pool(
- self.lrc, self.rrc, repo, ref, system_filename)
+ source_pool = morphlib.SourceResolver.create_source_pool(
+ self.lrc, self.rrc, repo, ref, system_filename,
+ update_repos = not self.app.settings['no-git-update'],
+ status_cb=self.app.status)
self.app.status(
msg='Resolving artifacts for %s' % system_filename, chatty=True)
diff --git a/morphlib/sourceresolver.py b/morphlib/sourceresolver.py
new file mode 100644
index 00000000..dee16ea1
--- /dev/null
+++ b/morphlib/sourceresolver.py
@@ -0,0 +1,176 @@
+# Copyright (C) 2014 Codethink Limited
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation; version 2 of the License.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License along
+# with this program; if not, write to the Free Software Foundation, Inc.,
+# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
+
+
+import cliapp
+
+import collections
+import logging
+
+import morphlib
+
+
+class SourceResolver(object):
+ '''Provides a way of resolving the set of sources for a given system.'''
+
+ def __init__(self, lrc, rrc, status_cb):
+ self.lrc = lrc
+ self.rrc = rrc
+
+ self.status = status_cb
+
+ def resolve_ref(self, reponame, ref, update=True):
+ '''Resolves commit and tree sha1s of the ref in a repo and returns it.
+
+ If update is True then this has the side-effect of updating
+ or cloning the repository into the local repo cache.
+ '''
+ absref = None
+
+ if self.lrc.has_repo(reponame):
+ repo = self.lrc.get_repo(reponame)
+ if update and repo.requires_update_for_ref(ref):
+ self.status(msg='Updating cached git repository %(reponame)s '
+ 'for ref %(ref)s', reponame=reponame, ref=ref)
+ repo.update()
+ # If the user passed --no-git-update, and the ref is a SHA1 not
+ # available locally, this call will raise an exception.
+ absref, tree = repo.resolve_ref(ref)
+ elif self.rrc is not None:
+ try:
+ absref, tree = self.rrc.resolve_ref(reponame, ref)
+ if absref is not None:
+ self.status(msg='Resolved %(reponame)s %(ref)s via remote '
+ 'repo cache',
+ reponame=reponame,
+ ref=ref,
+ chatty=True)
+ except BaseException, e:
+ logging.warning('Caught (and ignored) exception: %s' % str(e))
+ if absref is None:
+ if update:
+ self.status(msg='Caching git repository %(reponame)s',
+ reponame=reponame)
+ repo = self.lrc.cache_repo(reponame)
+ repo.update()
+ else:
+ repo = self.lrc.get_repo(reponame)
+ absref, tree = repo.resolve_ref(ref)
+ return absref, tree
+
+ def traverse_morphs(self, definitions_repo, definitions_ref,
+ system_filenames, update=True,
+ visit=lambda rn, rf, fn, arf, m: None,
+ definitions_original_ref=None):
+ morph_factory = morphlib.morphologyfactory.MorphologyFactory(
+ self.lrc, self.rrc, self.status)
+ definitions_queue = collections.deque(system_filenames)
+ chunk_in_definitions_repo_queue = []
+ chunk_in_source_repo_queue = []
+ resolved_refs = {}
+ resolved_morphologies = {}
+
+ # Resolve the (repo, ref) pair for the definitions repo, cache result.
+ definitions_absref, definitions_tree = self.resolve_ref(
+ definitions_repo, definitions_ref, update)
+
+ if definitions_original_ref:
+ definitions_ref = definitions_original_ref
+
+ while definitions_queue:
+ filename = definitions_queue.popleft()
+
+ key = (definitions_repo, definitions_absref, filename)
+ if not key in resolved_morphologies:
+ resolved_morphologies[key] = morph_factory.get_morphology(*key)
+ morphology = resolved_morphologies[key]
+
+ visit(definitions_repo, definitions_ref, filename,
+ definitions_absref, definitions_tree, morphology)
+ if morphology['kind'] == 'cluster':
+ raise cliapp.AppException(
+ "Cannot build a morphology of type 'cluster'.")
+ elif morphology['kind'] == 'system':
+ definitions_queue.extend(
+ morphlib.util.sanitise_morphology_path(s['morph'])
+ for s in morphology['strata'])
+ elif morphology['kind'] == 'stratum':
+ if morphology['build-depends']:
+ definitions_queue.extend(
+ morphlib.util.sanitise_morphology_path(s['morph'])
+ for s in morphology['build-depends'])
+ for c in morphology['chunks']:
+ if 'morph' not in c:
+ path = morphlib.util.sanitise_morphology_path(
+ c.get('morph', c['name']))
+ chunk_in_source_repo_queue.append(
+ (c['repo'], c['ref'], path))
+ continue
+ chunk_in_definitions_repo_queue.append(
+ (c['repo'], c['ref'], c['morph']))
+
+ for repo, ref, filename in chunk_in_definitions_repo_queue:
+ if (repo, ref) not in resolved_refs:
+ resolved_refs[repo, ref] = self.resolve_ref(repo, ref, update)
+ absref, tree = resolved_refs[repo, ref]
+ key = (definitions_repo, definitions_absref, filename)
+ if not key in resolved_morphologies:
+ resolved_morphologies[key] = morph_factory.get_morphology(*key)
+ morphology = resolved_morphologies[key]
+ visit(repo, ref, filename, absref, tree, morphology)
+
+ for repo, ref, filename in chunk_in_source_repo_queue:
+ if (repo, ref) not in resolved_refs:
+ resolved_refs[repo, ref] = self.resolve_ref(repo, ref, update)
+ absref, tree = resolved_refs[repo, ref]
+ key = (repo, absref, filename)
+ if key not in resolved_morphologies:
+ resolved_morphologies[key] = morph_factory.get_morphology(*key)
+ morphology = resolved_morphologies[key]
+ visit(repo, ref, filename, absref, tree, morphology)
+
+
+def create_source_pool(lrc, rrc, repo, ref, filename,
+ original_ref=None, update_repos=True,
+ status_cb=None):
+ '''Find all the sources involved in building a given system.
+
+ Given a system morphology, this function will traverse the tree of stratum
+ and chunk morphologies that the system points to and create appropriate
+ Source objects. These are added to a new SourcePool object, which is
+ returned.
+
+ Note that Git submodules are not considered 'sources' in the current
+ implementation, and so they must be handled separately.
+
+ The 'lrc' and 'rrc' parameters specify the local and remote Git repository
+ caches used for resolving the sources.
+
+ '''
+ pool = morphlib.sourcepool.SourcePool()
+
+ def add_to_pool(reponame, ref, filename, absref, tree, morphology):
+ sources = morphlib.source.make_sources(reponame, ref,
+ filename, absref,
+ tree, morphology)
+ for source in sources:
+ pool.add(source)
+
+ resolver = SourceResolver(lrc, rrc, status_cb)
+ resolver.traverse_morphs(repo, ref, [filename],
+ update=update_repos,
+ visit=add_to_pool,
+ definitions_original_ref=original_ref)
+ return pool
diff --git a/without-test-modules b/without-test-modules
index 55e5291d..530deb4f 100644
--- a/without-test-modules
+++ b/without-test-modules
@@ -52,3 +52,7 @@ distbuild/timer_event_source.py
distbuild/worker_build_scheduler.py
# Not unit tested, since it needs a full system branch
morphlib/buildbranch.py
+
+# Requires rather a lot of fake data in order to be unit tested; better to
+# leave it to the functional tests.
+morphlib/sourceresolver.py