From ec09b9d5476fa313d1b43ee02bc773d195ef98e8 Mon Sep 17 00:00:00 2001 From: Richard Maw Date: Fri, 10 Oct 2014 21:20:30 +0100 Subject: Remove queueing Artifact resolution is no longer recursive, so there's no need to requeue sources, removing this queue provides a significant speed up. Fixing an issue where artifact resoltuion was taking too long on some systems (several minutes in some cases). --- morphlib/artifactresolver.py | 22 ++++++---------------- 1 file changed, 6 insertions(+), 16 deletions(-) diff --git a/morphlib/artifactresolver.py b/morphlib/artifactresolver.py index 8c8b37d0..168c122c 100644 --- a/morphlib/artifactresolver.py +++ b/morphlib/artifactresolver.py @@ -82,7 +82,7 @@ class ArtifactResolver(object): self._added_artifacts.add(system) resolved_artifacts = self._resolve_system_dependencies( - systems, source, queue) + systems, source) for artifact in resolved_artifacts: if not artifact in self._added_artifacts: @@ -105,7 +105,7 @@ class ArtifactResolver(object): self._added_artifacts.add(stratum) resolved_artifacts = self._resolve_stratum_dependencies( - strata, source, queue) + strata, source) for artifact in resolved_artifacts: if not artifact in self._added_artifacts: @@ -126,15 +126,9 @@ class ArtifactResolver(object): return artifacts def _create_initial_queue(self): - if all([x.morphology['kind'] == 'chunk' for x in self._source_pool]): - return collections.deque(self._source_pool) - else: - sources = [x for x in self._source_pool - if x.morphology['kind'] != 'chunk'] - return collections.deque(sources) - - def _resolve_system_dependencies(self, systems, - source, queue): # pragma: no cover + return collections.deque(self._source_pool) + + def _resolve_system_dependencies(self, systems, source): # pragma: no cover artifacts = [] for info in source.morphology['strata']: @@ -156,11 +150,9 @@ class ArtifactResolver(object): source.add_dependency(stratum_artifact) artifacts.append(stratum_artifact) - queue.append(stratum_source) - return artifacts - def _resolve_stratum_dependencies(self, strata, source, queue): + def _resolve_stratum_dependencies(self, strata, source): artifacts = [] stratum_build_depends = [] @@ -191,8 +183,6 @@ class ArtifactResolver(object): source.add_dependency(other_stratum) - queue.append(other_source) - # 'name' here is the chunk artifact name name_to_processed_artifacts = {} -- cgit v1.2.1 From 97f5b7fdfc7cb89748ce477653aae5c54c16668b Mon Sep 17 00:00:00 2001 From: Richard Ipsum Date: Fri, 10 Oct 2014 21:14:01 +0100 Subject: Replace resolver queue with loop Since we're no longer adding anything to the queue we can replace it with a loop. This patch also splits the code up a little bit. --- morphlib/artifactresolver.py | 119 +++++++++++++++++++++---------------------- 1 file changed, 57 insertions(+), 62 deletions(-) diff --git a/morphlib/artifactresolver.py b/morphlib/artifactresolver.py index 168c122c..30fb5e5a 100644 --- a/morphlib/artifactresolver.py +++ b/morphlib/artifactresolver.py @@ -56,78 +56,74 @@ class ArtifactResolver(object): return [a for a in self._resolve_artifacts(source_pool) if not a.dependents] + def _resolve_system_artifacts(self, source, artifacts): # pragma: no cover + systems = [source.artifacts[name] + for name in source.split_rules.artifacts] + + for system in (s for s in systems if s not in self._added_artifacts): + artifacts.append(system) + self._added_artifacts.add(system) + + resolved_artifacts = self._resolve_system_dependencies(systems, source) + + for artifact in resolved_artifacts: + if not artifact in self._added_artifacts: + artifacts.append(artifact) + self._added_artifacts.add(artifact) + + def _resolve_stratum_artifacts(self, source, artifacts): + # Iterate split_rules.artifacts, rather than + # artifacts.values() to preserve ordering + strata = [source.artifacts[name] + for name in source.split_rules.artifacts + if name in source.artifacts] + + # If we were not given systems, return the strata here, + # rather than have the systems return them. + if not any(s.morphology['kind'] == 'system' + for s in self._source_pool): + for stratum in (s for s in strata + if s not in self._added_artifacts): + artifacts.append(stratum) + self._added_artifacts.add(stratum) + + resolved_artifacts = self._resolve_stratum_dependencies( + strata, source) + + for artifact in resolved_artifacts: + if not artifact in self._added_artifacts: + artifacts.append(artifact) + self._added_artifacts.add(artifact) + + def _resolve_chunk_artifacts(self, source, artifacts): + chunks = [source.artifacts[name] + for name in source.split_rules.artifacts] + # If we were only given chunks, return them here, rather than + # have the strata return them. + if not any(s.morphology['kind'] == 'stratum' + for s in self._source_pool): + for chunk in (c for c in chunks + if c not in self._added_artifacts): + artifacts.append(chunk) + self._added_artifacts.add(chunk) + def _resolve_artifacts(self, source_pool): self._source_pool = source_pool self._added_artifacts = set() + artifacts = [] - artifacts = self._resolve_artifacts_recursively() # TODO perform cycle detection, e.g. based on: # http://stackoverflow.com/questions/546655/finding-all-cycles-in-graph - return artifacts - def _resolve_artifacts_recursively(self): - artifacts = [] + resolvers = {'system': self._resolve_system_artifacts, + 'stratum': self._resolve_stratum_artifacts, + 'chunk': self._resolve_chunk_artifacts} - queue = self._create_initial_queue() - while queue: - source = queue.popleft() - - if source.morphology['kind'] == 'system': # pragma: no cover - systems = [source.artifacts[name] - for name in source.split_rules.artifacts] - - for system in (s for s in systems - if s not in self._added_artifacts): - artifacts.append(system) - self._added_artifacts.add(system) - - resolved_artifacts = self._resolve_system_dependencies( - systems, source) - - for artifact in resolved_artifacts: - if not artifact in self._added_artifacts: - artifacts.append(artifact) - self._added_artifacts.add(artifact) - elif source.morphology['kind'] == 'stratum': - # Iterate split_rules.artifacts, rather than - # artifacts.values() to preserve ordering - strata = [source.artifacts[name] - for name in source.split_rules.artifacts - if name in source.artifacts] - - # If we were not given systems, return the strata here, - # rather than have the systems return them. - if not any(s.morphology['kind'] == 'system' - for s in self._source_pool): - for stratum in (s for s in strata - if s not in self._added_artifacts): - artifacts.append(stratum) - self._added_artifacts.add(stratum) - - resolved_artifacts = self._resolve_stratum_dependencies( - strata, source) - - for artifact in resolved_artifacts: - if not artifact in self._added_artifacts: - artifacts.append(artifact) - self._added_artifacts.add(artifact) - elif source.morphology['kind'] == 'chunk': - chunks = [source.artifacts[name] - for name in source.split_rules.artifacts] - # If we were only given chunks, return them here, rather than - # have the strata return them. - if not any(s.morphology['kind'] == 'stratum' - for s in self._source_pool): - for chunk in (c for c in chunks - if c not in self._added_artifacts): - artifacts.append(chunk) - self._added_artifacts.add(chunk) + for source in self._source_pool: + resolvers[source.morphology['kind']](source, artifacts) return artifacts - def _create_initial_queue(self): - return collections.deque(self._source_pool) - def _resolve_system_dependencies(self, systems, source): # pragma: no cover artifacts = [] @@ -227,7 +223,6 @@ class ArtifactResolver(object): if chunk_artifact not in artifacts: artifacts.append(chunk_artifact) - # Add these chunks to the processed artifacts, so other # chunks may refer to them. name_to_processed_artifacts[info['name']] = \ -- cgit v1.2.1 From 639bca2a58e872238e991311676f448065093ea2 Mon Sep 17 00:00:00 2001 From: Richard Ipsum Date: Fri, 10 Oct 2014 20:03:17 +0100 Subject: Simplify _resolve_system_artifacts --- morphlib/artifactresolver.py | 16 ++++------------ 1 file changed, 4 insertions(+), 12 deletions(-) diff --git a/morphlib/artifactresolver.py b/morphlib/artifactresolver.py index 30fb5e5a..e53c7511 100644 --- a/morphlib/artifactresolver.py +++ b/morphlib/artifactresolver.py @@ -57,19 +57,11 @@ class ArtifactResolver(object): if not a.dependents] def _resolve_system_artifacts(self, source, artifacts): # pragma: no cover - systems = [source.artifacts[name] - for name in source.split_rules.artifacts] + systems = [source.artifacts[name] for name + in source.split_rules.artifacts] - for system in (s for s in systems if s not in self._added_artifacts): - artifacts.append(system) - self._added_artifacts.add(system) - - resolved_artifacts = self._resolve_system_dependencies(systems, source) - - for artifact in resolved_artifacts: - if not artifact in self._added_artifacts: - artifacts.append(artifact) - self._added_artifacts.add(artifact) + artifacts += systems + artifacts += self._resolve_system_dependencies(systems, source) def _resolve_stratum_artifacts(self, source, artifacts): # Iterate split_rules.artifacts, rather than -- cgit v1.2.1