diff options
| author | Adam Coldrick <adam.coldrick@codethink.co.uk> | 2015-03-19 09:34:58 +0000 |
|---|---|---|
| committer | Morph (on behalf of Adam Coldrick) <adam.coldrick@codethink.co.uk> | 2015-03-19 09:34:58 +0000 |
| commit | 7db4ee53fb5398dd8f4ae8f56778735fe6531178 (patch) | |
| tree | 01513d77326acd03b2da356ec2cd7f4761901b6b | |
| parent | 211d6317d22bace089da58875d280ae5e54d5d54 (diff) | |
| download | morph-7db4ee53fb5398dd8f4ae8f56778735fe6531178.tar.gz | |
Morph build 2ee8190abe87461992f5b7ed85fe2ee9
System branch: master
263 files changed, 4167 insertions, 2405 deletions
@@ -273,7 +273,7 @@ Run the checks before submitting a patch, please. Legalese -------- -Copyright (C) 2011-2014 Codethink Limited +Copyright (C) 2011-2015 Codethink Limited This program is free software; you can redistribute it and/or modify it under the terms of the GNU General Public License as published by @@ -285,6 +285,4 @@ MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for more details. You should have received a copy of the GNU General Public License along -with this program; if not, write to the Free Software Foundation, Inc., -51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. - +with this program. If not, see <http://www.gnu.org/licenses/>. @@ -2,7 +2,7 @@ # # Run test suite for morph. # -# Copyright (C) 2011-2014 Codethink Limited +# Copyright (C) 2011-2015 Codethink Limited # # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by @@ -14,8 +14,7 @@ # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License along -# with this program; if not, write to the Free Software Foundation, Inc., -# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. +# with this program. If not, see <http://www.gnu.org/licenses/>. set -e @@ -94,11 +93,11 @@ export PYTHONPATH if "$run_style" && [ -d .git ]; then - echo "Checking copyright statements" - if ! (git ls-files --cached -z | - xargs -0r scripts/check-copyright-year); then - exit 1 - fi + #echo "Checking copyright statements" + #if ! (git ls-files --cached -z | + # xargs -0r scripts/check-copyright-year); then + # exit 1 + #fi echo 'Checking source code for silliness' if ! (git ls-files --cached | diff --git a/distbuild-helper b/distbuild-helper index cdc1873e..d77fcaea 100755 --- a/distbuild-helper +++ b/distbuild-helper @@ -2,7 +2,7 @@ # # distbuild-helper -- helper process for Morph distributed building # -# Copyright (C) 2014 Codethink Limited +# Copyright (C) 2014-2015 Codethink Limited # # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by @@ -14,8 +14,7 @@ # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License along -# with this program; if not, write to the Free Software Foundation, Inc., -# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.. +# with this program. If not, see <http://www.gnu.org/licenses/>. import cliapp import errno diff --git a/distbuild/__init__.py b/distbuild/__init__.py index 52ad2cc2..fc74d480 100644 --- a/distbuild/__init__.py +++ b/distbuild/__init__.py @@ -1,6 +1,6 @@ # distbuild/__init__.py -- library for Morph's distributed build plugin # -# Copyright (C) 2012, 2014 Codethink Limited +# Copyright (C) 2012, 2014-2015 Codethink Limited # # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by @@ -12,8 +12,7 @@ # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License along -# with this program; if not, write to the Free Software Foundation, Inc., -# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.. +# with this program. If not, see <http://www.gnu.org/licenses/>. from stringbuffer import StringBuffer @@ -51,9 +50,8 @@ from worker_build_scheduler import (WorkerBuildQueuer, WorkerBuildFailed, WorkerBuildStepStarted) from build_controller import (BuildController, BuildFailed, BuildProgress, - BuildSteps, BuildStepStarted, - BuildStepAlreadyStarted, BuildOutput, - BuildStepFinished, BuildStepFailed, + BuildStepStarted, BuildStepAlreadyStarted, + BuildOutput, BuildStepFinished, BuildStepFailed, BuildFinished, BuildCancel, build_step_name, map_build_graph) from initiator import Initiator diff --git a/distbuild/ansible/hosts b/distbuild/ansible/hosts new file mode 100644 index 00000000..2fbb50c4 --- /dev/null +++ b/distbuild/ansible/hosts @@ -0,0 +1 @@ +localhost diff --git a/distbuild/ansible/restart.yaml b/distbuild/ansible/restart.yaml new file mode 100644 index 00000000..a587a95e --- /dev/null +++ b/distbuild/ansible/restart.yaml @@ -0,0 +1,30 @@ +# Restart all Morph instances on a Baserock distbuild network. +# +# This will abort any running builds. +# +# This is an Ansible playbook. Run with: +# +# ansible-playbook -i distbuild-hosts restart.yaml +# +# You will need a file named 'distbuild-hosts' describing your distbuild +# network. +--- + +# The order in which the processes are restarted is important! (Sadly). +- hosts: controller + gather_facts: false + tasks: + - name: restart controller processes + service: name={{ item }} state=restarted + with_items: + - morph-controller.service + - morph-controller-helper.service + +- hosts: workers + gather_facts: false + tasks: + - name: restart worker processes + service: name={{ item }} state=restarted + with_items: + - morph-worker.service + - morph-worker-helper.service diff --git a/distbuild/ansible/run-from-morph-source-tree.sh.template b/distbuild/ansible/run-from-morph-source-tree.sh.template new file mode 100644 index 00000000..0fa7ebd0 --- /dev/null +++ b/distbuild/ansible/run-from-morph-source-tree.sh.template @@ -0,0 +1,6 @@ +#!/bin/sh + +# Script to use version of a program from a checkout of the morph.git source tree. + +source_path=/srv/morph +PYTHONPATH="$source_path" "$source_path/{{ item }}" "$@" diff --git a/distbuild/ansible/use-latest-morph.yaml b/distbuild/ansible/use-latest-morph.yaml new file mode 100644 index 00000000..9f2716d2 --- /dev/null +++ b/distbuild/ansible/use-latest-morph.yaml @@ -0,0 +1,28 @@ +# Upgrade the version of Morph used on a Baserock distbuild network. +# +# This allows quickly testing a new version of Morph without having to fully +# build and deploy a new Baserock 'build' system. You should run the +# 'restart.yaml' playbook after this one. +# +# This is an Ansible playbook. Run with: +# +# ansible-playbook -i distbuild-hosts use-latest-morph.yaml +# +# You will need a file named 'distbuild-hosts' describing your distbuild +# network. + +--- +- hosts: all + gather_facts: false + vars: + - TROVE_HOST: git.baserock.org + - MORPH_VERSION: sam/distbuild-build-logs + tasks: + - name: check out Morph from Git + git: dest=/srv/morph version={{ MORPH_VERSION }} repo=git://{{ TROVE_HOST }}/baserock/baserock/morph accept_hostkey=True + + - name: ensure latest Morph is used + template: backup=yes src=run-from-morph-source-tree.sh.template dest=/usr/bin/{{ item }} mode=0755 + with_items: + - morph + - distbuild-helper diff --git a/distbuild/build_controller.py b/distbuild/build_controller.py index 387b410f..3a0dd9b0 100644 --- a/distbuild/build_controller.py +++ b/distbuild/build_controller.py @@ -1,6 +1,6 @@ # distbuild/build_controller.py -- control the steps for one build # -# Copyright (C) 2012, 2014 Codethink Limited +# Copyright (C) 2012, 2014-2015 Codethink Limited # # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by @@ -12,8 +12,7 @@ # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License along -# with this program; if not, write to the Free Software Foundation, Inc., -# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.. +# with this program. If not, see <http://www.gnu.org/licenses/>. import logging @@ -37,11 +36,6 @@ class _Start(object): pass class _Annotated(object): pass class _Built(object): pass -class _AnnotationFailed(object): - - def __init__(self, http_status_code, error_msg): - self.http_status_code = http_status_code - self.error_msg = error_msg class _GotGraph(object): @@ -49,11 +43,6 @@ class _GotGraph(object): self.artifact = artifact -class _GraphFailed(object): - - pass - - class BuildCancel(object): def __init__(self, id): @@ -81,13 +70,6 @@ class BuildProgress(object): self.message_text = message_text -class BuildSteps(object): - - def __init__(self, request_id, artifact): - self.id = request_id - self.artifact = artifact - - class BuildStepStarted(object): def __init__(self, request_id, step_name, worker_name): @@ -131,7 +113,7 @@ class _Abort(object): def build_step_name(artifact): '''Return user-comprehensible name for a given artifact.''' - return artifact.name + return artifact.source_name def map_build_graph(artifact, callback): @@ -142,7 +124,7 @@ def map_build_graph(artifact, callback): a = queue.pop() if a not in done: result.append(callback(a)) - queue.extend(a.source.dependencies) + queue.extend(a.dependencies) done.add(a) return result @@ -192,14 +174,13 @@ class BuildController(distbuild.StateMachine): 'graphing', self._maybe_finish_graph), ('graphing', self, _GotGraph, 'annotating', self._start_annotating), - ('graphing', self, _GraphFailed, None, None), + ('graphing', self, BuildFailed, None, None), ('graphing', self._initiator_connection, distbuild.InitiatorDisconnect, None, None), ('annotating', distbuild.HelperRouter, distbuild.HelperResult, 'annotating', self._maybe_handle_cache_response), - ('annotating', self, _AnnotationFailed, None, - self._notify_annotation_failed), + ('annotating', self, BuildFailed, None, None), ('annotating', self, _Annotated, 'building', self._queue_worker_builds), ('annotating', self._initiator_connection, @@ -244,6 +225,29 @@ class BuildController(distbuild.StateMachine): self.mainloop.queue_event(self, _Start()) + def fail(self, reason): + logging.error(reason) + message = BuildFailed(self._request['id'], reason) + + # The message is sent twice so that it can be matched both by state + # transitions listening for this specific controller instance, and by + # state transitions listening for messages from the BuildController + # class that then filter the message based on the request ID field. + self.mainloop.queue_event(self, message) + self.mainloop.queue_event(BuildController, message) + + def _request_command_execution(self, argv, request_id): + '''Tell the controller's distbuild-helper to run a command.''' + if self.mainloop.n_state_machines_of_type(distbuild.HelperRouter) == 0: + self.fail('No distbuild-helper process running on controller!') + + msg = distbuild.message('exec-request', + id=request_id, + argv=argv, + stdin_contents='') + req = distbuild.HelperRequest(msg) + self.mainloop.queue_event(distbuild.HelperRouter, req) + def _start_graphing(self, event_source, event): distbuild.crash_point() @@ -260,14 +264,10 @@ class BuildController(distbuild.StateMachine): ] if 'original_ref' in self._request: argv.append(self._request['original_ref']) - msg = distbuild.message('exec-request', - id=self._idgen.next(), - argv=argv, - stdin_contents='') - self._helper_id = msg['id'] - req = distbuild.HelperRequest(msg) - self.mainloop.queue_event(distbuild.HelperRouter, req) - + + self._helper_id = self._idgen.next() + self._request_command_execution(argv, self._helper_id) + progress = BuildProgress(self._request['id'], 'Computing build graph') self.mainloop.queue_event(BuildController, progress) @@ -281,25 +281,12 @@ class BuildController(distbuild.StateMachine): def _maybe_finish_graph(self, event_source, event): distbuild.crash_point() - def notify_failure(msg_text): - logging.error('Graph creation failed: %s' % msg_text) - - failed = BuildFailed( - self._request['id'], - 'Failed to compute build graph: %s' % msg_text) - self.mainloop.queue_event(BuildController, failed) - - self.mainloop.queue_event(self, _GraphFailed()) - def notify_success(artifact): logging.debug('Graph is finished') progress = BuildProgress( self._request['id'], 'Finished computing build graph') self.mainloop.queue_event(BuildController, progress) - - build_steps = BuildSteps(self._request['id'], artifact) - self.mainloop.queue_event(BuildController, build_steps) self.mainloop.queue_event(self, _GotGraph(artifact)) @@ -308,8 +295,7 @@ class BuildController(distbuild.StateMachine): error_text = self._artifact_error.peek() if event.msg['exit'] != 0 or error_text: - notify_failure('Problem with serialise-artifact: %s' - % error_text) + self.fail(error_text) if event.msg['exit'] != 0: return @@ -317,9 +303,9 @@ class BuildController(distbuild.StateMachine): text = self._artifact_data.peek() try: artifact = distbuild.deserialise_artifact(text) - except ValueError, e: + except ValueError as e: logging.error(traceback.format_exc()) - notify_failure(str(e)) + self.fail('Failed to compute build graph: %s' % e) return notify_success(artifact) @@ -351,7 +337,6 @@ class BuildController(distbuild.StateMachine): '(helper id: %s)' % self._helper_id) def _maybe_handle_cache_response(self, event_source, event): - def set_status(artifact): is_in_cache = cache_state[artifact.basename()] artifact.state = BUILT if is_in_cache else UNBUILT @@ -362,28 +347,25 @@ class BuildController(distbuild.StateMachine): logging.debug('Got cache response: %s' % repr(event.msg)) http_status_code = event.msg['status'] - error_msg = event.msg['body'] if http_status_code != httplib.OK: - logging.debug('Cache request failed with status: %s' - % event.msg['status']) - self.mainloop.queue_event(self, - _AnnotationFailed(http_status_code, error_msg)) + self.fail('Failed to annotate build graph: HTTP request to %s got ' + '%d: %s' % (self._artifact_cache_server, + http_status_code, event.msg['body'])) return cache_state = json.loads(event.msg['body']) map_build_graph(self._artifact, set_status) self.mainloop.queue_event(self, _Annotated()) - count = sum(map_build_graph(self._artifact, - lambda a: 1 if a.state == UNBUILT else 0)) - + unbuilt = len([a for a in self._artifact.walk() if a.state == UNBUILT]) + total = len([a for _ in self._artifact.walk()]) progress = BuildProgress( self._request['id'], - 'Need to build %d artifacts' % count) + 'Need to build %d artifacts, of %d total' % (unbuilt, total)) self.mainloop.queue_event(BuildController, progress) - if count == 0: + if total == 0: logging.info('There seems to be nothing to build') self.mainloop.queue_event(self, _Built()) @@ -391,7 +373,7 @@ class BuildController(distbuild.StateMachine): def is_ready_to_build(artifact): return (artifact.state == UNBUILT and all(a.state == BUILT - for a in artifact.source.dependencies)) + for a in artifact.dependencies)) return [a for a in map_build_graph(self._artifact, lambda a: a) @@ -427,19 +409,19 @@ class BuildController(distbuild.StateMachine): logging.debug( 'Requesting worker-build of %s (%s)' % - (artifact.name, artifact.source.cache_key)) + (artifact.name, artifact.cache_key)) request = distbuild.WorkerBuildRequest(artifact, self._request['id']) self.mainloop.queue_event(distbuild.WorkerBuildQueuer, request) artifact.state = BUILDING - if artifact.source.morphology['kind'] == 'chunk': + if artifact.kind == 'chunk': # Chunk artifacts are not built independently # so when we're building any chunk artifact # we're also building all the chunk artifacts # in this source for a in ready: - if a.source == artifact.source: + if a.cache_key == artifact.cache_key: a.state = BUILDING @@ -543,7 +525,7 @@ class BuildController(distbuild.StateMachine): def _find_artifact(self, cache_key): artifacts = map_build_graph(self._artifact, lambda a: a) - wanted = [a for a in artifacts if a.source.cache_key == cache_key] + wanted = [a for a in artifacts if a.cache_key == cache_key] if wanted: return wanted[0] else: @@ -569,10 +551,10 @@ class BuildController(distbuild.StateMachine): artifact.state = BUILT def set_state(a): - if a.source == artifact.source: + if a.cache_key == artifact.cache_key: a.state = BUILT - if artifact.source.morphology['kind'] == 'chunk': + if artifact.kind == 'chunk': # Building a single chunk artifact # yields all chunk artifacts for the given source # so we set the state of this source's artifacts @@ -581,14 +563,6 @@ class BuildController(distbuild.StateMachine): self._queue_worker_builds(None, event) - def _notify_annotation_failed(self, event_source, event): - errmsg = ('Failed to annotate build graph: http request got %d: %s' - % (event.http_status_code, event.error_msg)) - - logging.error(errmsg) - failed = BuildFailed(self._request['id'], errmsg) - self.mainloop.queue_event(BuildController, failed) - def _maybe_notify_build_failed(self, event_source, event): distbuild.crash_point() @@ -613,10 +587,7 @@ class BuildController(distbuild.StateMachine): self._request['id'], build_step_name(artifact)) self.mainloop.queue_event(BuildController, step_failed) - build_failed = BuildFailed( - self._request['id'], - 'Building failed for %s' % artifact.name) - self.mainloop.queue_event(BuildController, build_failed) + self.fail('Building failed for %s' % artifact.name) # Cancel any jobs waiting to be executed, since there is no point # running them if this build has failed, it would just waste @@ -640,8 +611,8 @@ class BuildController(distbuild.StateMachine): baseurl = urlparse.urljoin( self._artifact_cache_server, '/1.0/artifacts') filename = ('%s.%s.%s' % - (self._artifact.source.cache_key, - self._artifact.source.morphology['kind'], + (self._artifact.cache_key, + self._artifact.kind, self._artifact.name)) url = '%s?filename=%s' % (baseurl, urllib.quote(filename)) finished = BuildFinished(self._request['id'], [url]) diff --git a/distbuild/connection_machine.py b/distbuild/connection_machine.py index e75ebe56..b8248931 100644 --- a/distbuild/connection_machine.py +++ b/distbuild/connection_machine.py @@ -1,6 +1,6 @@ # distbuild/connection_machine.py -- state machine for connecting to server # -# Copyright (C) 2012, 2014 Codethink Limited +# Copyright (C) 2012, 2014-2015 Codethink Limited # # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by @@ -12,8 +12,7 @@ # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License along -# with this program; if not, write to the Free Software Foundation, Inc., -# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.. +# with this program. If not, see <http://www.gnu.org/licenses/>. import errno @@ -106,7 +105,7 @@ class ConnectionMachine(distbuild.StateMachine): distbuild.set_nonblocking(self._socket) try: self._socket.connect((self._addr, self._port)) - except socket.error, e: + except socket.error as e: if e.errno != errno.EINPROGRESS: raise socket.error( "%s (attempting connection to distbuild controller " @@ -118,7 +117,7 @@ class ConnectionMachine(distbuild.StateMachine): def _connect(self, event_source, event): try: self._socket.connect((self._addr, self._port)) - except socket.error, e: + except socket.error as e: logging.error( 'Failed to connect to %s:%s: %s' % (self._addr, self._port, str(e))) diff --git a/distbuild/crashpoint.py b/distbuild/crashpoint.py index 6e3eb3ef..98e6906c 100644 --- a/distbuild/crashpoint.py +++ b/distbuild/crashpoint.py @@ -1,6 +1,6 @@ # distbuild/crashpoint.py -- user-controlled crashing # -# Copyright (C) 2012, 2014 Codethink Limited +# Copyright (C) 2012, 2014-2015 Codethink Limited # # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by @@ -12,8 +12,7 @@ # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License along -# with this program; if not, write to the Free Software Foundation, Inc., -# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.. +# with this program. If not, see <http://www.gnu.org/licenses/>. '''Crash the application. diff --git a/distbuild/crashpoint_tests.py b/distbuild/crashpoint_tests.py index eb64115e..be073a96 100644 --- a/distbuild/crashpoint_tests.py +++ b/distbuild/crashpoint_tests.py @@ -1,6 +1,6 @@ # distbuild/crashpoint_tests.py -- unit tests for crashpoint.py # -# Copyright (C) 2012, 2014 Codethink Limited +# Copyright (C) 2012, 2014-2015 Codethink Limited # # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by @@ -12,8 +12,7 @@ # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License along -# with this program; if not, write to the Free Software Foundation, Inc., -# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.. +# with this program. If not, see <http://www.gnu.org/licenses/>. import unittest diff --git a/distbuild/distbuild_socket.py b/distbuild/distbuild_socket.py index ce69f29e..2e219bb3 100644 --- a/distbuild/distbuild_socket.py +++ b/distbuild/distbuild_socket.py @@ -1,6 +1,6 @@ # distbuild/distbuild_socket.py -- wrapper around Python 'socket' module. # -# Copyright (C) 2012, 2014 Codethink Limited +# Copyright (C) 2012, 2014-2015 Codethink Limited # # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by @@ -12,8 +12,7 @@ # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License along -# with this program; if not, write to the Free Software Foundation, Inc., -# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.. +# with this program. If not, see <http://www.gnu.org/licenses/>. import socket diff --git a/distbuild/eventsrc.py b/distbuild/eventsrc.py index 560b9b7a..85e4f5f9 100644 --- a/distbuild/eventsrc.py +++ b/distbuild/eventsrc.py @@ -1,6 +1,6 @@ # mainloop/eventsrc.py -- interface for event sources # -# Copyright (C) 2012, 2014 Codethink Limited +# Copyright (C) 2012, 2014-2015 Codethink Limited # # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by @@ -12,8 +12,7 @@ # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License along -# with this program; if not, write to the Free Software Foundation, Inc., -# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.. +# with this program. If not, see <http://www.gnu.org/licenses/>. class EventSource(object): diff --git a/distbuild/helper_router.py b/distbuild/helper_router.py index f7126093..5578f750 100644 --- a/distbuild/helper_router.py +++ b/distbuild/helper_router.py @@ -1,6 +1,6 @@ # distbuild/helper_router.py -- state machine for controller's helper comms # -# Copyright (C) 2012, 2014 Codethink Limited +# Copyright (C) 2012, 2014-2015 Codethink Limited # # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by @@ -12,8 +12,7 @@ # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License along -# with this program; if not, write to the Free Software Foundation, Inc., -# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.. +# with this program. If not, see <http://www.gnu.org/licenses/>. import logging diff --git a/distbuild/idgen.py b/distbuild/idgen.py index 41f2ffcf..ef5684d7 100644 --- a/distbuild/idgen.py +++ b/distbuild/idgen.py @@ -1,6 +1,6 @@ # distbuild/idgen.py -- generate unique identifiers # -# Copyright (C) 2012, 2014 Codethink Limited +# Copyright (C) 2012, 2014-2015 Codethink Limited # # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by @@ -12,8 +12,7 @@ # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License along -# with this program; if not, write to the Free Software Foundation, Inc., -# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.. +# with this program. If not, see <http://www.gnu.org/licenses/>. import logging diff --git a/distbuild/initiator.py b/distbuild/initiator.py index aaae7d62..eef4c9ec 100644 --- a/distbuild/initiator.py +++ b/distbuild/initiator.py @@ -1,6 +1,6 @@ # distbuild/initiator.py -- state machine for the initiator # -# Copyright (C) 2012, 2014 Codethink Limited +# Copyright (C) 2012, 2014-2015 Codethink Limited # # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by @@ -12,15 +12,15 @@ # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License along -# with this program; if not, write to the Free Software Foundation, Inc., -# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.. +# with this program. If not, see <http://www.gnu.org/licenses/>. import cliapp +import itertools import logging import os import random -import sys +import time import distbuild @@ -37,6 +37,20 @@ class _Failed(object): self.msg = msg +def create_build_directory(prefix='build'): + '''Create a new directory to store build logs. + + The directory will be named build-0, unless that directory already exists, + in which case it will be named build-1, and so on. + + ''' + for i in itertools.count(): + path = '%s-%02i' % (prefix, i) + if not os.path.exists(path): + os.mkdir(path) + return path + + class Initiator(distbuild.StateMachine): def __init__(self, cm, conn, app, repo_name, ref, morphology, @@ -49,11 +63,14 @@ class Initiator(distbuild.StateMachine): self._ref = ref self._morphology = morphology self._original_ref = original_ref - self._steps = None self._step_outputs = {} - self._step_output_dir = app.settings['initiator-step-output-dir'] self.debug_transitions = False + if app.settings['initiator-step-output-dir'] == '': + self._step_output_dir = create_build_directory() + else: + self._step_output_dir = app.settings['initiator-step-output-dir'] + def setup(self): distbuild.crash_point() @@ -83,11 +100,12 @@ class Initiator(distbuild.StateMachine): repo=self._repo_name, ref=self._ref, morphology=self._morphology, - original_ref=self._original_ref + original_ref=self._original_ref, + protocol_version=distbuild.protocol.VERSION ) self._jm.send(msg) logging.debug('Initiator: sent to controller: %s', repr(msg)) - + def _handle_json_message(self, event_source, event): distbuild.crash_point() @@ -97,7 +115,6 @@ class Initiator(distbuild.StateMachine): 'build-finished': self._handle_build_finished_message, 'build-failed': self._handle_build_failed_message, 'build-progress': self._handle_build_progress_message, - 'build-steps': self._handle_build_steps_message, 'step-started': self._handle_step_started_message, 'step-already-started': self._handle_step_already_started_message, 'step-output': self._handle_step_output_message, @@ -117,12 +134,6 @@ class Initiator(distbuild.StateMachine): def _handle_build_progress_message(self, msg): self._app.status(msg='Progress: %(msgtext)s', msgtext=msg['message']) - def _handle_build_steps_message(self, msg): - self._steps = msg['steps'] - self._app.status( - msg='Build steps in total: %(steps)d', - steps=len(self._steps)) - def _open_output(self, msg): assert msg['step_name'] not in self._step_outputs if self._step_output_dir: @@ -137,23 +148,33 @@ class Initiator(distbuild.StateMachine): self._step_outputs[msg['step_name']].close() del self._step_outputs[msg['step_name']] + def _get_output(self, msg): + return self._step_outputs[msg['step_name']] + + def _write_status_to_build_log(self, f, status): + f.write(time.strftime('%Y-%m-%d %H:%M:%S ') + status + '\n') + f.flush() + def _handle_step_already_started_message(self, msg): - self._app.status( - msg='%s is already building on %s' % (msg['step_name'], - msg['worker_name'])) + status = '%s is already building on %s' % ( + msg['step_name'], msg['worker_name']) + self._app.status(msg=status) + self._open_output(msg) + self._write_status_to_build_log(self._get_output(msg), status) def _handle_step_started_message(self, msg): - self._app.status( - msg='Started building %(step_name)s on %(worker_name)s', - step_name=msg['step_name'], - worker_name=msg['worker_name']) + status = 'Started building %s on %s' % ( + msg['step_name'], msg['worker_name']) + self._app.status(msg=status) + self._open_output(msg) + self._write_status_to_build_log(self._get_output(msg), status) def _handle_step_output_message(self, msg): step_name = msg['step_name'] if step_name in self._step_outputs: - f = self._step_outputs[step_name] + f = self._get_output(msg) f.write(msg['stdout']) f.write(msg['stderr']) f.flush() @@ -164,9 +185,10 @@ class Initiator(distbuild.StateMachine): def _handle_step_finished_message(self, msg): step_name = msg['step_name'] if step_name in self._step_outputs: - self._app.status( - msg='Finished building %(step_name)s', - step_name=step_name) + status = 'Finished building %s' % step_name + self._app.status(msg=status) + + self._write_status_to_build_log(self._get_output(msg), status) self._close_output(msg) else: logging.warning( @@ -175,9 +197,10 @@ class Initiator(distbuild.StateMachine): def _handle_step_failed_message(self, msg): step_name = msg['step_name'] if step_name in self._step_outputs: - self._app.status( - msg='Build failed: %(step_name)s', - step_name=step_name) + status = 'Build of %s failed.' % step_name + self._app.status(msg=status) + + self._write_status_to_build_log(self._get_output(msg), status) self._close_output(msg) else: logging.warning( @@ -208,3 +231,13 @@ class Initiator(distbuild.StateMachine): self.mainloop.queue_event(self._cm, distbuild.StopConnecting()) self._jm.close() + def handle_cancel(self): + # Note in each build-step.log file that the initiator cancelled: this + # makes it easier to tell whether a build was aborted due to a bug or + # dropped connection, or if the user cancelled with CTRL+C / SIGINT. + + for f in self._step_outputs.itervalues(): + self._write_status_to_build_log(f, 'Initiator cancelled') + f.close() + + self._step_outputs = {} diff --git a/distbuild/initiator_connection.py b/distbuild/initiator_connection.py index db982230..8d6c1f4c 100644 --- a/distbuild/initiator_connection.py +++ b/distbuild/initiator_connection.py @@ -1,6 +1,6 @@ # distbuild/initiator_connection.py -- communicate with initiator # -# Copyright (C) 2012, 2014 Codethink Limited +# Copyright (C) 2012, 2014-2015 Codethink Limited # # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by @@ -12,8 +12,7 @@ # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License along -# with this program; if not, write to the Free Software Foundation, Inc., -# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.. +# with this program. If not, see <http://www.gnu.org/licenses/>. import logging @@ -77,8 +76,6 @@ class InitiatorConnection(distbuild.StateMachine): 'idle', self._send_build_failed_message), ('idle', distbuild.BuildController, distbuild.BuildProgress, 'idle', self._send_build_progress_message), - ('idle', distbuild.BuildController, distbuild.BuildSteps, - 'idle', self._send_build_steps_message), ('idle', distbuild.BuildController, distbuild.BuildStepStarted, 'idle', self._send_build_step_started_message), ('idle', distbuild.BuildController, @@ -100,15 +97,31 @@ class InitiatorConnection(distbuild.StateMachine): logging.debug('InitiatorConnection: from %s: %r', self.initiator_name, event.msg) - if event.msg['type'] == 'build-request': - new_id = self._idgen.next() - self.our_ids.add(new_id) - self._route_map.add(event.msg['id'], new_id) - event.msg['id'] = new_id - build_controller = distbuild.BuildController( - self, event.msg, self.artifact_cache_server, - self.morph_instance) - self.mainloop.add_state_machine(build_controller) + try: + if event.msg['type'] == 'build-request': + if (event.msg.get('protocol_version') != + distbuild.protocol.VERSION): + msg = distbuild.message('build-failed', + id=event.msg['id'], + reason=('Protocol version mismatch between server & ' + 'initiator: distbuild network uses distbuild ' + 'protocol version %i, but client uses version' + ' %i.' % (distbuild.protocol.VERSION, + event.msg.get('protocol_version')))) + self.jm.send(msg) + self._log_send(msg) + return + new_id = self._idgen.next() + self.our_ids.add(new_id) + self._route_map.add(event.msg['id'], new_id) + event.msg['id'] = new_id + build_controller = distbuild.BuildController( + self, event.msg, self.artifact_cache_server, + self.morph_instance) + self.mainloop.add_state_machine(build_controller) + except (KeyError, ValueError) as ex: + logging.error('Invalid message from initiator: %s: exception %s', + event.msg, ex) def _disconnect(self, event_source, event): for id in self.our_ids: @@ -164,26 +177,6 @@ class InitiatorConnection(distbuild.StateMachine): self.jm.send(msg) self._log_send(msg) - def _send_build_steps_message(self, event_source, event): - - def make_step_dict(artifact): - return { - 'name': distbuild.build_step_name(artifact), - 'build-depends': [ - distbuild.build_step_name(x) - for x in artifact.source.dependencies - ] - } - - if event.id in self.our_ids: - step_names = distbuild.map_build_graph( - event.artifact, make_step_dict) - msg = distbuild.message('build-steps', - id=self._route_map.get_incoming_id(event.id), - steps=step_names) - self.jm.send(msg) - self._log_send(msg) - def _send_build_step_started_message(self, event_source, event): logging.debug('InitiatorConnection: build_step_started: ' 'id=%s step_name=%s worker_name=%s' % diff --git a/distbuild/jm.py b/distbuild/jm.py index 615100e4..c08a1958 100644 --- a/distbuild/jm.py +++ b/distbuild/jm.py @@ -1,6 +1,6 @@ # mainloop/jm.py -- state machine for JSON communication between nodes # -# Copyright (C) 2012, 2014 Codethink Limited +# Copyright (C) 2012, 2014-2015 Codethink Limited # # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by @@ -12,8 +12,7 @@ # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License along -# with this program; if not, write to the Free Software Foundation, Inc., -# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.. +# with this program. If not, see <http://www.gnu.org/licenses/>. import fcntl @@ -109,8 +108,13 @@ class JsonMachine(StateMachine): line = line.rstrip() if self.debug_json: logging.debug('JsonMachine: line: %s' % repr(line)) - msg = yaml.load(json.loads(line)) - self.mainloop.queue_event(self, JsonNewMessage(msg)) + msg = None + try: + msg = yaml.safe_load(json.loads(line)) + except Exception: + logging.error('Invalid input: %s' % line) + if msg: + self.mainloop.queue_event(self, JsonNewMessage(msg)) def _send_eof(self, event_source, event): self.mainloop.queue_event(self, JsonEof()) diff --git a/distbuild/json_router.py b/distbuild/json_router.py index 8b7b6457..b8d0ca55 100644 --- a/distbuild/json_router.py +++ b/distbuild/json_router.py @@ -1,6 +1,6 @@ # distbuild/json_router.py -- state machine to route JSON messages # -# Copyright (C) 2012, 2014 Codethink Limited +# Copyright (C) 2012, 2014-2015 Codethink Limited # # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by @@ -12,8 +12,7 @@ # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License along -# with this program; if not, write to the Free Software Foundation, Inc., -# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.. +# with this program. If not, see <http://www.gnu.org/licenses/>. import logging diff --git a/distbuild/mainloop.py b/distbuild/mainloop.py index f0e5eebc..e7c0cc3b 100644 --- a/distbuild/mainloop.py +++ b/distbuild/mainloop.py @@ -1,6 +1,6 @@ # mainloop/mainloop.py -- select-based main loop # -# Copyright (C) 2012, 2014 Codethink Limited +# Copyright (C) 2012, 2014-2015 Codethink Limited # # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by @@ -12,8 +12,7 @@ # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License along -# with this program; if not, write to the Free Software Foundation, Inc., -# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.. +# with this program. If not, see <http://www.gnu.org/licenses/>. import fcntl @@ -56,7 +55,13 @@ class MainLoop(object): def remove_state_machine(self, machine): logging.debug('MainLoop.remove_state_machine: %s' % machine) self._machines.remove(machine) - + + def state_machines_of_type(self, machine_type): + return [m for m in self._machines if isinstance(m, machine_type)] + + def n_state_machines_of_type(self, machine_type): + return len(self.state_machines_of_type(machine_type)) + def add_event_source(self, event_source): logging.debug('MainLoop.add_event_source: %s' % event_source) self._sources.append(event_source) diff --git a/distbuild/protocol.py b/distbuild/protocol.py index ffce1fe7..73d72d1d 100644 --- a/distbuild/protocol.py +++ b/distbuild/protocol.py @@ -1,6 +1,6 @@ # distbuild/protocol.py -- abstractions for the JSON messages # -# Copyright (C) 2012, 2014 Codethink Limited +# Copyright (C) 2012, 2014-2015 Codethink Limited # # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by @@ -12,28 +12,31 @@ # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License along -# with this program; if not, write to the Free Software Foundation, Inc., -# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.. +# with this program. If not, see <http://www.gnu.org/licenses/>. '''Construct protocol message objects (dicts).''' +# Version refers to an integer that should be incremented by one each time a +# time a change is introduced that would break server/initiator compatibility + + +VERSION = 1 + + _required_fields = { 'build-request': [ 'id', 'repo', 'ref', 'morphology', + 'protocol_version', ], 'build-progress': [ 'id', 'message', ], - 'build-steps': [ - 'id', - 'steps', - ], 'step-started': [ 'id', 'step_name', diff --git a/distbuild/proxy_event_source.py b/distbuild/proxy_event_source.py index 20080800..4f75819a 100644 --- a/distbuild/proxy_event_source.py +++ b/distbuild/proxy_event_source.py @@ -1,6 +1,6 @@ # distbuild/proxy_event_source.py -- proxy for temporary event sources # -# Copyright (C) 2012, 2014 Codethink Limited +# Copyright (C) 2012, 2014-2015 Codethink Limited # # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by @@ -12,8 +12,7 @@ # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License along -# with this program; if not, write to the Free Software Foundation, Inc., -# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.. +# with this program. If not, see <http://www.gnu.org/licenses/>. import errno diff --git a/distbuild/route_map.py b/distbuild/route_map.py index 6dd90d78..0d482e24 100644 --- a/distbuild/route_map.py +++ b/distbuild/route_map.py @@ -1,6 +1,6 @@ # distbuild/route_map.py -- map message ids for routing purposes # -# Copyright (C) 2012, 2014 Codethink Limited +# Copyright (C) 2012, 2014-2015 Codethink Limited # # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by @@ -12,8 +12,7 @@ # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License along -# with this program; if not, write to the Free Software Foundation, Inc., -# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.. +# with this program. If not, see <http://www.gnu.org/licenses/>. class RouteMap(object): diff --git a/distbuild/route_map_tests.py b/distbuild/route_map_tests.py index b5ceca70..1f6cee5e 100644 --- a/distbuild/route_map_tests.py +++ b/distbuild/route_map_tests.py @@ -1,6 +1,6 @@ # distbuild/route_map_tests.py -- unit tests for message routing # -# Copyright (C) 2012, 2014 Codethink Limited +# Copyright (C) 2012, 2014-2015 Codethink Limited # # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by @@ -12,8 +12,7 @@ # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License along -# with this program; if not, write to the Free Software Foundation, Inc., -# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.. +# with this program. If not, see <http://www.gnu.org/licenses/>. import unittest diff --git a/distbuild/serialise.py b/distbuild/serialise.py index a7c6c4b9..5f8872a6 100644 --- a/distbuild/serialise.py +++ b/distbuild/serialise.py @@ -1,6 +1,6 @@ # distbuild/serialise.py -- (de)serialise Artifact object graphs # -# Copyright (C) 2012, 2014 Codethink Limited +# Copyright (C) 2012, 2014-2015 Codethink Limited # # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by @@ -12,51 +12,72 @@ # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License along -# with this program; if not, write to the Free Software Foundation, Inc., -# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.. +# with this program. If not, see <http://www.gnu.org/licenses/>. import json +import logging import yaml import morphlib -import logging -def serialise_artifact(artifact): +class ArtifactReference(object): # pragma: no cover + + '''Container for some basic information about an artifact.''' + + def __init__(self, basename, encoded): + self._basename = basename + self._dict = encoded + + def __getattr__(self, name): + if not name.startswith('_'): + return self._dict.get(name) + else: + super(ArtifactReference, self).__getattr(name) + + def __setattr__(self, name, val): + if not name.startswith('_'): + self._dict[name] = val + else: + super(ArtifactReference, self).__setattr__(name, val) + + def basename(self): + return self._basename + + def walk(self): + done = set() + + def depth_first(a): + if a not in done: + done.add(a) + for dep in a.dependencies: + for ret in depth_first(dep): + yield ret + yield a + + return list(depth_first(self)) + + +def serialise_artifact(artifact, repo, ref): '''Serialise an Artifact object and its dependencies into string form.''' - def encode_morphology(morphology): - result = {} - for key in morphology.keys(): - result[key] = morphology[key] - return result - - def encode_source(source, prune_leaf=False): - source_dic = { - 'name': source.name, - 'repo': None, - 'repo_name': source.repo_name, - 'original_ref': source.original_ref, - 'sha1': source.sha1, - 'tree': source.tree, - 'morphology': id(source.morphology), + def encode_source(source): + s_dict = { 'filename': source.filename, - 'artifact_ids': [], - 'cache_id': source.cache_id, - 'cache_key': source.cache_key, - 'dependencies': [], + 'kind': source.morphology['kind'], + 'source_name': source.name, + 'source_repo': source.repo_name, + 'source_ref': source.original_ref, + 'source_sha1': source.sha1, + 'source_artifacts': [], + 'dependencies': [] } - if not prune_leaf: - source_dic['artifact_ids'].extend(id(artifact) for (_, artifact) - in source.artifacts.iteritems()) - source_dic['dependencies'].extend(id(d) - for d in source.dependencies) - - if source.morphology['kind'] == 'chunk': - source_dic['build_mode'] = source.build_mode - source_dic['prefix'] = source.prefix - return source_dic + for dep in source.dependencies: + s_dict['dependencies'].append(dep.basename()) + for sa in source.artifacts: + s_dict['source_artifacts'].append(sa) + return s_dict def encode_artifact(a): if artifact.source.morphology['kind'] == 'system': # pragma: no cover @@ -64,53 +85,61 @@ def serialise_artifact(artifact): else: arch = artifact.arch - return { - 'source_id': id(a.source), - 'name': a.name, + a_dict = { 'arch': arch, - 'dependents': [id(d) - for d in a.dependents], + 'cache_key': a.source.cache_key, + 'name': a.name, + 'repo': repo, + 'ref': ref, + } + return a_dict + + def encode_artifact_reference(a): # pragma: no cover + a_dict = { + 'arch': a.arch, + 'cache_key': a.cache_key, + 'name': a.name, + 'repo': a.repo, + 'ref': a.ref + } + s_dict = { + 'filename': a.filename, + 'kind': a.kind, + 'source_name': a.source_name, + 'source_repo': a.source_repo, + 'source_ref': a.source_ref, + 'source_sha1': a.source_sha1, + 'source_artifacts': [], + 'dependencies': [] } + for dep in a.dependencies: + s_dict['dependencies'].append(dep.basename()) + for sa in a.source_artifacts: + s_dict['source_artifacts'].append(sa) + return a_dict, s_dict encoded_artifacts = {} encoded_sources = {} - encoded_morphologies = {} - visited_artifacts = {} - - for a in artifact.walk(): - if id(a.source) not in encoded_sources: - for sa in a.source.artifacts.itervalues(): - if id(sa) not in encoded_artifacts: - visited_artifacts[id(sa)] = sa - encoded_artifacts[id(sa)] = encode_artifact(sa) - encoded_morphologies[id(a.source.morphology)] = \ - encode_morphology(a.source.morphology) - encoded_sources[id(a.source)] = encode_source(a.source) - - if id(a) not in encoded_artifacts: # pragma: no cover - visited_artifacts[id(a)] = a - encoded_artifacts[id(a)] = encode_artifact(a) - - # Include one level of dependents above encoded artifacts, as we need - # them to be able to tell whether two sources are in the same stratum. - for a in visited_artifacts.itervalues(): - for source in a.dependents: # pragma: no cover - if id(source) not in encoded_sources: - encoded_morphologies[id(source.morphology)] = \ - encode_morphology(source.morphology) - encoded_sources[id(source)] = \ - encode_source(source, prune_leaf=True) + + if isinstance(artifact, ArtifactReference): # pragma: no cover + root_filename = artifact.root_filename + a_dict, s_dict = encode_artifact_reference(artifact) + encoded_artifacts[artifact.basename()] = a_dict + encoded_sources[artifact.cache_key] = s_dict + else: + root_filename = artifact.source.filename + for a in artifact.walk(): + if a.basename() not in encoded_artifacts: # pragma: no cover + encoded_artifacts[a.basename()] = encode_artifact(a) + encoded_sources[a.source.cache_key] = encode_source(a.source) content = { - 'sources': encoded_sources, + 'root-artifact': artifact.basename(), + 'root-filename': root_filename, 'artifacts': encoded_artifacts, - 'morphologies': encoded_morphologies, - 'root_artifact': id(artifact), - 'default_split_rules': { - 'chunk': morphlib.artifactsplitrule.DEFAULT_CHUNK_RULES, - 'stratum': morphlib.artifactsplitrule.DEFAULT_STRATUM_RULES, - }, + 'sources': encoded_sources } + return json.dumps(yaml.dump(content)) @@ -123,95 +152,24 @@ def deserialise_artifact(encoded): purposes, by Morph. ''' - - def decode_morphology(le_dict): - '''Convert a dict into something that kinda acts like a Morphology. - - As it happens, we don't need the full Morphology so we cheat. - Cheating is good. - - ''' - - return morphlib.morphology.Morphology(le_dict) - - def decode_source(le_dict, morphology, split_rules): - '''Convert a dict into a Source object.''' - - source = morphlib.source.Source(le_dict['name'], - le_dict['repo_name'], - le_dict['original_ref'], - le_dict['sha1'], - le_dict['tree'], - morphology, - le_dict['filename'], - split_rules) - - if morphology['kind'] == 'chunk': - source.build_mode = le_dict['build_mode'] - source.prefix = le_dict['prefix'] - source.cache_id = le_dict['cache_id'] - source.cache_key = le_dict['cache_key'] - return source - - def decode_artifact(artifact_dict, source): - '''Convert dict into an Artifact object. - - Do not set dependencies, that will be dealt with later. - - ''' - - artifact = morphlib.artifact.Artifact(source, artifact_dict['name']) - artifact.arch = artifact_dict['arch'] - artifact.source = source - - return artifact - - le_dicts = yaml.load(json.loads(encoded)) - artifacts_dict = le_dicts['artifacts'] - sources_dict = le_dicts['sources'] - morphologies_dict = le_dicts['morphologies'] - root_artifact = le_dicts['root_artifact'] - assert root_artifact in artifacts_dict + content = yaml.load(json.loads(encoded)) + root = content['root-artifact'] + encoded_artifacts = content['artifacts'] + encoded_sources = content['sources'] artifacts = {} - sources = {} - morphologies = {id: decode_morphology(d) - for (id, d) in morphologies_dict.iteritems()} - - # Decode sources - for source_id, source_dict in sources_dict.iteritems(): - morphology = morphologies[source_dict['morphology']] - kind = morphology['kind'] - ruler = getattr(morphlib.artifactsplitrule, 'unify_%s_matches' % kind) - if kind in ('chunk', 'stratum'): - rules = ruler(morphology, le_dicts['default_split_rules'][kind]) - else: # pragma: no cover - rules = ruler(morphology) - sources[source_id] = decode_source(source_dict, morphology, rules) # decode artifacts - for artifact_id, artifact_dict in artifacts_dict.iteritems(): - source_id = artifact_dict['source_id'] - source = sources[source_id] - artifact = decode_artifact(artifact_dict, source) - artifacts[artifact_id] = artifact - - # add source artifacts reference - for source_id, source in sources.iteritems(): - source_dict = sources_dict[source_id] - source.artifacts = {artifacts[a].name: artifacts[a] - for a in source_dict['artifact_ids']} - - # add source dependencies - for source_id, source_dict in sources_dict.iteritems(): - source = sources[source_id] - source.dependencies = [artifacts[aid] - for aid in source_dict['dependencies']] - - # add artifact dependents - for artifact_id, artifact in artifacts.iteritems(): - artifact_dict = artifacts_dict[artifact_id] - artifact.dependents = [sources[sid] - for sid in artifact_dict['dependents']] - - return artifacts[root_artifact] + for basename, artifact_dict in encoded_artifacts.iteritems(): + artifact_dict.update(encoded_sources[artifact_dict['cache_key']]) + artifact = ArtifactReference(basename, artifact_dict) + artifact.root_filename = content['root-filename'] + artifacts[basename] = artifact + + # add dependencies + for basename, a_dict in encoded_artifacts.iteritems(): + artifact = artifacts[basename] + artifact.dependencies = [artifacts.get(dep) + for dep in artifact.dependencies] + + return artifacts[root] diff --git a/distbuild/serialise_tests.py b/distbuild/serialise_tests.py index d80c3dd7..2de3ab85 100644 --- a/distbuild/serialise_tests.py +++ b/distbuild/serialise_tests.py @@ -1,6 +1,6 @@ # distbuild/serialise_tests.py -- unit tests for Artifact serialisation # -# Copyright (C) 2012, 2014 Codethink Limited +# Copyright (C) 2012, 2014-2015 Codethink Limited # # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by @@ -12,8 +12,7 @@ # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License along -# with this program; if not, write to the Free Software Foundation, Inc., -# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.. +# with this program. If not, see <http://www.gnu.org/licenses/>. import unittest @@ -21,32 +20,6 @@ import unittest import distbuild -class MockMorphology(object): - - def __init__(self, name, kind): - self.dict = { - 'name': '%s.morphology.name' % name, - 'kind': kind, - 'chunks': [], - 'products': [ - { - 'artifact': name, - 'include': [r'.*'], - }, - ], - } - - @property - def needs_artifact_metadata_cached(self): - return self.dict['kind'] == 'stratum' - - def keys(self): - return self.dict.keys() - - def __getitem__(self, key): - return self.dict[key] - - class MockSource(object): build_mode = 'staging' @@ -58,7 +31,7 @@ class MockSource(object): self.original_ref = '%s.source.original_ref' % name self.sha1 = '%s.source.sha1' % name self.tree = '%s.source.tree' % name - self.morphology = MockMorphology(name, kind) + self.morphology = {'kind': kind} self.filename = '%s.source.filename' % name self.dependencies = [] self.cache_id = { @@ -79,6 +52,11 @@ class MockArtifact(object): self.name = name self.dependents = [] + def basename(self): + return '%s.%s.%s' % (self.source.cache_key, + self.source.morphology['kind'], + self.name) + def walk(self): # pragma: no cover done = set() @@ -101,53 +79,28 @@ class SerialisationTests(unittest.TestCase): self.art3 = MockArtifact('name3', 'chunk') self.art4 = MockArtifact('name4', 'chunk') - def assertEqualMorphologies(self, a, b): - self.assertEqual(sorted(a.keys()), sorted(b.keys())) - keys = sorted(a.keys()) - a_values = [a[k] for k in keys] - b_values = [b[k] for k in keys] - self.assertEqual(a_values, b_values) - self.assertEqual(a.needs_artifact_metadata_cached, - b.needs_artifact_metadata_cached) - - def assertEqualSources(self, a, b): - self.assertEqual(a.repo, b.repo) - self.assertEqual(a.repo_name, b.repo_name) - self.assertEqual(a.original_ref, b.original_ref) - self.assertEqual(a.sha1, b.sha1) - self.assertEqual(a.tree, b.tree) - self.assertEqualMorphologies(a.morphology, b.morphology) - self.assertEqual(a.filename, b.filename) - - def assertEqualArtifacts(self, a, b): - self.assertEqualSources(a.source, b.source) - self.assertEqual(a.name, b.name) - self.assertEqual(a.source.cache_id, b.source.cache_id) - self.assertEqual(a.source.cache_key, b.source.cache_key) - self.assertEqual(len(a.source.dependencies), - len(b.source.dependencies)) - for i in range(len(a.source.dependencies)): - self.assertEqualArtifacts(a.source.dependencies[i], - b.source.dependencies[i]) - def verify_round_trip(self, artifact): - encoded = distbuild.serialise_artifact(artifact) + encoded = distbuild.serialise_artifact(artifact, + artifact.source.repo_name, + artifact.source.sha1) decoded = distbuild.deserialise_artifact(encoded) - self.assertEqualArtifacts(artifact, decoded) + self.assertEqual(artifact.basename(), decoded.basename()) objs = {} queue = [decoded] while queue: obj = queue.pop() - k = obj.source.cache_key + k = obj.cache_key if k in objs: self.assertTrue(obj is objs[k]) else: objs[k] = obj - queue.extend(obj.source.dependencies) + queue.extend(obj.dependencies) def test_returns_string(self): - encoded = distbuild.serialise_artifact(self.art1) + encoded = distbuild.serialise_artifact(self.art1, + self.art1.source.repo_name, + self.art1.source.sha1) self.assertEqual(type(encoded), str) def test_works_without_dependencies(self): @@ -171,4 +124,3 @@ class SerialisationTests(unittest.TestCase): self.art3.source.dependencies = [self.art4] self.art1.source.dependencies = [self.art2, self.art3] self.verify_round_trip(self.art1) - diff --git a/distbuild/sm.py b/distbuild/sm.py index e773962b..fd1d04ac 100644 --- a/distbuild/sm.py +++ b/distbuild/sm.py @@ -1,6 +1,6 @@ # mainloop/sm.py -- state machine abstraction # -# Copyright (C) 2012, 2014 Codethink Limited +# Copyright (C) 2012, 2014-2015 Codethink Limited # # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by @@ -12,8 +12,7 @@ # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License along -# with this program; if not, write to the Free Software Foundation, Inc., -# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.. +# with this program. If not, see <http://www.gnu.org/licenses/>. import logging diff --git a/distbuild/sm_tests.py b/distbuild/sm_tests.py index 59b9c023..e1f3be9f 100644 --- a/distbuild/sm_tests.py +++ b/distbuild/sm_tests.py @@ -1,6 +1,6 @@ # distbuild/sm_tests.py -- unit tests for state machine abstraction # -# Copyright (C) 2012, 2014 Codethink Limited +# Copyright (C) 2012, 2014-2015 Codethink Limited # # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by @@ -12,8 +12,7 @@ # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License along -# with this program; if not, write to the Free Software Foundation, Inc., -# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.. +# with this program. If not, see <http://www.gnu.org/licenses/>. import unittest diff --git a/distbuild/sockbuf.py b/distbuild/sockbuf.py index fc0315b0..6feb8669 100644 --- a/distbuild/sockbuf.py +++ b/distbuild/sockbuf.py @@ -1,6 +1,6 @@ # mainloop/sockbuf.py -- a buffering, non-blocking socket I/O state machine # -# Copyright (C) 2012, 2014 Codethink Limited +# Copyright (C) 2012, 2014-2015 Codethink Limited # # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by @@ -12,8 +12,7 @@ # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License along -# with this program; if not, write to the Free Software Foundation, Inc., -# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.. +# with this program. If not, see <http://www.gnu.org/licenses/>. import logging @@ -144,7 +143,7 @@ class SocketBuffer(StateMachine): def _fill(self, event_source, event): try: data = event.sock.read(self._max_buffer) - except (IOError, OSError), e: + except (IOError, OSError) as e: logging.debug( '%s: _fill(): Exception %s from sock.read()', self, e) return [SocketError(event.sock, e)] @@ -164,7 +163,7 @@ class SocketBuffer(StateMachine): data = self._wbuf.read(max_write) try: n = event.sock.write(data) - except (IOError, OSError), e: + except (IOError, OSError) as e: logging.debug( '%s: _flush(): Exception %s from sock.write()', self, e) return [SocketError(event.sock, e)] diff --git a/distbuild/socketsrc.py b/distbuild/socketsrc.py index 15283140..daba1610 100644 --- a/distbuild/socketsrc.py +++ b/distbuild/socketsrc.py @@ -1,6 +1,6 @@ # mainloop/socketsrc.py -- events and event sources for sockets # -# Copyright (C) 2012, 2014 Codethink Limited +# Copyright (C) 2012, 2014-2015 Codethink Limited # # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by @@ -12,8 +12,7 @@ # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License along -# with this program; if not, write to the Free Software Foundation, Inc., -# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.. +# with this program. If not, see <http://www.gnu.org/licenses/>. import fcntl @@ -78,7 +77,7 @@ class ListeningSocketEventSource(EventSource): if self._accepting and self.sock.fileno() in r: try: conn, addr = self.sock.accept() - except socket.error, e: + except socket.error as e: return [SocketError(self.sock, e)] else: logging.info( diff --git a/distbuild/sockserv.py b/distbuild/sockserv.py index 156394e2..c9979328 100644 --- a/distbuild/sockserv.py +++ b/distbuild/sockserv.py @@ -1,6 +1,6 @@ # mainloop/sockserv.py -- socket server state machines # -# Copyright (C) 2012, 2014 Codethink Limited +# Copyright (C) 2012, 2014-2015 Codethink Limited # # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by @@ -12,8 +12,7 @@ # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License along -# with this program; if not, write to the Free Software Foundation, Inc., -# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.. +# with this program. If not, see <http://www.gnu.org/licenses/>. import logging diff --git a/distbuild/stringbuffer.py b/distbuild/stringbuffer.py index 2b94dd19..6371c841 100644 --- a/distbuild/stringbuffer.py +++ b/distbuild/stringbuffer.py @@ -1,6 +1,6 @@ # mainloop/stringbuffer.py -- efficient buffering of strings as a queue # -# Copyright (C) 2012, 2014 Codethink Limited +# Copyright (C) 2012, 2014-2015 Codethink Limited # # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by @@ -12,8 +12,7 @@ # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License along -# with this program; if not, write to the Free Software Foundation, Inc., -# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.. +# with this program. If not, see <http://www.gnu.org/licenses/>. class StringBuffer(object): diff --git a/distbuild/stringbuffer_tests.py b/distbuild/stringbuffer_tests.py index da324f20..5df8951a 100644 --- a/distbuild/stringbuffer_tests.py +++ b/distbuild/stringbuffer_tests.py @@ -1,6 +1,6 @@ # distbuild/stringbuffer_tests.py -- unit tests # -# Copyright (C) 2012, 2014 Codethink Limited +# Copyright (C) 2012, 2014-2015 Codethink Limited # # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by @@ -12,8 +12,7 @@ # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License along -# with this program; if not, write to the Free Software Foundation, Inc., -# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.. +# with this program. If not, see <http://www.gnu.org/licenses/>. import unittest diff --git a/distbuild/timer_event_source.py b/distbuild/timer_event_source.py index 4a2e81b7..5f5eeb1c 100644 --- a/distbuild/timer_event_source.py +++ b/distbuild/timer_event_source.py @@ -1,6 +1,6 @@ # distbuild/timer_event_source.py -- event source for timer events # -# Copyright (C) 2012, 2014 Codethink Limited +# Copyright (C) 2012, 2014-2015 Codethink Limited # # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by @@ -12,8 +12,7 @@ # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License along -# with this program; if not, write to the Free Software Foundation, Inc., -# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.. +# with this program. If not, see <http://www.gnu.org/licenses/>. import time diff --git a/distbuild/worker_build_scheduler.py b/distbuild/worker_build_scheduler.py index be732153..d00b0290 100644 --- a/distbuild/worker_build_scheduler.py +++ b/distbuild/worker_build_scheduler.py @@ -1,6 +1,6 @@ # distbuild/worker_build_scheduler.py -- schedule worker-builds on workers # -# Copyright (C) 2012, 2014 Codethink Limited +# Copyright (C) 2012, 2014-2015 Codethink Limited # # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by @@ -12,8 +12,7 @@ # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License along -# with this program; if not, write to the Free Software Foundation, Inc., -# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.. +# with this program. If not, see <http://www.gnu.org/licenses/>. import collections @@ -93,6 +92,12 @@ class _HaveAJob(object): def __init__(self, job): self.job = job +class _Disconnected(object): + + def __init__(self, who): + self.who = who + + class Job(object): def __init__(self, job_id, artifact, initiator_id): @@ -220,7 +225,10 @@ class WorkerBuildQueuer(distbuild.StateMachine): ('idle', WorkerConnection, _JobFinished, 'idle', self._set_job_finished), ('idle', WorkerConnection, _JobFailed, 'idle', - self._set_job_failed) + self._set_job_failed), + + ('idle', WorkerConnection, _Disconnected, 'idle', + self._handle_worker_disconnected), ] self.add_transitions(spec) @@ -262,13 +270,13 @@ class WorkerBuildQueuer(distbuild.StateMachine): logging.debug('Worker build step already started: %s' % event.artifact.basename()) progress = WorkerBuildStepAlreadyStarted(event.initiator_id, - event.artifact.source.cache_key, job.who.name()) + event.artifact.cache_key, job.who.name()) else: logging.debug('Job created but not building yet ' '(waiting for a worker to become available): %s' % event.artifact.basename()) progress = WorkerBuildWaiting(event.initiator_id, - event.artifact.source.cache_key) + event.artifact.cache_key) self.mainloop.queue_event(WorkerConnection, progress) else: @@ -279,7 +287,7 @@ class WorkerBuildQueuer(distbuild.StateMachine): self._give_job(job) else: progress = WorkerBuildWaiting(event.initiator_id, - event.artifact.source.cache_key) + event.artifact.cache_key) self.mainloop.queue_event(WorkerConnection, progress) def _handle_cancel(self, event_source, event): @@ -323,7 +331,7 @@ class WorkerBuildQueuer(distbuild.StateMachine): distbuild.crash_point() who = event.who - last_job = who.job() # the job this worker's just completed + last_job = who.current_job() # the job this worker's just completed if last_job: logging.debug('%s wants new job, just did %s', @@ -355,8 +363,22 @@ class WorkerBuildQueuer(distbuild.StateMachine): (job.artifact.name, worker.who.name())) self.mainloop.queue_event(worker.who, _HaveAJob(job)) - - + + def _handle_worker_disconnected(self, event): + self._remove_worker(self, event.who) + + def _remove_worker(self, worker): + logging.debug('WBQ: Removing worker %s from queue', worker.name()) + + # There should only be one InitiatorConnection instance per worker in + # the _available_workers list. But anything can happen in space! So we + # take care to remove all GiveJob messages in the list that came from + # the disconnected worker, not the first. + self._available_workers = filter( + lambda worker_msg: worker_msg.who != worker, + self._available_workers) + + class WorkerConnection(distbuild.StateMachine): '''Communicate with a single worker.''' @@ -372,9 +394,12 @@ class WorkerConnection(distbuild.StateMachine): self._writeable_cache_server = writeable_cache_server self._worker_cache_server_port = worker_cache_server_port self._morph_instance = morph_instance - self._helper_id = None - self._job = None - self._exec_response_msg = None + + self._active_jobs = dict() + self._current_job = None + self._current_job_exec_response = None + self._current_job_cache_request = None + self._debug_json = False addr, port = self._conn.getpeername() @@ -384,8 +409,8 @@ class WorkerConnection(distbuild.StateMachine): def name(self): return self._worker_name - def job(self): - return self._job + def current_job(self): + return self._current_job def setup(self): distbuild.crash_point() @@ -397,14 +422,15 @@ class WorkerConnection(distbuild.StateMachine): spec = [ # state, source, event_class, new_state, callback - ('idle', self._jm, distbuild.JsonEof, None, self._reconnect), + ('idle', self._jm, distbuild.JsonEof, None, self._disconnected), ('idle', self, _HaveAJob, 'building', self._start_build), ('building', distbuild.BuildController, distbuild.BuildCancel, 'building', self._maybe_cancel), - ('building', self._jm, distbuild.JsonEof, None, self._reconnect), + ('building', self._jm, distbuild.JsonEof, None, + self._disconnected), ('building', self._jm, distbuild.JsonNewMessage, 'building', self._handle_json_message), ('building', self, _BuildFailed, 'idle', self._request_job), @@ -412,6 +438,7 @@ class WorkerConnection(distbuild.StateMachine): ('building', self, _BuildFinished, 'caching', self._request_caching), + ('caching', self._jm, distbuild.JsonEof, None, self._disconnected), ('caching', distbuild.HelperRouter, distbuild.HelperResult, 'caching', self._maybe_handle_helper_result), ('caching', self, _Cached, 'idle', self._request_job), @@ -423,58 +450,69 @@ class WorkerConnection(distbuild.StateMachine): def _maybe_cancel(self, event_source, build_cancel): - if build_cancel.id not in self._job.initiators: + if build_cancel.id not in self._current_job.initiators: return # event not relevant logging.debug('WC: BuildController %r requested a cancel', event_source) - if (len(self._job.initiators) == 1): + job = self._current_job + if (len(job.initiators) == 1): logging.debug('WC: Cancelling running job %s ' 'with job id %s running on %s', - self._job.artifact.basename(), - self._job.id, + job.artifact.basename(), job.id, self.name()) - msg = distbuild.message('exec-cancel', id=self._job.id) + msg = distbuild.message('exec-cancel', id=job.id) self._jm.send(msg) self.mainloop.queue_event(self, _BuildCancelled()) else: logging.debug('WC: Not cancelling running job %s with job id %s, ' 'other initiators want it done: %s', - self._job.artifact.basename(), - self._job.id, - [i for i in self._job.initiators - if i != build_cancel.id]) + job.artifact.basename(), + job.id, + [i for i in job.initiators if i != build_cancel.id]) - self._job.initiators.remove(build_cancel.id) + job.initiators.remove(build_cancel.id) - def _reconnect(self, event_source, event): + def _disconnected(self, event_source, event): distbuild.crash_point() - logging.debug('WC: Triggering reconnect') + logging.debug('WC: Disconnected from worker %s' % self.name()) + self.mainloop.queue_event(InitiatorConnection, _Disconnected(self)) + self.mainloop.queue_event(self._cm, distbuild.Reconnect()) def _start_build(self, event_source, event): distbuild.crash_point() - self._job = event.job - self._helper_id = None - self._exec_response_msg = None + job = event.job + + if job.id in self._active_jobs: + logging.warn('Duplicate job %s for worker %s', job.id, self.name()) + + if self._current_job_exec_response or self._current_job_cache_request: + logging.warn('Caching not finished for %s', self._current_job.id) + + self._active_jobs[job.id] = job + self._current_job = job logging.debug('WC: starting build: %s for %s' % - (self._job.artifact.name, self._job.initiators)) + (job.artifact.name, job.initiators)) argv = [ self._morph_instance, 'worker-build', '--build-log-on-stdout', - self._job.artifact.name, + job.artifact.name, ] + msg = distbuild.message('exec-request', - id=self._job.id, + id=job.id, argv=argv, - stdin_contents=distbuild.serialise_artifact(self._job.artifact), + stdin_contents=distbuild.serialise_artifact(job.artifact, + job.artifact.repo, + job.artifact.ref), ) self._jm.send(msg) @@ -482,10 +520,10 @@ class WorkerConnection(distbuild.StateMachine): logging.debug('WC: sent to worker %s: %r' % (self._worker_name, msg)) - started = WorkerBuildStepStarted(self._job.initiators, - self._job.artifact.source.cache_key, self.name()) + started = WorkerBuildStepStarted(job.initiators, + job.artifact.cache_key, self.name()) - self.mainloop.queue_event(WorkerConnection, _JobStarted(self._job)) + self.mainloop.queue_event(WorkerConnection, _JobStarted(job)) self.mainloop.queue_event(WorkerConnection, started) def _handle_json_message(self, event_source, event): @@ -500,37 +538,50 @@ class WorkerConnection(distbuild.StateMachine): 'exec-output': self._handle_exec_output, 'exec-response': self._handle_exec_response, } - + handler = handlers[event.msg['type']] - handler(event.msg) + job = self._active_jobs.get(event.msg['id']) + + if job: + handler(event.msg, job) + else: + logging.warn('Received %s for unknown job %s', + event.msg['type'], event.msg['id']) + + def _handle_exec_output(self, msg, job): + '''Handle output from a job that the worker is or was running.''' - def _handle_exec_output(self, msg): new = dict(msg) - new['ids'] = self._job.initiators + new['ids'] = job.initiators + logging.debug('WC: emitting: %s', repr(new)) self.mainloop.queue_event( WorkerConnection, - WorkerBuildOutput(new, self._job.artifact.source.cache_key)) + WorkerBuildOutput(new, job.artifact.cache_key)) - def _handle_exec_response(self, msg): - logging.debug('WC: finished building: %s' % self._job.artifact.name) - logging.debug('initiators that need to know: %s' - % self._job.initiators) + def _handle_exec_response(self, msg, job): + '''Handle completion of a job that the worker is or was running.''' + + logging.debug('WC: finished building: %s' % job.artifact.name) + logging.debug('initiators that need to know: %s' % job.initiators) new = dict(msg) - new['ids'] = self._job.initiators + new['ids'] = job.initiators if new['exit'] != 0: # Build failed. - new_event = WorkerBuildFailed(new, - self._job.artifact.source.cache_key) + new_event = WorkerBuildFailed(new, job.artifact.cache_key) self.mainloop.queue_event(WorkerConnection, new_event) - self.mainloop.queue_event(WorkerConnection, _JobFailed(self._job)) + self.mainloop.queue_event(WorkerConnection, _JobFailed(job)) self.mainloop.queue_event(self, _BuildFailed()) else: # Build succeeded. We have more work to do: caching the result. self.mainloop.queue_event(self, _BuildFinished()) - self._exec_response_msg = new + self._current_job_exec_response = new + + # The job is no longer considered active, because the worker is + # finished with it so we won't receive any more messages about it. + del self._active_jobs[job.id] def _request_job(self, event_source, event): distbuild.crash_point() @@ -544,15 +595,16 @@ class WorkerConnection(distbuild.StateMachine): logging.debug('Requesting shared artifact cache to get artifacts') - kind = self._job.artifact.source.morphology['kind'] + job = self._current_job + kind = job.artifact.kind if kind == 'chunk': - source_artifacts = self._job.artifact.source.artifacts + source_artifacts = job.artifact.source_artifacts suffixes = ['%s.%s' % (kind, name) for name in source_artifacts] suffixes.append('build-log') else: - filename = '%s.%s' % (kind, self._job.artifact.name) + filename = '%s.%s' % (kind, job.artifact.name) suffixes = [filename] if kind == 'stratum': @@ -568,22 +620,22 @@ class WorkerConnection(distbuild.StateMachine): '/1.0/fetch?host=%s:%d&cacheid=%s&artifacts=%s' % (urllib.quote(worker_host), self._worker_cache_server_port, - urllib.quote(self._job.artifact.source.cache_key), + urllib.quote(job.artifact.cache_key), suffixes)) msg = distbuild.message( 'http-request', id=self._request_ids.next(), url=url, method='GET', body=None, headers=None) - self._helper_id = msg['id'] + self._current_job_cache_request = msg['id'] req = distbuild.HelperRequest(msg) self.mainloop.queue_event(distbuild.HelperRouter, req) - progress = WorkerBuildCaching(self._job.initiators, - self._job.artifact.source.cache_key) + progress = WorkerBuildCaching(job.initiators, + job.artifact.cache_key) self.mainloop.queue_event(WorkerConnection, progress) def _maybe_handle_helper_result(self, event_source, event): - if event.msg['id'] == self._helper_id: + if event.msg['id'] == self._current_job_cache_request: distbuild.crash_point() logging.debug('caching: event.msg: %s' % repr(event.msg)) @@ -591,8 +643,8 @@ class WorkerConnection(distbuild.StateMachine): logging.debug('Shared artifact cache population done') new_event = WorkerBuildFinished( - self._exec_response_msg, - self._job.artifact.source.cache_key) + self._current_job_exec_response, + self._current_job.artifact.cache_key) self.mainloop.queue_event(WorkerConnection, new_event) self.mainloop.queue_event(self, _Cached()) else: @@ -607,13 +659,17 @@ class WorkerConnection(distbuild.StateMachine): # The BuildController will not try to cancel jobs that have # been marked as failed. self.mainloop.queue_event(WorkerConnection, - _JobFailed(self._job)) + _JobFailed(self._current_job)) new_event = WorkerBuildFailed( - self._exec_response_msg, - self._job.artifact.source.cache_key) + self._current_job_exec_response, + self._current_job.artifact.cache_key) self.mainloop.queue_event(WorkerConnection, new_event) self.mainloop.queue_event(self, _BuildFailed()) - self.mainloop.queue_event(WorkerConnection, _JobFinished(self._job)) + self.mainloop.queue_event(WorkerConnection, + _JobFinished(self._current_job)) + + self._current_job_exec_response = None + self._current_job_cache_request = None @@ -1,6 +1,6 @@ #!/usr/bin/python # -# Copyright (C) 2011-2012 Codethink Limited +# Copyright (C) 2011-2012,2015 Codethink Limited # # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by @@ -12,8 +12,7 @@ # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License along -# with this program; if not, write to the Free Software Foundation, Inc., -# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. +# with this program. If not, see <http://www.gnu.org/licenses/>. import morphlib diff --git a/morph-cache-server b/morph-cache-server index 4af3cee3..007cfbe8 100755 --- a/morph-cache-server +++ b/morph-cache-server @@ -1,6 +1,6 @@ #!/usr/bin/env python # -# Copyright (C) 2013, 2014 Codethink Limited +# Copyright (C) 2013, 2014-2015 Codethink Limited # # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by @@ -12,8 +12,7 @@ # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License along -# with this program; if not, write to the Free Software Foundation, Inc., -# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. +# with this program. If not, see <http://www.gnu.org/licenses/>. import base64 @@ -1,4 +1,4 @@ -.\" Copyright (C) 2012 Codethink Limited +.\" Copyright (C) 2012,2015 Codethink Limited .\" .\" This program is free software; you can redistribute it and/or modify .\" it under the terms of the GNU General Public License as published by @@ -10,8 +10,7 @@ .\" GNU General Public License for more details. .\" .\" You should have received a copy of the GNU General Public License along -.\" with this program; if not, write to the Free Software Foundation, Inc., -.\" 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. +.\" with this program. If not, see <http://www.gnu.org/licenses/>. .\" .TH MORPH 1 .SH NAME diff --git a/morphcacheserver/__init__.py b/morphcacheserver/__init__.py index 2c25ce28..c646c1a9 100644 --- a/morphcacheserver/__init__.py +++ b/morphcacheserver/__init__.py @@ -1,4 +1,4 @@ -# Copyright (C) 2013 Codethink Limited +# Copyright (C) 2013,2015 Codethink Limited # # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by @@ -10,8 +10,7 @@ # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License along -# with this program; if not, write to the Free Software Foundation, Inc., -# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. +# with this program. If not, see <http://www.gnu.org/licenses/>. import repocache diff --git a/morphcacheserver/repocache.py b/morphcacheserver/repocache.py index 305c187c..d45cf86a 100644 --- a/morphcacheserver/repocache.py +++ b/morphcacheserver/repocache.py @@ -1,4 +1,4 @@ -# Copyright (C) 2013,2014 Codethink Limited +# Copyright (C) 2013,2014-2015 Codethink Limited # # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by @@ -10,8 +10,7 @@ # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License along -# with this program; if not, write to the Free Software Foundation, Inc., -# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. +# with this program. If not, see <http://www.gnu.org/licenses/>. import cliapp diff --git a/morphlib/__init__.py b/morphlib/__init__.py index f98c11aa..7c462aad 100644 --- a/morphlib/__init__.py +++ b/morphlib/__init__.py @@ -1,4 +1,4 @@ -# Copyright (C) 2011-2014 Codethink Limited +# Copyright (C) 2011-2015 Codethink Limited # # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by @@ -10,8 +10,7 @@ # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License along -# with this program; if not, write to the Free Software Foundation, Inc., -# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. +# with this program. If not, see <http://www.gnu.org/licenses/>. '''Baserock library.''' @@ -39,7 +38,7 @@ __version__ = gitversion.version # List of architectures that Morph supports valid_archs = ['armv7l', 'armv7lhf', 'armv7b', 'testarch', - 'x86_32', 'x86_64', 'ppc64'] + 'x86_32', 'x86_64', 'ppc64', 'armv8l64', 'armv8b64'] class Error(cliapp.AppException): @@ -56,7 +55,7 @@ import buildbranch import buildcommand import buildenvironment import buildsystem -import builder2 +import builder import cachedrepo import cachekeycomputer import extensions @@ -68,7 +67,6 @@ import gitindex import localartifactcache import localrepocache import mountableimage -import morphologyfactory import morphologyfinder import morphology import morphloader @@ -79,6 +77,7 @@ import repoaliasresolver import savefile import source import sourcepool +import sourceresolver import stagingarea import stopwatch import sysbranchdir diff --git a/morphlib/app.py b/morphlib/app.py index c3c9c970..c8fe397d 100644 --- a/morphlib/app.py +++ b/morphlib/app.py @@ -1,4 +1,4 @@ -# Copyright (C) 2011-2014 Codethink Limited +# Copyright (C) 2011-2015 Codethink Limited # # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by @@ -10,18 +10,16 @@ # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License along -# with this program; if not, write to the Free Software Foundation, Inc., -# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. +# with this program. If not, see <http://www.gnu.org/licenses/>. import cliapp -import collections import logging import os +import pipes import sys import time import urlparse -import warnings import extensions import morphlib @@ -140,6 +138,14 @@ class Morph(cliapp.Application): 'always push temporary build branches to the ' 'remote repository', group=group_build) + self.settings.choice (['local-changes'], + ['include', 'ignore'], + 'the `build` and `deploy` commands detect ' + 'uncommitted/unpushed local changes and operate ' + 'operate from a temporary branch containing ' + 'those changes. Disable this behaviour with the ' + '`ignore` setting.', + group=group_build) group_storage = 'Storage Options' self.settings.string(['tempdir'], @@ -290,154 +296,10 @@ class Morph(cliapp.Application): morphlib.util.sanitise_morphology_path(args[2])) args = args[3:] - def create_source_pool(self, lrc, rrc, repo, ref, filename, - original_ref=None): - pool = morphlib.sourcepool.SourcePool() - - def add_to_pool(reponame, ref, filename, absref, tree, morphology): - sources = morphlib.source.make_sources(reponame, ref, - filename, absref, - tree, morphology) - for source in sources: - pool.add(source) - - self.traverse_morphs(repo, ref, [filename], lrc, rrc, - update=not self.settings['no-git-update'], - visit=add_to_pool, - definitions_original_ref=original_ref) - return pool - - def resolve_ref(self, lrc, rrc, reponame, ref, update=True): - '''Resolves commit and tree sha1s of the ref in a repo and returns it. - - If update is True then this has the side-effect of updating - or cloning the repository into the local repo cache. - ''' - absref = None - - if lrc.has_repo(reponame): - repo = lrc.get_repo(reponame) - if update and repo.requires_update_for_ref(ref): - self.status(msg='Updating cached git repository %(reponame)s ' - 'for ref %(ref)s', reponame=reponame, ref=ref) - repo.update() - # If the user passed --no-git-update, and the ref is a SHA1 not - # available locally, this call will raise an exception. - absref, tree = repo.resolve_ref(ref) - elif rrc is not None: - try: - absref, tree = rrc.resolve_ref(reponame, ref) - if absref is not None: - self.status(msg='Resolved %(reponame)s %(ref)s via remote ' - 'repo cache', - reponame=reponame, - ref=ref, - chatty=True) - except BaseException, e: - logging.warning('Caught (and ignored) exception: %s' % str(e)) - if absref is None: - if update: - self.status(msg='Caching git repository %(reponame)s', - reponame=reponame) - repo = lrc.cache_repo(reponame) - repo.update() - else: - repo = lrc.get_repo(reponame) - absref, tree = repo.resolve_ref(ref) - return absref, tree - - def traverse_morphs(self, definitions_repo, definitions_ref, - system_filenames, lrc, rrc, update=True, - visit=lambda rn, rf, fn, arf, m: None, - definitions_original_ref=None): - morph_factory = morphlib.morphologyfactory.MorphologyFactory(lrc, rrc, - self) - definitions_queue = collections.deque(system_filenames) - chunk_in_definitions_repo_queue = [] - chunk_in_source_repo_queue = [] - resolved_refs = {} - resolved_morphologies = {} - - # Resolve the (repo, ref) pair for the definitions repo, cache result. - definitions_absref, definitions_tree = self.resolve_ref( - lrc, rrc, definitions_repo, definitions_ref, update) - - if definitions_original_ref: - definitions_ref = definitions_original_ref - - while definitions_queue: - filename = definitions_queue.popleft() - - key = (definitions_repo, definitions_absref, filename) - if not key in resolved_morphologies: - resolved_morphologies[key] = morph_factory.get_morphology(*key) - morphology = resolved_morphologies[key] - - visit(definitions_repo, definitions_ref, filename, - definitions_absref, definitions_tree, morphology) - if morphology['kind'] == 'cluster': - raise cliapp.AppException( - "Cannot build a morphology of type 'cluster'.") - elif morphology['kind'] == 'system': - definitions_queue.extend( - morphlib.util.sanitise_morphology_path(s['morph']) - for s in morphology['strata']) - elif morphology['kind'] == 'stratum': - if morphology['build-depends']: - definitions_queue.extend( - morphlib.util.sanitise_morphology_path(s['morph']) - for s in morphology['build-depends']) - for c in morphology['chunks']: - if 'morph' not in c: - path = morphlib.util.sanitise_morphology_path( - c.get('morph', c['name'])) - chunk_in_source_repo_queue.append( - (c['repo'], c['ref'], path)) - continue - chunk_in_definitions_repo_queue.append( - (c['repo'], c['ref'], c['morph'])) - - for repo, ref, filename in chunk_in_definitions_repo_queue: - if (repo, ref) not in resolved_refs: - resolved_refs[repo, ref] = self.resolve_ref( - lrc, rrc, repo, ref, update) - absref, tree = resolved_refs[repo, ref] - key = (definitions_repo, definitions_absref, filename) - if not key in resolved_morphologies: - resolved_morphologies[key] = morph_factory.get_morphology(*key) - morphology = resolved_morphologies[key] - visit(repo, ref, filename, absref, tree, morphology) - - for repo, ref, filename in chunk_in_source_repo_queue: - if (repo, ref) not in resolved_refs: - resolved_refs[repo, ref] = self.resolve_ref( - lrc, rrc, repo, ref, update) - absref, tree = resolved_refs[repo, ref] - key = (repo, absref, filename) - if key not in resolved_morphologies: - resolved_morphologies[key] = morph_factory.get_morphology(*key) - morphology = resolved_morphologies[key] - visit(repo, ref, filename, absref, tree, morphology) - - def cache_repo_and_submodules(self, cache, url, ref, done): - subs_to_process = set() - subs_to_process.add((url, ref)) - while subs_to_process: - url, ref = subs_to_process.pop() - done.add((url, ref)) - cached_repo = cache.cache_repo(url) - cached_repo.update() - - try: - submodules = morphlib.git.Submodules(self, cached_repo.path, - ref) - submodules.load() - except morphlib.git.NoModulesFileError: - pass - else: - for submod in submodules: - if (submod.url, submod.commit) not in done: - subs_to_process.add((submod.url, submod.commit)) + def _write_status(self, text): + timestamp = time.strftime('%Y-%m-%d %H:%M:%S', time.gmtime()) + self.output.write('%s %s\n' % (timestamp, text)) + self.output.flush() def status(self, **kwargs): '''Show user a status update. @@ -475,11 +337,22 @@ class Morph(cliapp.Application): ok = verbose or error or (not quiet and not chatty) if ok: - timestamp = time.strftime('%Y-%m-%d %H:%M:%S', time.gmtime()) - self.output.write('%s %s\n' % (timestamp, text)) - self.output.flush() + self._write_status(text) - def runcmd(self, argv, *args, **kwargs): + def _commandline_as_message(self, argv, args): + '''Create a status string for a command that's about to be executed.''' + + commands = [] + for command in [argv] + list(args): + if isinstance(command, list): + command_str = ' '.join(map(pipes.quote, command)) + else: + command_str = pipes.quote(command) + commands.append(command_str) + + return '# ' + ' | '.join(commands) + + def _prepare_for_runcmd(self, argv, args, kwargs): if 'env' not in kwargs: kwargs['env'] = dict(os.environ) @@ -489,28 +362,25 @@ class Morph(cliapp.Application): else: print_command = True - # convert the command line arguments into a string - commands = [argv] + list(args) - for command in commands: - if isinstance(command, list): - for i in xrange(0, len(command)): - command[i] = str(command[i]) - commands = [' '.join(command) for command in commands] - - # print the command line - if print_command: - self.status(msg='# %(cmdline)s', - cmdline=' | '.join(commands), - chatty=True) + if print_command and self.settings['verbose']: + # Don't call self.status() here, to avoid writing the message to + # the log as well as to the console. The cliapp.runcmd() function + # will also log the command, and it's messy having it logged twice. + self._write_status(self._commandline_as_message(argv, args)) # Log the environment. prev = getattr(self, 'prev_env', {}) morphlib.util.log_environment_changes(self, kwargs['env'], prev) self.prev_env = kwargs['env'] - # run the command line + def runcmd(self, argv, *args, **kwargs): + self._prepare_for_runcmd(argv, args, kwargs) return cliapp.Application.runcmd(self, argv, *args, **kwargs) + def runcmd_unchecked(self, argv, *args, **kwargs): + self._prepare_for_runcmd(argv, args, kwargs) + return cliapp.Application.runcmd_unchecked(self, argv, *args, **kwargs) + def parse_args(self, args, configs_only=False): return self.settings.parse_args(args, configs_only=configs_only, @@ -563,7 +433,7 @@ class Morph(cliapp.Application): self._help(True) def help_extensions(self, args): - exts = extensions.list_extensions(self.settings['build-ref-prefix']) + exts = extensions.list_extensions() template = "Extensions:\n %s\n" ext_string = '\n '.join(exts) self.output.write(template % (ext_string)) diff --git a/morphlib/artifact.py b/morphlib/artifact.py index 7a40a81a..47b16823 100644 --- a/morphlib/artifact.py +++ b/morphlib/artifact.py @@ -1,4 +1,4 @@ -# Copyright (C) 2012, 2013, 2014 Codethink Limited +# Copyright (C) 2012, 2013, 2014-2015 Codethink Limited # # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by @@ -10,8 +10,7 @@ # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License along -# with this program; if not, write to the Free Software Foundation, Inc., -# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. +# with this program. If not, see <http://www.gnu.org/licenses/>. class Artifact(object): diff --git a/morphlib/artifact_tests.py b/morphlib/artifact_tests.py index abd8767e..106459c8 100644 --- a/morphlib/artifact_tests.py +++ b/morphlib/artifact_tests.py @@ -1,4 +1,4 @@ -# Copyright (C) 2012-2014 Codethink Limited +# Copyright (C) 2012-2015 Codethink Limited # # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by @@ -10,8 +10,7 @@ # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License along -# with this program; if not, write to the Free Software Foundation, Inc., -# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. +# with this program. If not, see <http://www.gnu.org/licenses/>. import copy diff --git a/morphlib/artifactcachereference.py b/morphlib/artifactcachereference.py index 8211f6b5..55a26b6d 100644 --- a/morphlib/artifactcachereference.py +++ b/morphlib/artifactcachereference.py @@ -1,4 +1,4 @@ -# Copyright (C) 2012 Codethink Limited +# Copyright (C) 2012,2015 Codethink Limited # # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by @@ -10,8 +10,7 @@ # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License along -# with this program; if not, write to the Free Software Foundation, Inc., -# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. +# with this program. If not, see <http://www.gnu.org/licenses/>. class ArtifactCacheReference(object): diff --git a/morphlib/artifactresolver.py b/morphlib/artifactresolver.py index e53c7511..b49c1905 100644 --- a/morphlib/artifactresolver.py +++ b/morphlib/artifactresolver.py @@ -1,4 +1,4 @@ -# Copyright (C) 2012-2014 Codethink Limited +# Copyright (C) 2012-2015 Codethink Limited # # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by @@ -10,8 +10,7 @@ # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License along -# with this program; if not, write to the Free Software Foundation, Inc., -# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. +# with this program. If not, see <http://www.gnu.org/licenses/>. import cliapp @@ -195,13 +194,14 @@ class ArtifactResolver(object): chunk_source.add_dependency(other_stratum) # Add dependencies between chunks mentioned in this stratum - for name in build_depends: # pragma: no cover - if name not in name_to_processed_artifacts: - raise DependencyOrderError( - source, info['name'], name) - other_artifacts = name_to_processed_artifacts[name] - for other_artifact in other_artifacts: - chunk_source.add_dependency(other_artifact) + if build_depends is not None: + for name in build_depends: # pragma: no cover + if name not in name_to_processed_artifacts: + raise DependencyOrderError( + source, info['name'], name) + other_artifacts = name_to_processed_artifacts[name] + for other_artifact in other_artifacts: + chunk_source.add_dependency(other_artifact) # Add build dependencies between our stratum's artifacts # and the chunk artifacts produced by this stratum. diff --git a/morphlib/artifactresolver_tests.py b/morphlib/artifactresolver_tests.py index b958da4f..141ff948 100644 --- a/morphlib/artifactresolver_tests.py +++ b/morphlib/artifactresolver_tests.py @@ -1,4 +1,4 @@ -# Copyright (C) 2012-2014 Codethink Limited +# Copyright (C) 2012-2015 Codethink Limited # # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by @@ -10,8 +10,7 @@ # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License along -# with this program; if not, write to the Free Software Foundation, Inc., -# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. +# with this program. If not, see <http://www.gnu.org/licenses/>. import itertools diff --git a/morphlib/artifactsplitrule.py b/morphlib/artifactsplitrule.py index 1511d694..b5ebdf83 100644 --- a/morphlib/artifactsplitrule.py +++ b/morphlib/artifactsplitrule.py @@ -1,4 +1,4 @@ -# Copyright (C) 2013-2014 Codethink Limited +# Copyright (C) 2013-2015 Codethink Limited # # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by @@ -10,8 +10,7 @@ # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License along -# with this program; if not, write to the Free Software Foundation, Inc., -# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. +# with this program. If not, see <http://www.gnu.org/licenses/>. import collections diff --git a/morphlib/bins.py b/morphlib/bins.py index 560e68bb..2e8ba0b3 100644 --- a/morphlib/bins.py +++ b/morphlib/bins.py @@ -1,4 +1,4 @@ -# Copyright (C) 2011-2014 Codethink Limited +# Copyright (C) 2011-2015 Codethink Limited # # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by @@ -10,8 +10,7 @@ # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License along -# with this program; if not, write to the Free Software Foundation, Inc., -# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. +# with this program. If not, see <http://www.gnu.org/licenses/>. '''Functions for dealing with Baserock binaries. @@ -75,7 +74,7 @@ if sys.version_info < (2, 7, 3): # pragma: no cover else: if sys.platform != "os2emx": os.chown(targetpath, u, g) - except EnvironmentError, e: + except EnvironmentError as e: raise ExtractError("could not change owner") tarfile.TarFile.chown = fixed_chown @@ -187,7 +186,7 @@ def unpack_binary_from_file(f, dirname): # pragma: no cover prepare_extract(tarinfo, targetpath) try: ret = real(tarinfo, targetpath) - except (IOError, OSError), e: + except (IOError, OSError) as e: if e.errno != errno.EEXIST: if e.filename is None: e.filename = targetpath diff --git a/morphlib/bins_tests.py b/morphlib/bins_tests.py index 60361ece..3895680f 100644 --- a/morphlib/bins_tests.py +++ b/morphlib/bins_tests.py @@ -1,4 +1,4 @@ -# Copyright (C) 2011-2014 Codethink Limited +# Copyright (C) 2011-2015 Codethink Limited # # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by @@ -10,8 +10,7 @@ # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License along -# with this program; if not, write to the Free Software Foundation, Inc., -# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. +# with this program. If not, see <http://www.gnu.org/licenses/>. import gzip diff --git a/morphlib/branchmanager.py b/morphlib/branchmanager.py index a33b4ccb..92a1f4be 100644 --- a/morphlib/branchmanager.py +++ b/morphlib/branchmanager.py @@ -1,4 +1,4 @@ -# Copyright (C) 2013 Codethink Limited +# Copyright (C) 2013,2015 Codethink Limited # # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by @@ -10,8 +10,7 @@ # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License along -# with this program; if not, write to the Free Software Foundation, Inc., -# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. +# with this program. If not, see <http://www.gnu.org/licenses/>. import cliapp @@ -92,7 +91,7 @@ class LocalRefManager(object): op, args = d.pop() try: op(*args) - except Exception, e: + except Exception as e: exceptions.append((op, args, e)) if exceptions: raise RefCleanupError(primary, exceptions) @@ -201,7 +200,7 @@ class RemoteRefManager(object): remote, refspecs = d.pop() try: remote.push(*refspecs) - except Exception, e: + except Exception as e: exceptions.append((remote, refspecs, e)) if exceptions: raise RefCleanupError(primary, exceptions) diff --git a/morphlib/branchmanager_tests.py b/morphlib/branchmanager_tests.py index cf3be73c..9f3740d7 100644 --- a/morphlib/branchmanager_tests.py +++ b/morphlib/branchmanager_tests.py @@ -1,4 +1,4 @@ -# Copyright (C) 2013-2014 Codethink Limited +# Copyright (C) 2013-2015 Codethink Limited # # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by @@ -10,8 +10,7 @@ # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License along -# with this program; if not, write to the Free Software Foundation, Inc., -# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. +# with this program. If not, see <http://www.gnu.org/licenses/>. import cliapp diff --git a/morphlib/buildbranch.py b/morphlib/buildbranch.py index 2d529133..80cecd75 100644 --- a/morphlib/buildbranch.py +++ b/morphlib/buildbranch.py @@ -1,4 +1,4 @@ -# Copyright (C) 2013-2014 Codethink Limited +# Copyright (C) 2013-2015 Codethink Limited # # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by @@ -10,8 +10,7 @@ # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License along -# with this program; if not, write to the Free Software Foundation, Inc., -# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. +# with this program. If not, see <http://www.gnu.org/licenses/>. import collections @@ -35,6 +34,15 @@ class BuildBranchCleanupError(cliapp.AppException): % locals()) +class NoReposError(cliapp.AppException): + def __init__(self, bb, ignored): + self.bb = bb + cliapp.AppException.__init__( + self, "No repos were found in system branch (ignored %i which " + "didn't have the right morph.uuid setting)" % (ignored)) + + + class BuildBranch(object): '''Represent the sources modified in a system branch. @@ -61,12 +69,12 @@ class BuildBranch(object): self._branch_root = sb.get_config('branch.root') branch_uuid = sb.get_config('branch.uuid') - for gd in sb.list_git_directories(): + for count, gd in enumerate(sb.list_git_directories()): try: repo_uuid = gd.get_config('morph.uuid') except cliapp.AppException: # Not a repository cloned by morph, ignore - break + continue build_ref = os.path.join('refs/heads', build_ref_prefix, branch_uuid, repo_uuid) # index is commit of workspace + uncommitted changes may want @@ -76,6 +84,9 @@ class BuildBranch(object): index.set_to_tree(gd.resolve_ref_to_tree(gd.HEAD)) self._to_push[gd] = (build_ref, index) + if len(self._to_push) == 0: + raise NoReposError(self, count) + rootinfo, = ((gd, index) for gd, (build_ref, index) in self._to_push.iteritems() if gd.get_config('morph.repository') == self._branch_root) @@ -103,7 +114,7 @@ class BuildBranch(object): for morphology in morphologies: loader.unset_defaults(morphology) sha1 = gd.store_blob(loader.save_to_string(morphology)) - yield 0100644, sha1, morphology.filename + yield 0o100644, sha1, morphology.filename def inject_build_refs(self, loader, use_local_repos, inject_cb=lambda **kwargs: None): @@ -281,7 +292,7 @@ class BuildBranch(object): func, args, kwargs = self._cleanup.pop() try: func(*args, **kwargs) - except Exception, e: + except Exception as e: exceptions.append(e) if exceptions: raise BuildBranchCleanupError(self, exceptions) diff --git a/morphlib/buildcommand.py b/morphlib/buildcommand.py index 438badb3..be8a1507 100644 --- a/morphlib/buildcommand.py +++ b/morphlib/buildcommand.py @@ -1,4 +1,4 @@ -# Copyright (C) 2011-2014 Codethink Limited +# Copyright (C) 2011-2015 Codethink Limited # # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by @@ -10,8 +10,7 @@ # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License along -# with this program; if not, write to the Free Software Foundation, Inc., -# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. +# with this program. If not, see <http://www.gnu.org/licenses/>. import itertools @@ -94,10 +93,12 @@ class BuildCommand(object): ''' self.app.status(msg='Creating source pool', chatty=True) - srcpool = self.app.create_source_pool( + srcpool = morphlib.sourceresolver.create_source_pool( self.lrc, self.rrc, repo_name, ref, filename, - original_ref=original_ref) - + cachedir=self.app.settings['cachedir'], + original_ref=original_ref, + update_repos=not self.app.settings['no-git-update'], + status_cb=self.app.status) return srcpool def validate_sources(self, srcpool): @@ -125,11 +126,21 @@ class BuildCommand(object): root_arch = root_artifact.source.morphology['arch'] host_arch = morphlib.util.get_host_architecture() - if root_arch != host_arch: - raise morphlib.Error( - 'Are you trying to cross-build? ' - 'Host architecture is %s but target is %s' - % (host_arch, root_arch)) + + if root_arch == host_arch: + return + + # Since the armv8 instruction set is nearly entirely armv7 compatible, + # and since the incompatibilities are appropriately trapped in the + # kernel, we can safely run any armv7 toolchain natively on armv8. + if host_arch == 'armv8l' and root_arch in ('armv7l', 'armv7lhf'): + return + if host_arch == 'armv8b' and root_arch in ('armv7b', 'armv7bhf'): + return + + raise morphlib.Error( + 'Are you trying to cross-build? Host architecture is %s but ' + 'target is %s' % (host_arch, root_arch)) @staticmethod def _validate_has_non_bootstrap_chunks(srcpool): @@ -260,7 +271,7 @@ class BuildCommand(object): def build_in_order(self, root_artifact): '''Build everything specified in a build order.''' - self.app.status(msg='Building a set of sources', chatty=True) + self.app.status(msg='Building a set of sources') build_env = root_artifact.build_env ordered_sources = list(self.get_ordered_sources(root_artifact.walk())) old_prefix = self.app.status_prefix @@ -375,39 +386,8 @@ class BuildCommand(object): '''Update the local git repository cache with the sources.''' repo_name = source.repo_name - if self.app.settings['no-git-update']: - self.app.status(msg='Not updating existing git repository ' - '%(repo_name)s ' - 'because of no-git-update being set', - chatty=True, - repo_name=repo_name) - source.repo = self.lrc.get_repo(repo_name) - return - - if self.lrc.has_repo(repo_name): - source.repo = self.lrc.get_repo(repo_name) - try: - sha1 = source.sha1 - source.repo.resolve_ref(sha1) - self.app.status(msg='Not updating git repository ' - '%(repo_name)s because it ' - 'already contains sha1 %(sha1)s', - chatty=True, repo_name=repo_name, - sha1=sha1) - except morphlib.cachedrepo.InvalidReferenceError: - self.app.status(msg='Updating %(repo_name)s', - repo_name=repo_name) - source.repo.update() - else: - self.app.status(msg='Cloning %(repo_name)s', - repo_name=repo_name) - source.repo = self.lrc.cache_repo(repo_name) - - # Update submodules. - done = set() - self.app.cache_repo_and_submodules( - self.lrc, source.repo.url, - source.sha1, done) + source.repo = self.lrc.get_updated_repo(repo_name, ref=source.sha1) + self.lrc.ensure_submodules(source.repo, source.sha1) def cache_artifacts_locally(self, artifacts): '''Get artifacts missing from local cache from remote cache.''' @@ -517,7 +497,7 @@ class BuildCommand(object): staging_area.install_artifact(handle) if target_source.build_mode == 'staging': - morphlib.builder2.ldconfig(self.app.runcmd, staging_area.dirname) + morphlib.builder.ldconfig(self.app.runcmd, staging_area.dirname) def build_and_cache(self, staging_area, source, setup_mounts): '''Build a source and put its artifacts into the local cache.''' @@ -525,7 +505,7 @@ class BuildCommand(object): self.app.status(msg='Starting actual build: %(name)s ' '%(sha1)s', name=source.name, sha1=source.sha1[:7]) - builder = morphlib.builder2.Builder( + builder = morphlib.builder.Builder( self.app, staging_area, self.lac, self.rac, self.lrc, self.app.settings['max-jobs'], setup_mounts) return builder.build_and_cache(source) @@ -563,4 +543,12 @@ class InitiatorBuildCommand(BuildCommand): self.MAX_RETRIES) loop.add_state_machine(cm) - loop.run() + try: + loop.run() + except KeyboardInterrupt: + # This will run if the user presses Ctrl+C or sends SIGINT during + # the build. It won't trigger on SIGTERM, SIGKILL or unhandled + # Python exceptions. + logging.info('Received KeyboardInterrupt, aborting.') + for initiator in loop.state_machines_of_type(distbuild.Initiator): + initiator.handle_cancel() diff --git a/morphlib/buildenvironment.py b/morphlib/buildenvironment.py index 68e7e756..6ec82d45 100644 --- a/morphlib/buildenvironment.py +++ b/morphlib/buildenvironment.py @@ -1,4 +1,4 @@ -# Copyright (C) 2012-2013 Codethink Limited +# Copyright (C) 2012-2013, 2015 Codethink Limited # # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by @@ -10,8 +10,7 @@ # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License along -# with this program; if not, write to the Free Software Foundation, Inc., -# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. +# with this program. If not, see <http://www.gnu.org/licenses/>. import copy import cliapp @@ -115,10 +114,14 @@ class BuildEnvironment(): # than leaving it up to individual morphologies. if arch == 'x86_32': cpu = 'i686' + elif arch == 'armv8l64': # pragma: no cover + cpu = 'aarch64' + elif arch == 'armv8b64': # pragma: no cover + cpu = 'aarch64_be' else: cpu = arch - if arch.startswith('arm'): + if arch.startswith('armv7'): abi = 'eabi' else: abi = '' diff --git a/morphlib/buildenvironment_tests.py b/morphlib/buildenvironment_tests.py index 7ae7c2d5..59a56aa2 100644 --- a/morphlib/buildenvironment_tests.py +++ b/morphlib/buildenvironment_tests.py @@ -1,4 +1,4 @@ -# Copyright (C) 2012-2013 Codethink Limited +# Copyright (C) 2012-2013,2015 Codethink Limited # # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by @@ -10,8 +10,7 @@ # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License along -# with this program; if not, write to the Free Software Foundation, Inc., -# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. +# with this program. If not, see <http://www.gnu.org/licenses/>. import copy diff --git a/morphlib/builder2.py b/morphlib/builder.py index f71f21db..04ebd149 100644 --- a/morphlib/builder2.py +++ b/morphlib/builder.py @@ -1,4 +1,4 @@ -# Copyright (C) 2012-2014 Codethink Limited +# Copyright (C) 2012-2015 Codethink Limited # # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by @@ -10,13 +10,9 @@ # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License along -# with this program; if not, write to the Free Software Foundation, Inc., -# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. +# with this program. If not, see <http://www.gnu.org/licenses/>. -from collections import defaultdict -import datetime -import errno import json import logging import os @@ -28,12 +24,12 @@ import time import traceback import subprocess import tempfile -import gzip import cliapp import morphlib from morphlib.artifactcachereference import ArtifactCacheReference +from morphlib.util import error_message_for_containerised_commandline import morphlib.gitversion SYSTEM_INTEGRATION_PATH = os.path.join('baserock', 'system-integration') @@ -145,23 +141,6 @@ def download_depends(constituents, lac, rac, metadatas=None): src.close() -def get_chunk_files(f): # pragma: no cover - tar = tarfile.open(fileobj=f) - for member in tar.getmembers(): - if member.type is not tarfile.DIRTYPE: - yield member.name - tar.close() - - -def get_stratum_files(f, lac): # pragma: no cover - for ca in (ArtifactCacheReference(a) - for a in json.load(f, encoding='unicode-escape')): - cf = lac.get(ca) - for filename in get_chunk_files(cf): - yield filename - cf.close() - - class BuilderBase(object): '''Base class for building artifacts.''' @@ -310,7 +289,7 @@ class ChunkBuilder(BuilderBase): self.create_devices(destdir) os.rename(temppath, logpath) - except BaseException, e: + except BaseException as e: logging.error('Caught exception: %s' % str(e)) logging.info('Cleaning up staging area') self.staging_area.chroot_close() @@ -396,7 +375,7 @@ class ChunkBuilder(BuilderBase): if stdout: stdout.flush() - except cliapp.AppException, e: + except cliapp.AppException as e: if not stdout: with open(logfilepath, 'r') as log: self.app.output.write("%s failed\n" % step) @@ -426,7 +405,7 @@ class ChunkBuilder(BuilderBase): with morphlib.savefile.SaveFile(script_path, 'w') as f: f.write("#!/bin/sh\nset -xeu\n") f.write(script) - os.chmod(script_path, 0555) + os.chmod(script_path, 0o555) rel_script_path = os.path.join(SYSTEM_INTEGRATION_PATH, script_name) @@ -644,7 +623,7 @@ class SystemBuilder(BuilderBase): # pragma: no cover f.write('SUPPORT_URL="http://wiki.baserock.org/mailinglist"\n') f.write('BUG_REPORT_URL="http://wiki.baserock.org/mailinglist"\n') - os.chmod(os_release_file, 0644) + os.chmod(os_release_file, 0o644) def run_system_integration_commands(self, rootdir): # pragma: no cover ''' Run the system integration commands ''' @@ -665,12 +644,19 @@ class SystemBuilder(BuilderBase): # pragma: no cover ) try: for bin in sorted(os.listdir(sys_integration_dir)): - self.app.runcmd( - morphlib.util.containerised_cmdline( - [os.path.join(SYSTEM_INTEGRATION_PATH, bin)], - root=rootdir, mounts=to_mount, mount_proc=True), - env=env) - except BaseException, e: + argv = [os.path.join(SYSTEM_INTEGRATION_PATH, bin)] + container_config = dict( + root=rootdir, mounts=to_mount, mount_proc=True) + cmdline = morphlib.util.containerised_cmdline( + argv, **container_config) + exit, out, err = self.app.runcmd_unchecked( + cmdline, env=env) + if exit != 0: + logging.debug('Command returned code %i', exit) + msg = error_message_for_containerised_commandline( + argv, err, container_config) + raise cliapp.AppException(msg) + except BaseException as e: self.app.status( msg='Error while running system integration commands', error=True) diff --git a/morphlib/builder2_tests.py b/morphlib/builder_tests.py index 4fd0807a..a571e3d0 100644 --- a/morphlib/builder2_tests.py +++ b/morphlib/builder_tests.py @@ -1,4 +1,4 @@ -# Copyright (C) 2012-2014 Codethink Limited +# Copyright (C) 2012-2015 Codethink Limited # # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by @@ -10,8 +10,7 @@ # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License along -# with this program; if not, write to the Free Software Foundation, Inc., -# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. +# with this program. If not, see <http://www.gnu.org/licenses/>. import json @@ -20,6 +19,7 @@ import StringIO import unittest import morphlib +import morphlib.gitdir_tests class FakeBuildSystem(object): @@ -50,8 +50,9 @@ class FakeSource(object): } self.name = 'a' - self.repo = morphlib.cachedrepo.CachedRepo(FakeApp(), 'repo', - 'url', 'path') + with morphlib.gitdir_tests.allow_nonexistant_git_repos(): + self.repo = morphlib.cachedrepo.CachedRepo( + FakeApp(), 'repo', 'url', 'path') self.repo_name = 'url' self.original_ref = 'e' self.sha1 = 'f' @@ -154,7 +155,7 @@ class BuilderBaseTests(unittest.TestCase): self.repo_cache = None self.build_env = FakeBuildEnv() self.max_jobs = 1 - self.builder = morphlib.builder2.BuilderBase(self.app, + self.builder = morphlib.builder.BuilderBase(self.app, self.staging_area, self.artifact_cache, None, @@ -193,7 +194,7 @@ class BuilderBaseTests(unittest.TestCase): fh = rac.put(a) fh.write(a.name) fh.close() - morphlib.builder2.download_depends(afacts, lac, rac) + morphlib.builder.download_depends(afacts, lac, rac) self.assertTrue(all(lac.has(a) for a in afacts)) def test_downloads_depends_metadata(self): @@ -207,7 +208,7 @@ class BuilderBaseTests(unittest.TestCase): fh = rac.put_artifact_metadata(a, 'meta') fh.write('metadata') fh.close() - morphlib.builder2.download_depends(afacts, lac, rac, ('meta',)) + morphlib.builder.download_depends(afacts, lac, rac, ('meta',)) self.assertTrue(all(lac.has(a) for a in afacts)) self.assertTrue(all(lac.has_artifact_metadata(a, 'meta') for a in afacts)) @@ -217,5 +218,5 @@ class ChunkBuilderTests(unittest.TestCase): def setUp(self): self.app = FakeApp() - self.build = morphlib.builder2.ChunkBuilder(self.app, None, None, + self.build = morphlib.builder.ChunkBuilder(self.app, None, None, None, None, None, 1, False) diff --git a/morphlib/buildsystem.py b/morphlib/buildsystem.py index fb99e70e..7ed50d11 100644 --- a/morphlib/buildsystem.py +++ b/morphlib/buildsystem.py @@ -1,4 +1,4 @@ -# Copyright (C) 2012-2014 Codethink Limited +# Copyright (C) 2012-2015 Codethink Limited # # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by @@ -10,8 +10,7 @@ # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License along -# with this program; if not, write to the Free Software Foundation, Inc., -# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. +# with this program. If not, see <http://www.gnu.org/licenses/>. import os diff --git a/morphlib/buildsystem_tests.py b/morphlib/buildsystem_tests.py index 56ba64d7..80898ebd 100644 --- a/morphlib/buildsystem_tests.py +++ b/morphlib/buildsystem_tests.py @@ -1,4 +1,4 @@ -# Copyright (C) 2012-2014 Codethink Limited +# Copyright (C) 2012-2015 Codethink Limited # # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by @@ -10,8 +10,7 @@ # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License along -# with this program; if not, write to the Free Software Foundation, Inc., -# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. +# with this program. If not, see <http://www.gnu.org/licenses/>. import os diff --git a/morphlib/cachedrepo.py b/morphlib/cachedrepo.py index aad3d84e..23639043 100644 --- a/morphlib/cachedrepo.py +++ b/morphlib/cachedrepo.py @@ -1,4 +1,4 @@ -# Copyright (C) 2012-2014 Codethink Limited +# Copyright (C) 2012-2015 Codethink Limited # # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by @@ -10,31 +10,17 @@ # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License along -# with this program; if not, write to the Free Software Foundation, Inc., -# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. +# with this program. If not, see <http://www.gnu.org/licenses/>. import cliapp -import logging + import os +import tempfile import morphlib -class InvalidReferenceError(cliapp.AppException): - - def __init__(self, repo, ref): - cliapp.AppException.__init__( - self, 'Ref %s is an invalid reference for repo %s' % (ref, repo)) - - -class UnresolvedNamedReferenceError(cliapp.AppException): - - def __init__(self, repo, ref): - cliapp.AppException.__init__( - self, 'Ref %s is not a SHA1 ref for repo %s' % (ref, repo)) - - class CheckoutDirectoryExistsError(cliapp.AppException): def __init__(self, repo, target_dir): @@ -105,63 +91,52 @@ class CachedRepo(object): self.is_mirror = not url.startswith('file://') self.already_updated = False - def ref_exists(self, ref): - '''Returns True if the given ref exists in the repo''' + self._gitdir = morphlib.gitdir.GitDirectory(path) - try: - self._rev_parse(ref) - except cliapp.AppException: - return False - return True + def ref_exists(self, ref): # pragma: no cover + '''Returns True if the given ref exists in the repo''' + return self._gitdir.ref_exists(ref) - def resolve_ref(self, ref): - '''Attempts to resolve a ref into its SHA1 and tree SHA1. + def resolve_ref_to_commit(self, ref): # pragma: no cover + '''Resolve a named ref to a commit SHA1. - Raises an InvalidReferenceError if the ref is not found in the - repository. + Raises gitdir.InvalidRefError if the ref does not exist. ''' + return self._gitdir.resolve_ref_to_commit(ref) - try: - absref = self._rev_parse(ref) - except cliapp.AppException: - raise InvalidReferenceError(self, ref) + def resolve_ref_to_tree(self, ref): # pragma: no cover + '''Resolve a named ref to a tree SHA1. - try: - tree = self._show_tree_hash(absref) - except cliapp.AppException: - raise InvalidReferenceError(self, ref) + Raises gitdir.InvalidRefError if the ref does not exist. - return absref, tree + ''' + return self._gitdir.resolve_ref_to_tree(ref) - def cat(self, ref, filename): - '''Attempts to read a file given a SHA1 ref. + def read_file(self, filename, ref): # pragma: no cover + '''Attempts to read a file from a given ref. - Raises an UnresolvedNamedReferenceError if the ref is not a SHA1 - ref. Raises an InvalidReferenceError if the SHA1 ref is not found - in the repository. Raises an IOError if the requested file is not - found in the ref. + Raises a gitdir.InvalidRefError if the ref is not found in the + repository. Raises an IOError if the requested file is not found in + the ref. ''' + return self._gitdir.read_file(filename, ref) - if not morphlib.git.is_valid_sha1(ref): - raise UnresolvedNamedReferenceError(self, ref) - try: - sha1 = self._rev_parse(ref) - except cliapp.AppException: - raise InvalidReferenceError(self, ref) + def list_files(self, ref, recurse=True): # pragma: no cover + '''Return filenames found in the tree pointed to by the given ref. - try: - return self._cat_file(sha1, filename) - except cliapp.AppException: - raise IOError('File %s does not exist in ref %s of repo %s' % - (filename, ref, self)) + Returns a gitdir.InvalidRefError if the ref is not found in the + repository. + + ''' + return self._gitdir.list_files(ref, recurse) def clone_checkout(self, ref, target_dir): '''Clone from the cache into the target path and check out a given ref. Raises a CheckoutDirectoryExistsError if the target - directory already exists. Raises an InvalidReferenceError if the + directory already exists. Raises a gitdir.InvalidRefError if the ref is not found in the repository. Raises a CheckoutError if something else goes wrong while copying the repository or checking out the SHA1 ref. @@ -171,14 +146,14 @@ class CachedRepo(object): if os.path.exists(target_dir): raise CheckoutDirectoryExistsError(self, target_dir) - self.resolve_ref(ref) + self._gitdir.resolve_ref_to_commit(ref) self._clone_into(target_dir, ref) def checkout(self, ref, target_dir): '''Unpacks the repository in a directory and checks out a commit ref. - Raises an InvalidReferenceError if the ref is not found in the + Raises an gitdir.InvalidRefError if the ref is not found in the repository. Raises a CopyError if something goes wrong with the copy of the repository. Raises a CheckoutError if something else goes wrong while copying the repository or checking out the SHA1 ref. @@ -193,25 +168,29 @@ class CachedRepo(object): # take care to turn the copy into something as good as a real clone. self._copy_repository(self.path, target_dir) - self._checkout_ref(ref, target_dir) + self._checkout_ref_in_clone(ref, target_dir) - def ls_tree(self, ref): - '''Return file names found in root tree. Does not recurse to subtrees. + def extract_commit(self, ref, target_dir): + '''Extract files from a given commit into target_dir. - Raises an UnresolvedNamedReferenceError if the ref is not a SHA1 - ref. Raises an InvalidReferenceError if the SHA1 ref is not found - in the repository. + This is different to a 'checkout': a checkout assumes a working tree + associated with a repository. Here, the repository is immutable (it's + in the cache) and we just want to look at the files in a quick way + (quicker than going 'git cat-file everything'). - ''' + This seems marginally quicker than doing a shallow clone. Running + `morph list-artifacts` 10 times gave an average time of 1.334s + using `git clone --depth 1` and an average time of 1.261s using + this code. - if not morphlib.git.is_valid_sha1(ref): - raise UnresolvedNamedReferenceError(self, ref) - try: - sha1 = self._rev_parse(ref) - except cliapp.AppException: - raise InvalidReferenceError(self, ref) + ''' + if not os.path.exists(target_dir): + os.makedirs(target_dir) - return self._ls_tree(sha1) + with tempfile.NamedTemporaryFile() as index_file: + index = self._gitdir.get_index(index_file=index_file.name) + index.set_to_tree(ref) + index.checkout(working_tree=target_dir) def requires_update_for_ref(self, ref): '''Returns False if there's no need to update this cached repo. @@ -232,7 +211,7 @@ class CachedRepo(object): # Named refs that are valid SHA1s will confuse this code. ref_can_change = not morphlib.git.is_valid_sha1(ref) - if ref_can_change or not self.ref_exists(ref): + if ref_can_change or not self._gitdir.ref_exists(ref): return True else: return False @@ -249,9 +228,10 @@ class CachedRepo(object): return try: - self._update() + self._gitdir.update_remotes( + echo_stderr=self.app.settings['verbose']) self.already_updated = True - except cliapp.AppException, e: + except cliapp.AppException: raise UpdateError(self) def _runcmd(self, *args, **kwargs): # pragma: no cover @@ -259,27 +239,11 @@ class CachedRepo(object): kwargs['cwd'] = self.path return self.app.runcmd(*args, **kwargs) - def _rev_parse(self, ref): # pragma: no cover - return morphlib.git.gitcmd(self._runcmd, 'rev-parse', '--verify', - '%s^{commit}' % ref)[0:40] - - def _show_tree_hash(self, absref): # pragma: no cover - return morphlib.git.gitcmd(self._runcmd, 'rev-parse', '--verify', - '%s^{tree}' % absref).strip() - - def _ls_tree(self, ref): # pragma: no cover - result = morphlib.git.gitcmd(self._runcmd, 'ls-tree', - '--name-only', ref) - return result.split('\n') - - def _cat_file(self, ref, filename): # pragma: no cover - return morphlib.git.gitcmd(self._runcmd, 'cat-file', 'blob', - '%s:%s' % (ref, filename)) - - def _clone_into(self, target_dir, ref): #pragma: no cover + def _clone_into(self, target_dir, ref): # pragma: no cover '''Actually perform the clone''' try: - morphlib.git.clone_into(self._runcmd, self.path, target_dir, ref) + morphlib.git.clone_into(self._runcmd, self.path, target_dir, + ref) except cliapp.AppException: raise CloneError(self, target_dir) @@ -290,16 +254,15 @@ class CachedRepo(object): except cliapp.AppException: raise CopyError(self, target_dir) - def _checkout_ref(self, ref, target_dir): # pragma: no cover + def _checkout_ref_in_clone(self, ref, clone_dir): # pragma: no cover + # This is a separate GitDirectory instance. Don't confuse it with the + # internal ._gitdir attribute! + working_gitdir = morphlib.gitdir.GitDirectory(clone_dir) try: - morphlib.git.checkout_ref(self._runcmd, target_dir, ref) - except cliapp.AppException: - raise CheckoutError(self, ref, target_dir) - - def _update(self): # pragma: no cover - morphlib.git.gitcmd(self._runcmd, 'remote', 'update', - 'origin', '--prune', - echo_stderr=self.app.settings['verbose']) + working_gitdir.checkout(ref) + except cliapp.AppException as e: + raise CheckoutError(self, ref, clone_dir) + return working_gitdir def __str__(self): # pragma: no cover return self.url diff --git a/morphlib/cachedrepo_tests.py b/morphlib/cachedrepo_tests.py index d3ae331a..404a0778 100644 --- a/morphlib/cachedrepo_tests.py +++ b/morphlib/cachedrepo_tests.py @@ -1,4 +1,4 @@ -# Copyright (C) 2012-2014 Codethink Limited +# Copyright (C) 2012-2015 Codethink Limited # # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by @@ -10,8 +10,7 @@ # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License along -# with this program; if not, write to the Free Software Foundation, Inc., -# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. +# with this program. If not, see <http://www.gnu.org/licenses/>. import logging @@ -22,14 +21,33 @@ import fs.tempfs import cliapp import morphlib +import morphlib.gitdir_tests -class CachedRepoTests(unittest.TestCase): +class FakeApplication(object): + + def __init__(self): + self.settings = { + 'verbose': True + } + + +class FakeIndex(object): + + def __init__(self, index_file): + self.index_file = index_file + self.ref = None + + def set_to_tree(self, ref): + self.ref = ref + + def checkout(self, working_tree=None): + if working_tree: + with open(os.path.join(working_tree, 'foo.morph'), 'w') as f: + f.write('contents of foo.morph') - EXAMPLE_MORPH = '''{ - "name": "foo", - "kind": "chunk" - }''' + +class CachedRepoTests(unittest.TestCase): known_commit = 'a4da32f5a81c8bc6d660404724cedc3bc0914a75' bad_sha1_known_to_rev_parse = 'cafecafecafecafecafecafecafecafecafecafe' @@ -44,85 +62,46 @@ class CachedRepoTests(unittest.TestCase): 'master': 'e28a23812eadf2fce6583b8819b9c5dbd36b9fb9', 'baserock/morph': '8b780e2e6f102fcf400ff973396566d36d730501' } + ref = ref.rstrip('^{commit}') try: return output[ref] except KeyError: raise cliapp.AppException('git rev-parse --verify %s' % ref) - def show_tree_hash(self, absref): - output = { - 'e28a23812eadf2fce6583b8819b9c5dbd36b9fb9': - 'eeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeee', - '8b780e2e6f102fcf400ff973396566d36d730501': - 'ffffffffffffffffffffffffffffffffffffffff', - 'a4da32f5a81c8bc6d660404724cedc3bc0914a75': - 'dddddddddddddddddddddddddddddddddddddddd' - } - try: - return output[absref] - except KeyError: - raise cliapp.AppException('git log -1 --format=format:%%T %s' % - absref) - - def cat_file(self, ref, filename): - output = { - 'e28a23812eadf2fce6583b8819b9c5dbd36b9fb9:foo.morph': - self.EXAMPLE_MORPH - } - try: - return output['%s:%s' % (ref, filename)] - except KeyError: - raise cliapp.AppException( - 'git cat-file blob %s:%s' % (ref, filename)) - def copy_repository(self, source_dir, target_dir): if target_dir.endswith('failed-checkout'): raise morphlib.cachedrepo.CopyError(self.repo, target_dir) def checkout_ref(self, ref, target_dir): - if ref == 'a4da32f5a81c8bc6d660404724cedc3bc0914a75': - raise morphlib.cachedrepo.CloneError(self.repo, target_dir) - elif ref == '079bbfd447c8534e464ce5d40b80114c2022ebf4': + if ref == '079bbfd447c8534e464ce5d40b80114c2022ebf4': raise morphlib.cachedrepo.CheckoutError(self.repo, ref, target_dir) else: with open(os.path.join(target_dir, 'foo.morph'), 'w') as f: f.write('contents of foo.morph') - def ls_tree(self, ref): - output = { - 'e28a23812eadf2fce6583b8819b9c5dbd36b9fb9': - ['foo.morph'] - } - try: - return output[ref] - except KeyError: - raise cliapp.AppException('git ls-tree --name-only %s' % (ref)) - def clone_into(self, target_dir, ref): if target_dir.endswith('failed-checkout'): raise morphlib.cachedrepo.CloneError(self.repo, target_dir) self.clone_target = target_dir self.clone_ref = ref - def update_successfully(self): + def update_successfully(self, **kwargs): pass - def update_with_failure(self): + def update_with_failure(self, **kwargs): raise cliapp.AppException('git remote update origin') + def get_index(self, index_file=None): + return FakeIndex(index_file) + def setUp(self): self.repo_name = 'foo' self.repo_url = 'git://foo.bar/foo.git' self.repo_path = '/tmp/foo' - self.repo = morphlib.cachedrepo.CachedRepo( - object(), self.repo_name, self.repo_url, self.repo_path) - self.repo._rev_parse = self.rev_parse - self.repo._show_tree_hash = self.show_tree_hash - self.repo._cat_file = self.cat_file - self.repo._copy_repository = self.copy_repository - self.repo._checkout_ref = self.checkout_ref - self.repo._ls_tree = self.ls_tree - self.repo._clone_into = self.clone_into + with morphlib.gitdir_tests.allow_nonexistant_git_repos(): + self.repo = morphlib.cachedrepo.CachedRepo( + FakeApplication(), self.repo_name, self.repo_url, + self.repo_path) self.tempfs = fs.tempfs.TempFS() def test_constructor_sets_name_and_url_and_path(self): @@ -130,87 +109,47 @@ class CachedRepoTests(unittest.TestCase): self.assertEqual(self.repo.url, self.repo_url) self.assertEqual(self.repo.path, self.repo_path) - def test_ref_exists(self): - self.assertEqual(self.repo.ref_exists('master'), True) - - def test_ref_does_not_exist(self): - self.assertEqual(self.repo.ref_exists('non-existant-ref'), False) - - def test_resolve_named_ref_master(self): - sha1, tree = self.repo.resolve_ref('master') - self.assertEqual(sha1, 'e28a23812eadf2fce6583b8819b9c5dbd36b9fb9') - self.assertEqual(tree, 'eeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeee') - - def test_resolve_named_ref_baserock_morph(self): - sha1, tree = self.repo.resolve_ref('baserock/morph') - self.assertEqual(sha1, '8b780e2e6f102fcf400ff973396566d36d730501') - self.assertEqual(tree, 'ffffffffffffffffffffffffffffffffffffffff') - - def test_fail_resolving_invalid_named_ref(self): - self.assertRaises(morphlib.cachedrepo.InvalidReferenceError, - self.repo.resolve_ref, 'foo/bar') - - def test_resolve_sha1_ref(self): - sha1, tree = self.repo.resolve_ref( - 'e28a23812eadf2fce6583b8819b9c5dbd36b9fb9') - self.assertEqual(sha1, 'e28a23812eadf2fce6583b8819b9c5dbd36b9fb9') - self.assertEqual(tree, 'eeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeee') - - def test_fail_resolving_an_invalid_sha1_ref(self): - self.assertRaises(morphlib.cachedrepo.InvalidReferenceError, - self.repo.resolve_ref, - self.bad_sha1_known_to_rev_parse) - - def test_cat_existing_file_in_existing_ref(self): - data = self.repo.cat('e28a23812eadf2fce6583b8819b9c5dbd36b9fb9', - 'foo.morph') - self.assertEqual(data, self.EXAMPLE_MORPH) - - def test_fail_cat_file_in_invalid_ref(self): - self.assertRaises( - morphlib.cachedrepo.InvalidReferenceError, self.repo.cat, - '079bbfd447c8534e464ce5d40b80114c2022ebf4', - 'doesnt-matter-whether-this-file-exists') - - def test_fail_cat_non_existent_file_in_existing_ref(self): - self.assertRaises(IOError, self.repo.cat, - 'e28a23812eadf2fce6583b8819b9c5dbd36b9fb9', - 'file-that-does-not-exist') - - def test_fail_cat_non_existent_file_in_invalid_ref(self): - self.assertRaises( - morphlib.cachedrepo.InvalidReferenceError, self.repo.cat, - '079bbfd447c8534e464ce5d40b80114c2022ebf4', - 'file-that-does-not-exist') - - def test_fail_because_cat_in_named_ref_is_not_allowed(self): - self.assertRaises(morphlib.cachedrepo.UnresolvedNamedReferenceError, - self.repo.cat, 'master', 'doesnt-matter') - def test_fail_clone_checkout_into_existing_directory(self): + self.repo._gitdir.checkout = self.checkout_ref + self.repo._clone_into = self.clone_into + self.assertRaises(morphlib.cachedrepo.CheckoutDirectoryExistsError, self.repo.clone_checkout, 'e28a23812eadf2fce6583b8819b9c5dbd36b9fb9', self.tempfs.root_path) def test_fail_checkout_due_to_clone_error(self): + self.repo._gitdir._rev_parse = self.rev_parse + self.repo._clone_into = self.clone_into + self.assertRaises( morphlib.cachedrepo.CloneError, self.repo.clone_checkout, 'a4da32f5a81c8bc6d660404724cedc3bc0914a75', self.tempfs.getsyspath('failed-checkout')) def test_fail_checkout_due_to_copy_error(self): + self.repo._gitdir._rev_parse = self.rev_parse + self.repo._copy_repository = self.copy_repository + self.assertRaises(morphlib.cachedrepo.CopyError, self.repo.checkout, 'a4da32f5a81c8bc6d660404724cedc3bc0914a75', self.tempfs.getsyspath('failed-checkout')) def test_fail_checkout_from_invalid_ref(self): + self.repo._gitdir._rev_parse = self.rev_parse + self.repo._copy_repository = self.copy_repository + self.repo._checkout_ref_in_clone = self.checkout_ref + self.assertRaises( morphlib.cachedrepo.CheckoutError, self.repo.checkout, '079bbfd447c8534e464ce5d40b80114c2022ebf4', self.tempfs.getsyspath('checkout-from-invalid-ref')) def test_checkout_from_existing_ref_into_new_directory(self): + self.repo._gitdir._rev_parse = self.rev_parse + self.repo._copy_repository = self.copy_repository + self.repo._checkout_ref_in_clone = self.checkout_ref + unpack_dir = self.tempfs.getsyspath('unpack-dir') self.repo.checkout('e28a23812eadf2fce6583b8819b9c5dbd36b9fb9', unpack_dir) @@ -219,35 +158,38 @@ class CachedRepoTests(unittest.TestCase): morph_filename = os.path.join(unpack_dir, 'foo.morph') self.assertTrue(os.path.exists(morph_filename)) - def test_ls_tree_in_existing_ref(self): - data = self.repo.ls_tree('e28a23812eadf2fce6583b8819b9c5dbd36b9fb9') - self.assertEqual(data, ['foo.morph']) - - def test_fail_ls_tree_in_invalid_ref(self): - self.assertRaises( - morphlib.cachedrepo.InvalidReferenceError, self.repo.ls_tree, - '079bbfd447c8534e464ce5d40b80114c2022ebf4') + def test_extract_commit_into_new_directory(self): + self.repo._gitdir.get_index = self.get_index + unpack_dir = self.tempfs.getsyspath('unpack-dir') + self.repo.extract_commit('e28a23812eadf2fce6583b8819b9c5dbd36b9fb9', + unpack_dir) + self.assertTrue(os.path.exists(unpack_dir)) - def test_fail_because_ls_tree_in_named_ref_is_not_allowed(self): - self.assertRaises(morphlib.cachedrepo.UnresolvedNamedReferenceError, - self.repo.ls_tree, 'master') + morph_filename = os.path.join(unpack_dir, 'foo.morph') + self.assertTrue(os.path.exists(morph_filename)) def test_successful_update(self): - self.repo._update = self.update_successfully + self.repo._gitdir.update_remotes = self.update_successfully self.repo.update() def test_failing_update(self): - self.repo._update = self.update_with_failure + self.repo._gitdir.update_remotes = self.update_with_failure self.assertRaises(morphlib.cachedrepo.UpdateError, self.repo.update) def test_no_update_if_local(self): - self.repo = morphlib.cachedrepo.CachedRepo( - object(), 'local:repo', 'file:///local/repo/', '/local/repo/') - self.repo._update = self.update_with_failure + with morphlib.gitdir_tests.allow_nonexistant_git_repos(): + self.repo = morphlib.cachedrepo.CachedRepo( + object(), 'local:repo', 'file:///local/repo/', '/local/repo/') + self.repo._gitdir.update_remotes = self.update_with_failure + self.repo._gitdir._rev_parse = self.rev_parse + self.assertFalse(self.repo.requires_update_for_ref(self.known_commit)) self.repo.update() def test_clone_checkout(self): + self.repo._gitdir._rev_parse = self.rev_parse + self.repo._clone_into = self.clone_into + self.repo.clone_checkout('master', '/.DOES_NOT_EXIST') self.assertEqual(self.clone_target, '/.DOES_NOT_EXIST') self.assertEqual(self.clone_ref, 'master') @@ -256,11 +198,14 @@ class CachedRepoTests(unittest.TestCase): # If the SHA1 is present locally already there's no need to update. # If it's a named ref then it might have changed in the remote, so we # must still update. + self.repo._gitdir._rev_parse = self.rev_parse + self.assertFalse(self.repo.requires_update_for_ref(self.known_commit)) self.assertTrue(self.repo.requires_update_for_ref('named_ref')) def test_no_need_to_update_repo_if_already_updated(self): - self.repo._update = self.update_successfully + self.repo._gitdir.update_remotes = self.update_successfully + self.repo._gitdir._rev_parse = self.rev_parse self.assertTrue(self.repo.requires_update_for_ref('named_ref')) self.repo.update() diff --git a/morphlib/cachekeycomputer.py b/morphlib/cachekeycomputer.py index c3a01b9e..33eef082 100644 --- a/morphlib/cachekeycomputer.py +++ b/morphlib/cachekeycomputer.py @@ -1,4 +1,4 @@ -# Copyright (C) 2012-2014 Codethink Limited +# Copyright (C) 2012-2015 Codethink Limited # # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by @@ -10,8 +10,7 @@ # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License along -# with this program; if not, write to the Free Software Foundation, Inc., -# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. +# with this program. If not, see <http://www.gnu.org/licenses/>. import hashlib diff --git a/morphlib/cachekeycomputer_tests.py b/morphlib/cachekeycomputer_tests.py index 57739983..d0a4657f 100644 --- a/morphlib/cachekeycomputer_tests.py +++ b/morphlib/cachekeycomputer_tests.py @@ -1,4 +1,4 @@ -# Copyright (C) 2012-2014 Codethink Limited +# Copyright (C) 2012-2015 Codethink Limited # # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by @@ -10,8 +10,7 @@ # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License along -# with this program; if not, write to the Free Software Foundation, Inc., -# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. +# with this program. If not, see <http://www.gnu.org/licenses/>. import copy diff --git a/morphlib/extensions.py b/morphlib/extensions.py index b270d304..aa0622b8 100644 --- a/morphlib/extensions.py +++ b/morphlib/extensions.py @@ -1,4 +1,4 @@ -# Copyright (C) 2014 Codethink Limited +# Copyright (C) 2014-2015 Codethink Limited # # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by @@ -10,8 +10,7 @@ # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License along -# with this program; if not, write to the Free Software Foundation, Inc., -# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. +# with this program. If not, see <http://www.gnu.org/licenses/>. import asyncore import asynchat @@ -38,6 +37,7 @@ class ExtensionNotExecutableError(ExtensionError): pass def _get_root_repo(): + workspace = morphlib.workspace.open('.') system_branch = morphlib.sysbranchdir.open_from_within('.') root_repo_dir = morphlib.gitdir.GitDirectory( system_branch.get_git_directory_name( @@ -77,7 +77,8 @@ def _list_extensions(kind): try: repo_extension_filenames = \ _list_repo_extension_filenames(kind) - except (sysbranchdir.NotInSystemBranch): + except (morphlib.workspace.NotInWorkspace, + sysbranchdir.NotInSystemBranch): # Squash this and just return no system branch extensions pass morph_extension_filenames = _list_morph_extension_filenames(kind) @@ -147,7 +148,7 @@ class get_extension_filename(): fd, ext_filename = tempfile.mkstemp() os.write(fd, ext_contents) os.close(fd) - os.chmod(ext_filename, 0700) + os.chmod(ext_filename, 0o700) self.delete = True self.ext_filename = ext_filename diff --git a/morphlib/extractedtarball.py b/morphlib/extractedtarball.py index fd98cd92..95c0582d 100644 --- a/morphlib/extractedtarball.py +++ b/morphlib/extractedtarball.py @@ -1,4 +1,4 @@ -# Copyright (C) 2012-2013 Codethink Limited +# Copyright (C) 2012-2013,2015 Codethink Limited # # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by @@ -10,8 +10,7 @@ # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License along -# with this program; if not, write to the Free Software Foundation, Inc., -# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. +# with this program. If not, see <http://www.gnu.org/licenses/>. import cliapp @@ -42,7 +41,7 @@ class ExtractedTarball(object): # pragma: no cover self.tempdir = tempfile.mkdtemp(dir=self.app.settings['tempdir']) try: morphlib.bins.unpack_binary(self.tarball, self.tempdir) - except BaseException, e: + except BaseException as e: logging.error('Caught exception: %s' % str(e)) logging.debug('Removing temporary directory %s' % self.tempdir) shutil.rmtree(self.tempdir) @@ -54,7 +53,7 @@ class ExtractedTarball(object): # pragma: no cover tarball=os.path.basename(self.tarball), chatty=True) try: shutil.rmtree(self.tempdir) - except BaseException, e: + except BaseException as e: logging.warning( 'Error when removing temporary directory %s: %s' % (self.tempdir, str(e))) diff --git a/morphlib/exts/add-config-files.configure b/morphlib/exts/add-config-files.configure index 0094cf6b..2cf96fd1 100755 --- a/morphlib/exts/add-config-files.configure +++ b/morphlib/exts/add-config-files.configure @@ -1,5 +1,5 @@ #!/bin/sh -# Copyright (C) 2013 Codethink Limited +# Copyright (C) 2013,2015 Codethink Limited # # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by @@ -11,8 +11,7 @@ # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License along -# with this program; if not, write to the Free Software Foundation, Inc., -# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. +# with this program. If not, see <http://www.gnu.org/licenses/>. # Copy all files located in $SRC_CONFIG_DIR to the image /etc. diff --git a/morphlib/exts/fstab.configure b/morphlib/exts/fstab.configure index a1287ea4..3bbc9102 100755 --- a/morphlib/exts/fstab.configure +++ b/morphlib/exts/fstab.configure @@ -1,5 +1,5 @@ #!/usr/bin/python -# Copyright (C) 2013 Codethink Limited +# Copyright (C) 2013,2015 Codethink Limited # # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by @@ -11,8 +11,7 @@ # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License along -# with this program; if not, write to the Free Software Foundation, Inc., -# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. +# with this program. If not, see <http://www.gnu.org/licenses/>. # # =*= License: GPL-2 =*= diff --git a/morphlib/exts/initramfs.write b/morphlib/exts/initramfs.write index f8af6d84..1059defa 100755 --- a/morphlib/exts/initramfs.write +++ b/morphlib/exts/initramfs.write @@ -1,5 +1,5 @@ #!/bin/sh -# Copyright (C) 2014 Codethink Limited +# Copyright (C) 2014-2015 Codethink Limited # # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by @@ -11,8 +11,7 @@ # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License along -# with this program; if not, write to the Free Software Foundation, Inc., -# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. +# with this program. If not, see <http://www.gnu.org/licenses/>. # # =*= License: GPL-2 =*= diff --git a/morphlib/exts/initramfs.write.help b/morphlib/exts/initramfs.write.help index 29a9d266..54d3ae8c 100644 --- a/morphlib/exts/initramfs.write.help +++ b/morphlib/exts/initramfs.write.help @@ -1,4 +1,19 @@ +# Copyright (C) 2014, 2015 Codethink Limited +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation; version 2 of the License. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License along +# with this program; if not, see <http://www.gnu.org/licenses/>. + help: | + Create an initramfs for a system by taking an existing system and converting it to the appropriate format. @@ -33,3 +48,8 @@ help: | initramfs: type: initramfs location: boot/initramfs.gz + + Parameters: + + * location: the path where the initramfs will be installed (e.g. + `boot/initramfs.gz`) in the above example diff --git a/morphlib/exts/install-files.configure b/morphlib/exts/install-files.configure index 04dc5f18..58cf373a 100755 --- a/morphlib/exts/install-files.configure +++ b/morphlib/exts/install-files.configure @@ -1,5 +1,5 @@ #!/usr/bin/python -# Copyright (C) 2013-2014 Codethink Limited +# Copyright (C) 2013-2015 Codethink Limited # # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by @@ -11,8 +11,7 @@ # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License along -# with this program; if not, write to the Free Software Foundation, Inc., -# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. +# with this program. If not, see <http://www.gnu.org/licenses/>. ''' A Morph configuration extension for adding arbitrary files to a system diff --git a/morphlib/exts/install-files.configure.help b/morphlib/exts/install-files.configure.help index eb3aab0c..991c26c8 100644 --- a/morphlib/exts/install-files.configure.help +++ b/morphlib/exts/install-files.configure.help @@ -1,3 +1,17 @@ +# Copyright (C) 2014, 2015 Codethink Limited +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation; version 2 of the License. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License along +# with this program; if not, see <http://www.gnu.org/licenses/>. + help: | Install a set of files onto a system diff --git a/morphlib/exts/kvm.check b/morphlib/exts/kvm.check index 1bb4007a..62d76453 100755 --- a/morphlib/exts/kvm.check +++ b/morphlib/exts/kvm.check @@ -1,5 +1,5 @@ #!/usr/bin/python -# Copyright (C) 2014 Codethink Limited +# Copyright (C) 2014-2015 Codethink Limited # # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by @@ -11,12 +11,12 @@ # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License along -# with this program; if not, write to the Free Software Foundation, Inc., -# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. +# with this program. If not, see <http://www.gnu.org/licenses/>. '''Preparatory checks for Morph 'kvm' write extension''' import cliapp +import os import re import urlparse @@ -43,8 +43,10 @@ class KvmPlusSshCheckExtension(morphlib.writeexts.WriteExtension): ssh_host, vm_name, vm_path = self.check_and_parse_location(location) self.check_ssh_connectivity(ssh_host) + self.check_can_create_file_at_given_path(ssh_host, vm_path) self.check_no_existing_libvirt_vm(ssh_host, vm_name) self.check_extra_disks_exist(ssh_host, self.parse_attach_disks()) + self.check_virtual_networks_are_started(ssh_host) def check_and_parse_location(self, location): '''Check and parse the location argument to get relevant data.''' @@ -73,6 +75,26 @@ class KvmPlusSshCheckExtension(morphlib.writeexts.WriteExtension): 'write extension to deploy upgrades to existing machines.' % (ssh_host, vm_name)) + def check_can_create_file_at_given_path(self, ssh_host, vm_path): + + def check_can_write_to_given_path(): + try: + cliapp.ssh_runcmd(ssh_host, ['touch', vm_path]) + except cliapp.AppException as e: + raise cliapp.AppException("Can't write to location %s on %s" + % (vm_path, ssh_host)) + else: + cliapp.ssh_runcmd(ssh_host, ['rm', vm_path]) + + try: + cliapp.ssh_runcmd(ssh_host, ['test', '-e', vm_path]) + except cliapp.AppException as e: + # vm_path doesn't already exist, so let's test we can write + check_can_write_to_given_path() + else: + raise cliapp.AppException('%s already exists on %s' + % (vm_path, ssh_host)) + def check_extra_disks_exist(self, ssh_host, filename_list): for filename in filename_list: try: @@ -81,4 +103,50 @@ class KvmPlusSshCheckExtension(morphlib.writeexts.WriteExtension): raise cliapp.AppException('Did not find file %s on host %s' % (filename, ssh_host)) + def check_virtual_networks_are_started(self, ssh_host): + + def check_virtual_network_is_started(network_name): + cmd = ['virsh', '-c', 'qemu:///system', 'net-info', network_name] + net_info = cliapp.ssh_runcmd(ssh_host, cmd).split('\n') + + def pretty_concat(lines): + return '\n'.join(['\t%s' % line for line in lines]) + + for line in net_info: + m = re.match('^Active:\W*(\w+)\W*', line) + if m: + break + else: + raise cliapp.AppException( + "Got unexpected output parsing output of `%s':\n%s" + % (' '.join(cmd), pretty_concat(net_info))) + + network_active = m.group(1) == 'yes' + + if not network_active: + raise cliapp.AppException("Network '%s' is not started" + % network_name) + + def name(nic_entry): + if ',' in nic_entry: + # NETWORK_NAME,mac=12:34,model=e1000... + return nic_entry[:nic_entry.find(',')] + else: + return nic_entry # NETWORK_NAME + + if 'NIC_CONFIG' in os.environ: + nics = os.environ['NIC_CONFIG'].split() + + # --network bridge= is used to specify a bridge + # --network user is used to specify a form of NAT + # (see the virt-install(1) man page) + networks = [name(n) for n in nics if not n.startswith('bridge=') + and not n.startswith('user')] + else: + networks = ['default'] + + for network in networks: + check_virtual_network_is_started(network) + + KvmPlusSshCheckExtension().run() diff --git a/morphlib/exts/kvm.write b/morphlib/exts/kvm.write index 16f188b5..0d0c095b 100755 --- a/morphlib/exts/kvm.write +++ b/morphlib/exts/kvm.write @@ -1,5 +1,5 @@ #!/usr/bin/python -# Copyright (C) 2012-2014 Codethink Limited +# Copyright (C) 2012-2015 Codethink Limited # # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by @@ -11,11 +11,14 @@ # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License along -# with this program; if not, write to the Free Software Foundation, Inc., -# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. +# with this program. If not, see <http://www.gnu.org/licenses/>. -'''A Morph deployment write extension for deploying to KVM+libvirt.''' +'''A Morph deployment write extension for deploying to KVM+libvirt. + +See file kvm.write.help for documentation + +''' import cliapp @@ -30,40 +33,20 @@ import morphlib.writeexts class KvmPlusSshWriteExtension(morphlib.writeexts.WriteExtension): - '''Create a KVM/LibVirt virtual machine during Morph's deployment. - - The location command line argument is the pathname of the disk image - to be created. The user is expected to provide the location argument - using the following syntax: - - kvm+ssh://HOST/GUEST/PATH - - where: - - * HOST is the host on which KVM/LibVirt is running - * GUEST is the name of the guest virtual machine on that host - * PATH is the path to the disk image that should be created, - on that host - - The extension will connect to HOST via ssh to run libvirt's - command line management tools. - - ''' - location_pattern = '^/(?P<guest>[^/]+)(?P<path>/.+)$' def process_args(self, args): if len(args) != 2: raise cliapp.AppException('Wrong number of command line args') - + temp_root, location = args ssh_host, vm_name, vm_path = self.parse_location(location) autostart = self.get_environment_boolean('AUTOSTART') - + fd, raw_disk = tempfile.mkstemp() os.close(fd) self.create_local_system(temp_root, raw_disk) - + try: self.transfer(raw_disk, ssh_host, vm_path) self.create_libvirt_guest(ssh_host, vm_name, vm_path, autostart) @@ -105,7 +88,7 @@ class KvmPlusSshWriteExtension(morphlib.writeexts.WriteExtension): def create_libvirt_guest(self, ssh_host, vm_name, vm_path, autostart): '''Create the libvirt virtual machine.''' - + self.status(msg='Creating libvirt/kvm virtual machine') attach_disks = self.parse_attach_disks() @@ -135,4 +118,3 @@ class KvmPlusSshWriteExtension(morphlib.writeexts.WriteExtension): ['virsh', '--connect', 'qemu:///system', 'autostart', vm_name]) KvmPlusSshWriteExtension().run() - diff --git a/morphlib/exts/kvm.write.help b/morphlib/exts/kvm.write.help index 8b5053a5..812a5309 100644 --- a/morphlib/exts/kvm.write.help +++ b/morphlib/exts/kvm.write.help @@ -1,4 +1,90 @@ +# Copyright (C) 2014, 2015 Codethink Limited +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation; version 2 of the License. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License along +# with this program; if not, see <http://www.gnu.org/licenses/>. + help: | - The INITRAMFS_PATH option can be used to specify the location of an - initramfs for syslinux to tell Linux to use, rather than booting - the rootfs directly. + + Deploy a Baserock system as a *new* KVM/LibVirt virtual machine. + + Use the `ssh-rsync` write extension to deploy upgrades to an *existing* VM + + Parameters: + + * location: a custom URL scheme of the form `kvm+ssh://HOST/GUEST/PATH`, + where: + * HOST is the name of the host on which KVM/LibVirt is running + * GUEST is the name of the guest VM on that host + * PATH is the path to the disk image that should be created, + on that host. For example, + `kvm+ssh://alice@192.168.122.1/testsys/home/alice/testys.img` where + * `alice@192.168.122.1` is the target host as given to ssh, + **from within the development host** (which may be + different from the target host's normal address); + * `testsys` is the name of the new guest VM'; + * `/home/alice/testys.img` is the pathname of the disk image files + on the target host. + + * HOSTNAME=name: the hostname of the **guest** VM within the network into + which it is being deployed + + * DISK_SIZE=X: the size of the VM's primary virtual hard disk. `X` should + use a suffix of `K`, `M`, or `G` (in upper or lower case) to indicate + kilo-, mega-, or gigabytes. For example, `DISK_SIZE=100G` would create a + 100 gigabyte disk image. **This parameter is mandatory**. + + * RAM_SIZE=X: The amount of RAM that the virtual machine should allocate + for itself from the host. `X` is interpreted in the same was as for + DISK_SIZE`, and defaults to `1G` + + * VCPUS=n: the number of virtual CPUs for the VM. Allowed values 1-32. Do + not use more CPU cores than you have available physically (real cores, no + hyperthreads) + + * INITRAMFS_PATH=path: the location of an initramfs for the bootloader to + tell Linux to use, rather than booting the rootfs directly. + + * AUTOSTART=<VALUE>` - boolean. If it is set, the VM will be started when + it has been deployed. + + * DTB_PATH=path: **(MANDATORY)** for systems that require a device tree + binary - Give the full path (without a leading /) to the location of the + DTB in the built system image . The deployment will fail if `path` does + not exist. + + * BOOTLOADER_INSTALL=value: the bootloader to be installed + **(MANDATORY)** for non-x86 systems + + allowed values = + - 'extlinux' (default) - the extlinux bootloader will + be installed + - 'none' - no bootloader will be installed by `morph deploy`. A + bootloader must be installed manually. This value must be used when + deploying non-x86 systems such as ARM. + + * BOOTLOADER_CONFIG_FORMAT=value: the bootloader format to be used. + If not specified for x86-32 and x86-64 systems, 'extlinux' will be used + + allowed values = + - 'extlinux' + + * KERNEL_ARGS=args: optional additional kernel command-line parameters to + be appended to the default set. The default set is: + + 'rw init=/sbin/init rootfstype=btrfs \ + rootflags=subvol=systems/default/run \ + root=[name or UUID of root filesystem]' + + (See https://www.kernel.org/doc/Documentation/kernel-parameters.txt) + + (See `morph help deploy` for details of how to pass parameters to write + extensions) diff --git a/morphlib/exts/nfsboot.check b/morphlib/exts/nfsboot.check index 806e560a..e273f61c 100755 --- a/morphlib/exts/nfsboot.check +++ b/morphlib/exts/nfsboot.check @@ -1,5 +1,5 @@ #!/usr/bin/python -# Copyright (C) 2014 Codethink Limited +# Copyright (C) 2014-2015 Codethink Limited # # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by @@ -11,8 +11,7 @@ # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License along -# with this program; if not, write to the Free Software Foundation, Inc., -# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. +# with this program. If not, see <http://www.gnu.org/licenses/>. '''Preparatory checks for Morph 'nfsboot' write extension''' diff --git a/morphlib/exts/nfsboot.configure b/morphlib/exts/nfsboot.configure index 660d9c39..6a68dc48 100755 --- a/morphlib/exts/nfsboot.configure +++ b/morphlib/exts/nfsboot.configure @@ -1,5 +1,5 @@ #!/bin/sh -# Copyright (C) 2013-2014 Codethink Limited +# Copyright (C) 2013-2015 Codethink Limited # # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by @@ -11,8 +11,7 @@ # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License along -# with this program; if not, write to the Free Software Foundation, Inc., -# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. +# with this program. If not, see <http://www.gnu.org/licenses/>. # Remove all networking interfaces. On nfsboot systems, eth0 is set up diff --git a/morphlib/exts/nfsboot.write b/morphlib/exts/nfsboot.write index 8d3d6df7..d928775e 100755 --- a/morphlib/exts/nfsboot.write +++ b/morphlib/exts/nfsboot.write @@ -1,5 +1,5 @@ #!/usr/bin/python -# Copyright (C) 2013-2014 Codethink Limited +# Copyright (C) 2013-2015 Codethink Limited # # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by @@ -11,12 +11,21 @@ # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License along -# with this program; if not, write to the Free Software Foundation, Inc., -# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. +# with this program. If not, see <http://www.gnu.org/licenses/>. '''A Morph deployment write extension for deploying to an nfsboot server +*** DO NOT USE *** +- This was written before 'proper' deployment mechanisms were in place +It is unlikely to work at all and will not work correctly + +Use the pxeboot write extension instead + +*** + + + An nfsboot server is defined as a baserock system that has tftp and nfs servers running, the tftp server is exporting the contents of /srv/nfsboot/tftp/ and the user has sufficient permissions to create nfs roots @@ -125,7 +134,7 @@ class NFSBootWriteExtension(morphlib.writeexts.WriteExtension): self.status(msg='Creating destination directories') try: - cliapp.ssh_runcmd('root@%s' % location, + cliapp.ssh_runcmd('root@%s' % location, ['mkdir', '-p', orig_path, run_path]) except cliapp.AppException: raise cliapp.AppException('Could not create dirs %s and %s on %s' @@ -191,4 +200,3 @@ mv "$temp" "$target" NFSBootWriteExtension().run() - diff --git a/morphlib/exts/nfsboot.write.help b/morphlib/exts/nfsboot.write.help index 598b1b23..186c479a 100644 --- a/morphlib/exts/nfsboot.write.help +++ b/morphlib/exts/nfsboot.write.help @@ -1,6 +1,27 @@ +# Copyright (C) 2014, 2015 Codethink Limited +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation; version 2 of the License. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License along +# with this program; if not, see <http://www.gnu.org/licenses/>. + help: | + *** DO NOT USE *** + - This was written before 'proper' deployment mechanisms were in place. + It is unlikely to work at all, and will not work correctly. + + Use the pxeboot write extension instead + + *** Deploy a system image and kernel to an nfsboot server. - + An nfsboot server is defined as a baserock system that has tftp and nfs servers running, the tftp server is exporting the contents of /srv/nfsboot/tftp/ and the user has sufficient diff --git a/morphlib/exts/openstack.check b/morphlib/exts/openstack.check index edc37cc1..4c21b604 100755 --- a/morphlib/exts/openstack.check +++ b/morphlib/exts/openstack.check @@ -1,5 +1,5 @@ #!/usr/bin/python -# Copyright (C) 2014 Codethink Limited +# Copyright (C) 2014-2015 Codethink Limited # # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by @@ -11,8 +11,7 @@ # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License along -# with this program; if not, write to the Free Software Foundation, Inc., -# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. +# with this program. If not, see <http://www.gnu.org/licenses/>. '''Preparatory checks for Morph 'openstack' write extension''' @@ -77,9 +76,16 @@ class OpenStackCheckExtension(morphlib.writeexts.WriteExtension): '--os-password', password, '--os-auth-url', auth_url, 'image-list'] - try: - cliapp.runcmd(cmdline) - except cliapp.AppException: - raise cliapp.AppException('Wrong OpenStack credentals.') + + exit, out, err = cliapp.runcmd_unchecked(cmdline) + + if exit != 0: + if err.startswith('The request you have made requires ' + 'authentication. (HTTP 401)'): + raise cliapp.AppException('Invalid OpenStack credentials.') + else: + raise cliapp.AppException( + 'Failed to connect to OpenStack instance at %s: %s' % + (auth_url, err)) OpenStackCheckExtension().run() diff --git a/morphlib/exts/openstack.write b/morphlib/exts/openstack.write index 516fe367..67e07c18 100755 --- a/morphlib/exts/openstack.write +++ b/morphlib/exts/openstack.write @@ -1,5 +1,5 @@ #!/usr/bin/python -# Copyright (C) 2013 - 2014 Codethink Limited +# Copyright (C) 2013-2015 Codethink Limited # # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by @@ -11,8 +11,7 @@ # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License along -# with this program; if not, write to the Free Software Foundation, Inc., -# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. +# with this program. If not, see <http://www.gnu.org/licenses/>. '''A Morph deployment write extension for deploying to OpenStack.''' @@ -28,40 +27,12 @@ import morphlib.writeexts class OpenStackWriteExtension(morphlib.writeexts.WriteExtension): - '''Configure a raw disk image into an OpenStack host. - - The raw disk image is created during Morph's deployment and the - image is deployed in OpenStack using python-glanceclient. - - The location command line argument is the authentification url - of the OpenStack server using the following syntax: - - http://HOST:PORT/VERSION - - where - - * HOST is the host running OpenStack - * PORT is the port which is using OpenStack for authentifications. - * VERSION is the authentification version of OpenStack (Only v2.0 - supported) - - This extension needs in the environment the following variables: - - * OPENSTACK_USER is the username to use in the deployment. - * OPENSTACK_TENANT is the project name to use in the deployment. - * OPENSTACK_IMAGENAME is the name of the image to create. - * OPENSTACK_PASSWORD is the password of the user. - - - The extension will connect to OpenStack using python-glanceclient - to configure a raw image. - - ''' + '''See openstack.write.help for documentation''' def process_args(self, args): if len(args) != 2: raise cliapp.AppException('Wrong number of command line args') - + temp_root, location = args os_params = self.get_openstack_parameters() @@ -69,7 +40,7 @@ class OpenStackWriteExtension(morphlib.writeexts.WriteExtension): fd, raw_disk = tempfile.mkstemp() os.close(fd) self.create_local_system(temp_root, raw_disk) - self.status(msg='Temporary disk image has been created at %s' + self.status(msg='Temporary disk image has been created at %s' % raw_disk) self.set_extlinux_root_to_virtio(raw_disk) @@ -79,8 +50,7 @@ class OpenStackWriteExtension(morphlib.writeexts.WriteExtension): def set_extlinux_root_to_virtio(self, raw_disk): '''Re-configures extlinux to use virtio disks''' self.status(msg='Updating extlinux.conf') - mp = self.mount(raw_disk) - try: + with self.mount(raw_disk) as mp: path = os.path.join(mp, 'extlinux.conf') with open(path) as f: @@ -91,9 +61,6 @@ class OpenStackWriteExtension(morphlib.writeexts.WriteExtension): with open(path, "w") as f: f.write(extlinux_conf) - finally: - self.unmount(mp) - def get_openstack_parameters(self): '''Get the environment variables needed. @@ -124,4 +91,3 @@ class OpenStackWriteExtension(morphlib.writeexts.WriteExtension): self.status(msg='Image configured.') OpenStackWriteExtension().run() - diff --git a/morphlib/exts/openstack.write.help b/morphlib/exts/openstack.write.help new file mode 100644 index 00000000..26983060 --- /dev/null +++ b/morphlib/exts/openstack.write.help @@ -0,0 +1,51 @@ +# Copyright (C) 2014, 2015 Codethink Limited +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation; version 2 of the License. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License along +# with this program; if not, see <http://www.gnu.org/licenses/>. + +help: | + + Deploy a Baserock system as a *new* OpenStack virtual machine. + (Use the `ssh-rsync` write extension to deploy upgrades to an *existing* + VM) + + Deploys the system to the OpenStack host using python-glanceclient. + + Parameters: + + * location: the authentication url of the OpenStack server using the + following syntax: + + http://HOST:PORT/VERSION + + where + + * HOST is the host running OpenStack + * PORT is the port which is using OpenStack for authentications. + * VERSION is the authentication version of OpenStack (Only v2.0 + supported) + + * OPENSTACK_USER=username: the username to use in the `--os-username` + argument to `glance`. + + * OPENSTACK_TENANT=tenant: the project name to use in the + `--os-tenant-name` argument to `glance`. + + * OPENSTACK_IMAGENAME=imagename: the name of the image to use in the + `--name` argument to `glance`. + + * OPENSTACK_PASSWORD=password: the password of the OpenStack user. (We + recommend passing this on the command-line, rather than setting an + environment variable or storing it in a cluster cluster definition file.) + + (See `morph help deploy` for details of how to pass parameters to write + extensions) diff --git a/morphlib/exts/rawdisk.check b/morphlib/exts/rawdisk.check index acdc4de1..9be0ce91 100755 --- a/morphlib/exts/rawdisk.check +++ b/morphlib/exts/rawdisk.check @@ -1,5 +1,5 @@ #!/usr/bin/python -# Copyright (C) 2014 Codethink Limited +# Copyright (C) 2014-2015 Codethink Limited # # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by @@ -11,8 +11,7 @@ # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License along -# with this program; if not, write to the Free Software Foundation, Inc., -# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. +# with this program. If not, see <http://www.gnu.org/licenses/>. '''Preparatory checks for Morph 'rawdisk' write extension''' @@ -33,10 +32,11 @@ class RawdiskCheckExtension(morphlib.writeexts.WriteExtension): location = args[0] upgrade = self.get_environment_boolean('UPGRADE') if upgrade: - if not os.path.isfile(location): - raise cliapp.AppException( - 'Cannot upgrade %s: it is not an existing disk image' % - location) + if not self.is_device(location): + if not os.path.isfile(location): + raise cliapp.AppException( + 'Cannot upgrade %s: it is not an existing disk image' % + location) version_label = os.environ.get('VERSION_LABEL') if version_label is None: @@ -44,9 +44,10 @@ class RawdiskCheckExtension(morphlib.writeexts.WriteExtension): 'VERSION_LABEL was not given. It is required when ' 'upgrading an existing system.') else: - if os.path.exists(location): - raise cliapp.AppException( - 'Target %s already exists. Use `morph upgrade` if you ' - 'want to update an existing image.' % location) + if not self.is_device(location): + if os.path.exists(location): + raise cliapp.AppException( + 'Target %s already exists. Use `morph upgrade` if you ' + 'want to update an existing image.' % location) RawdiskCheckExtension().run() diff --git a/morphlib/exts/rawdisk.write b/morphlib/exts/rawdisk.write index 12db4398..6f2d45ba 100755 --- a/morphlib/exts/rawdisk.write +++ b/morphlib/exts/rawdisk.write @@ -1,5 +1,5 @@ #!/usr/bin/python -# Copyright (C) 2012-2014 Codethink Limited +# Copyright (C) 2012-2015 Codethink Limited # # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by @@ -11,8 +11,7 @@ # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License along -# with this program; if not, write to the Free Software Foundation, Inc., -# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. +# with this program. If not, see <http://www.gnu.org/licenses/>. '''A Morph deployment write extension for raw disk images.''' @@ -29,81 +28,77 @@ import morphlib.writeexts class RawDiskWriteExtension(morphlib.writeexts.WriteExtension): - '''Create a raw disk image during Morph's deployment. - - If the image already exists, it is upgraded. - - The location command line argument is the pathname of the disk image - to be created/upgraded. - - ''' + '''See rawdisk.write.help for documentation''' def process_args(self, args): if len(args) != 2: raise cliapp.AppException('Wrong number of command line args') - + temp_root, location = args - if os.path.isfile(location): + upgrade = self.get_environment_boolean('UPGRADE') + + if upgrade: self.upgrade_local_system(location, temp_root) else: try: - self.create_local_system(temp_root, location) - self.status(msg='Disk image has been created at %s' % location) + if not self.is_device(location): + with self.created_disk_image(location): + self.format_btrfs(location) + self.create_system(temp_root, location) + self.status(msg='Disk image has been created at %s' % + location) + else: + self.format_btrfs(location) + self.create_system(temp_root, location) + self.status(msg='System deployed to %s' % location) except Exception: - self.status(msg='Failure to create disk image at %s' % + self.status(msg='Failure to deploy system to %s' % location) - if os.path.exists(location): - os.remove(location) raise def upgrade_local_system(self, raw_disk, temp_root): self.complete_fstab_for_btrfs_layout(temp_root) - mp = self.mount(raw_disk) + with self.mount(raw_disk) as mp: + version_label = self.get_version_label(mp) + self.status(msg='Updating image to a new version with label %s' % + version_label) - version_label = self.get_version_label(mp) - self.status(msg='Updating image to a new version with label %s' % - version_label) + version_root = os.path.join(mp, 'systems', version_label) + os.mkdir(version_root) - version_root = os.path.join(mp, 'systems', version_label) - os.mkdir(version_root) + old_orig = os.path.join(mp, 'systems', 'default', 'orig') + new_orig = os.path.join(version_root, 'orig') + cliapp.runcmd( + ['btrfs', 'subvolume', 'snapshot', old_orig, new_orig]) - old_orig = os.path.join(mp, 'systems', 'default', 'orig') - new_orig = os.path.join(version_root, 'orig') - cliapp.runcmd( - ['btrfs', 'subvolume', 'snapshot', old_orig, new_orig]) + cliapp.runcmd( + ['rsync', '-a', '--checksum', '--numeric-ids', '--delete', + temp_root + os.path.sep, new_orig]) - cliapp.runcmd( - ['rsync', '-a', '--checksum', '--numeric-ids', '--delete', - temp_root + os.path.sep, new_orig]) + self.create_run(version_root) - self.create_run(version_root) + default_path = os.path.join(mp, 'systems', 'default') + if os.path.exists(default_path): + os.remove(default_path) + else: + # we are upgrading and old system that does + # not have an updated extlinux config file + if self.bootloader_config_is_wanted(): + self.generate_bootloader_config(mp) + self.install_bootloader(mp) + os.symlink(version_label, default_path) - default_path = os.path.join(mp, 'systems', 'default') - if os.path.exists(default_path): - os.remove(default_path) - else: - # we are upgrading and old system that does - # not have an updated extlinux config file if self.bootloader_config_is_wanted(): - self.generate_bootloader_config(mp) - self.install_bootloader(mp) - os.symlink(version_label, default_path) - - if self.bootloader_config_is_wanted(): - self.install_kernel(version_root, temp_root) - - self.unmount(mp) + self.install_kernel(version_root, temp_root) def get_version_label(self, mp): version_label = os.environ.get('VERSION_LABEL') if version_label is None: - self.unmount(mp) raise cliapp.AppException('VERSION_LABEL was not given') if os.path.exists(os.path.join(mp, 'systems', version_label)): - self.unmount(mp) raise cliapp.AppException('VERSION_LABEL %s already exists' % version_label) @@ -111,4 +106,3 @@ class RawDiskWriteExtension(morphlib.writeexts.WriteExtension): RawDiskWriteExtension().run() - diff --git a/morphlib/exts/rawdisk.write.help b/morphlib/exts/rawdisk.write.help index 298d441c..52ed73fb 100644 --- a/morphlib/exts/rawdisk.write.help +++ b/morphlib/exts/rawdisk.write.help @@ -1,11 +1,82 @@ +# Copyright (C) 2014, 2015 Codethink Limited +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation; version 2 of the License. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License along +# with this program; if not, see <http://www.gnu.org/licenses/>. + help: | - Create a raw disk image during Morph's deployment. - - If the image already exists, it is upgraded. - The `location` argument is a pathname to the image to be - created or upgraded. + Write a system produced by Morph to a physical disk, or to a file that can + be used as a virtual disk. The target will be formatted as a single Btrfs + partition, with the system image written to a subvolume in /systems, and + other subvolumes created for /home, /opt, /root, /srv and /var. + + When written to a physical drive, the drive can be used as the boot device + for a 'real' machine. + + When written to a file, the file can be used independently of `morph` to + create virtual machines with KVM / libvirt, OpenStack or, after converting + it to VDI format, VirtualBox. + + `morph deploy` will fail if the file specified by `location` already + exists. + + If used in `morph upgrade`, the rootfs produced by 'morph build' is added + to the existing raw disk image or device as an additional btrfs sub-volume. + `morph upgrade` will fail if the file specified by `location` does not + exist, or is not a Baserock raw disk image. (Most users are unlikely to + need or use this functionality: it is useful mainly for developers working + on the Baserock tools.) + + Parameters: + + * location: the pathname of the disk image to be created/upgraded, or the + path to the physical device. + + * VERSION_LABEL=label - should contain only alpha-numeric + characters and the '-' (hyphen) character. Mandatory if being used with + `morph update` + + * INITRAMFS_PATH=path: the location of an initramfs for the bootloader to + tell Linux to use, rather than booting the rootfs directly. + + * DTB_PATH=path: **(MANDATORY)** for systems that require a device tree + binary - Give the full path (without a leading /) to the location of the + DTB in the built system image . The deployment will fail if `path` does + not exist. + + * BOOTLOADER_INSTALL=value: the bootloader to be installed + **(MANDATORY)** for non-x86 systems + + allowed values = + - 'extlinux' (default) - the extlinux bootloader will + be installed + - 'none' - no bootloader will be installed by `morph deploy`. A + bootloader must be installed manually. This value must be used when + deploying non-x86 systems such as ARM. + + * BOOTLOADER_CONFIG_FORMAT=value: the bootloader format to be used. + If not specified for x86-32 and x86-64 systems, 'extlinux' will be used + + allowed values = + - 'extlinux' + + * KERNEL_ARGS=args: optional additional kernel command-line parameters to + be appended to the default set. The default set is: + + 'rw init=/sbin/init rootfstype=btrfs \ + rootflags=subvol=systems/default/run \ + root=[name or UUID of root filesystem]' + + (See https://www.kernel.org/doc/Documentation/kernel-parameters.txt) - The INITRAMFS_PATH option can be used to specify the location of an - initramfs for syslinux to tell Linux to use, rather than booting - the rootfs directly. + (See `morph help deploy` for details of how to pass parameters to write + extensions) diff --git a/morphlib/exts/set-hostname.configure b/morphlib/exts/set-hostname.configure index e44c5d56..4b2424d8 100755 --- a/morphlib/exts/set-hostname.configure +++ b/morphlib/exts/set-hostname.configure @@ -1,5 +1,5 @@ #!/bin/sh -# Copyright (C) 2013 Codethink Limited +# Copyright (C) 2013,2015 Codethink Limited # # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by @@ -11,8 +11,7 @@ # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License along -# with this program; if not, write to the Free Software Foundation, Inc., -# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. +# with this program. If not, see <http://www.gnu.org/licenses/>. # Set hostname on system from HOSTNAME. diff --git a/morphlib/exts/simple-network.configure b/morphlib/exts/simple-network.configure index b98b202c..61113325 100755 --- a/morphlib/exts/simple-network.configure +++ b/morphlib/exts/simple-network.configure @@ -1,5 +1,5 @@ #!/usr/bin/python -# Copyright (C) 2013 Codethink Limited +# Copyright (C) 2013,2015 Codethink Limited # # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by @@ -11,16 +11,17 @@ # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License along -# with this program; if not, write to the Free Software Foundation, Inc., -# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. +# with this program. If not, see <http://www.gnu.org/licenses/>. -'''A Morph deployment configuration extension to handle /etc/network/interfaces +'''A Morph deployment configuration extension to handle network configutation -This extension prepares /etc/network/interfaces with the interfaces specified -during deployment. +This extension prepares /etc/network/interfaces and networkd .network files +in /etc/systemd/network/ with the interfaces specified during deployment. If no network configuration is provided, eth0 will be configured for DHCP -with the hostname of the system. +with the hostname of the system in the case of /etc/network/interfaces. +In the case of networkd, any interface starting by e* will be configured +for DHCP ''' @@ -37,20 +38,74 @@ class SimpleNetworkError(morphlib.Error): class SimpleNetworkConfigurationExtension(cliapp.Application): - '''Configure /etc/network/interfaces + '''Configure /etc/network/interfaces and generate networkd .network files - Reading NETWORK_CONFIG, this extension sets up /etc/network/interfaces. + Reading NETWORK_CONFIG, this extension sets up /etc/network/interfaces + and .network files in /etc/systemd/network/. ''' def process_args(self, args): - network_config = os.environ.get( - "NETWORK_CONFIG", "lo:loopback;eth0:dhcp,hostname=$(hostname)") + network_config = os.environ.get("NETWORK_CONFIG") - self.status(msg="Processing NETWORK_CONFIG=%(nc)s", nc=network_config) + self.rename_networkd_chunk_file(args) - stanzas = self.parse_network_stanzas(network_config) - iface_file = self.generate_iface_file(stanzas) + if network_config is None: + self.generate_default_network_config(args) + else: + self.status(msg="Processing NETWORK_CONFIG=%(nc)s", + nc=network_config) + + stanzas = self.parse_network_stanzas(network_config) + + self.generate_interfaces_file(args, stanzas) + self.generate_networkd_files(args, stanzas) + + def rename_networkd_chunk_file(self, args): + """Rename the 10-dchp.network file generated in the systemd chunk + The systemd chunk will place something in 10-dhcp.network, which will + have higher precedence than anything added in this extension (we + start at 50-*). + + We should check for that file and rename it instead remove it in + case the file is being used by the user. + + Until both the following happen, we should continue to rename that + default config file: + + 1. simple-network.configure is always run when systemd is included + 2. We've been building systems without systemd including that default + networkd config for long enough that nobody should be including + that config file. + """ + file_path = os.path.join(args[0], "etc", "systemd", "network", + "10-dhcp.network") + try: + os.rename(file_path, file_path + ".morph") + self.status(msg="Renaming networkd file from systemd chunk: %(f)s \ + to %(f)s.morph", f=file_path) + except OSError: + pass + + def generate_default_network_config(self, args): + """Generate default network config: DHCP in all the interfaces""" + + default_network_config_interfaces = "lo:loopback;" \ + "eth0:dhcp,hostname=$(hostname)" + default_network_config_networkd = "e*:dhcp" + + stanzas_interfaces = self.parse_network_stanzas( + default_network_config_interfaces) + stanzas_networkd = self.parse_network_stanzas( + default_network_config_networkd) + + self.generate_interfaces_file(args, stanzas_interfaces) + self.generate_networkd_files(args, stanzas_networkd) + + def generate_interfaces_file(self, args, stanzas): + """Generate /etc/network/interfaces file""" + + iface_file = self.generate_iface_file(stanzas) with open(os.path.join(args[0], "etc/network/interfaces"), "w") as f: f.write(iface_file) @@ -83,6 +138,74 @@ class SimpleNetworkConfigurationExtension(cliapp.Application): lines += [""] return "\n".join(lines) + def generate_networkd_files(self, args, stanzas): + """Generate .network files""" + + for i, stanza in enumerate(stanzas, 50): + iface_file = self.generate_networkd_file(stanza) + + if iface_file is None: + continue + + path = os.path.join(args[0], "etc", "systemd", "network", + "%s-%s.network" % (i, stanza['name'])) + + with open(path, "w") as f: + f.write(iface_file) + + def generate_networkd_file(self, stanza): + """Generate an .network file from the provided data.""" + + name = stanza['name'] + itype = stanza['type'] + pairs = stanza['args'].items() + + if itype == "loopback": + return + + lines = ["[Match]"] + lines += ["Name=%s\n" % name] + lines += ["[Network]"] + if itype == "dhcp": + lines += ["DHCP=yes"] + else: + lines += self.generate_networkd_entries(pairs) + + return "\n".join(lines) + + def generate_networkd_entries(self, pairs): + """Generate networkd configuration entries with the other parameters""" + + address = None + netmask = None + gateway = None + lines = [] + for pair in pairs: + if pair[0] == 'address': + address = pair[1] + elif pair[0] == 'netmask': + netmask = pair[1] + elif pair[0] == 'gateway': + gateway = pair[1] + + if address and netmask: + network_suffix = self.convert_net_mask_to_cidr_suffix (netmask); + address_line = address + '/' + str(network_suffix) + lines += ["Address=%s" % address_line] + elif address or netmask: + raise Exception('address and netmask must be specified together') + + if gateway is not None: + lines += ["Gateway=%s" % gateway] + + return lines + + def convert_net_mask_to_cidr_suffix(self, mask): + """Convert dotted decimal form of a subnet mask to CIDR suffix notation + + For example: 255.255.255.0 -> 24 + """ + return sum(bin(int(x)).count('1') for x in mask.split('.')) def parse_network_stanzas(self, config): """Parse a network config environment variable into stanzas. diff --git a/morphlib/exts/ssh-rsync.check b/morphlib/exts/ssh-rsync.check index 11446c28..c3bdfd29 100755 --- a/morphlib/exts/ssh-rsync.check +++ b/morphlib/exts/ssh-rsync.check @@ -1,5 +1,5 @@ #!/usr/bin/python -# Copyright (C) 2014 Codethink Limited +# Copyright (C) 2014-2015 Codethink Limited # # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by @@ -11,8 +11,7 @@ # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License along -# with this program; if not, write to the Free Software Foundation, Inc., -# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. +# with this program. If not, see <http://www.gnu.org/licenses/>. '''Preparatory checks for Morph 'ssh-rsync' write extension''' diff --git a/morphlib/exts/ssh-rsync.write b/morphlib/exts/ssh-rsync.write index 2d7258ba..6d596500 100755 --- a/morphlib/exts/ssh-rsync.write +++ b/morphlib/exts/ssh-rsync.write @@ -1,5 +1,5 @@ #!/usr/bin/python -# Copyright (C) 2013-2014 Codethink Limited +# Copyright (C) 2013-2015 Codethink Limited # # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by @@ -11,8 +11,7 @@ # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License along -# with this program; if not, write to the Free Software Foundation, Inc., -# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. +# with this program. If not, see <http://www.gnu.org/licenses/>. '''A Morph deployment write extension for upgrading systems over ssh.''' @@ -37,14 +36,8 @@ def ssh_runcmd_ignore_failure(location, command, **kwargs): class SshRsyncWriteExtension(morphlib.writeexts.WriteExtension): - '''Upgrade a running baserock system with ssh and rsync. - - It assumes the system is baserock-based and has a btrfs partition. - - The location command line argument is the 'user@hostname' string - that will be passed to ssh and rsync - - ''' + '''See ssh-rsync.write.help for documentation''' + def find_root_disk(self, location): '''Read /proc/mounts on location to find which device contains "/"''' diff --git a/morphlib/exts/ssh-rsync.write.help b/morphlib/exts/ssh-rsync.write.help new file mode 100644 index 00000000..f3f79ed5 --- /dev/null +++ b/morphlib/exts/ssh-rsync.write.help @@ -0,0 +1,50 @@ +# Copyright (C) 2014, 2015 Codethink Limited +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation; version 2 of the License. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License along +# with this program; if not, see <http://www.gnu.org/licenses/>. + +help: | + + Upgrade a Baserock system which is already deployed: + - as a KVM/LibVirt, OpenStack or vbox-ssh virtual machine; + - on a Jetson board. + + Copies a binary delta over to the target system and arranges for it + to be bootable. + + The recommended way to use this extension is by calling `morph upgrade`. + Using `morph deploy --upgrade` is deprecated. + + The upgrade will fail if: + - no VM is deployed and running at `location`; + - the target system is not a Baserock system; + - the target's filesystem and its layout are not compatible with that + created by `morph deploy`." + + See also the 'Upgrading a Baserock installation' section of the 'Using + Baserock` page at wiki.baserock.org + http://wiki.baserock.org/devel-with/#index8h2 + + Parameters: + + * location: the 'user@hostname' string that will be used by ssh and rsync. + 'user' will always be `root` and `hostname` the hostname or address of + the system being upgraded. + + * VERSION_LABEL=label - **(MANDATORY)** should contain only alpha-numeric + characters and the '-' (hyphen) character. + + * AUTOSTART=<VALUE>` - boolean. If it is set, the VM will be started when + it has been deployed. + + (See `morph help deploy` for details of how to pass parameters to write + extensions) diff --git a/morphlib/exts/sysroot.check b/morphlib/exts/sysroot.check new file mode 100755 index 00000000..8ed965bd --- /dev/null +++ b/morphlib/exts/sysroot.check @@ -0,0 +1,29 @@ +#!/bin/sh +# Copyright (C) 2015 Codethink Limited +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation; version 2 of the License. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License along +# with this program. If not, see <http://www.gnu.org/licenses/>. + +# Preparatory checks for Morph 'sysroot' write extension + +set -eu + +location="$1" +if [ -d "$location" ]; then + echo >&2 "ERROR: Deployment directory already exists: $location" + exit 1 +fi + +if [ "$UPGRADE" == "yes" ]; then + echo >&2 "ERROR: Cannot upgrade a sysroot deployment" + exit 1 +fi diff --git a/morphlib/exts/sysroot.write b/morphlib/exts/sysroot.write index 1ae4864f..0ad8d630 100755 --- a/morphlib/exts/sysroot.write +++ b/morphlib/exts/sysroot.write @@ -1,5 +1,5 @@ #!/bin/sh -# Copyright (C) 2014 Codethink Limited +# Copyright (C) 2014,2015 Codethink Limited # # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by @@ -11,16 +11,13 @@ # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License along -# with this program; if not, write to the Free Software Foundation, Inc., -# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. +# with this program. If not, see <http://www.gnu.org/licenses/>. # A Morph write extension to deploy to another directory set -eu -# Ensure the target is an empty directory mkdir -p "$2" -find "$2" -mindepth 1 -delete # Move the contents of our source directory to our target # Previously we would (cd "$1" && find -print0 | cpio -0pumd "$absolute_path") diff --git a/morphlib/exts/tar.check b/morphlib/exts/tar.check index cbeaf163..f2304d46 100755 --- a/morphlib/exts/tar.check +++ b/morphlib/exts/tar.check @@ -1,5 +1,5 @@ #!/bin/sh -# Copyright (C) 2014 Codethink Limited +# Copyright (C) 2014-2015 Codethink Limited # # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by @@ -11,8 +11,7 @@ # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License along -# with this program; if not, write to the Free Software Foundation, Inc., -# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. +# with this program. If not, see <http://www.gnu.org/licenses/>. # Preparatory checks for Morph 'tar' write extension diff --git a/morphlib/exts/tar.write b/morphlib/exts/tar.write index 333626b5..01b545b4 100755 --- a/morphlib/exts/tar.write +++ b/morphlib/exts/tar.write @@ -1,5 +1,5 @@ #!/bin/sh -# Copyright (C) 2013 Codethink Limited +# Copyright (C) 2013,2015 Codethink Limited # # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by @@ -11,8 +11,7 @@ # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License along -# with this program; if not, write to the Free Software Foundation, Inc., -# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. +# with this program. If not, see <http://www.gnu.org/licenses/>. # A Morph write extension to deploy to a .tar file diff --git a/morphlib/exts/tar.write.help b/morphlib/exts/tar.write.help index f052ac03..b45c61fa 100644 --- a/morphlib/exts/tar.write.help +++ b/morphlib/exts/tar.write.help @@ -1,5 +1,19 @@ +# Copyright (C) 2014, 2015 Codethink Limited +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation; version 2 of the License. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License along +# with this program; if not, see <http://www.gnu.org/licenses/>. + help: | Create a .tar file of the deployed system. - + The `location` argument is a pathname to the .tar file to be created. diff --git a/morphlib/exts/vdaboot.configure b/morphlib/exts/vdaboot.configure index b88eb3a8..60de925b 100755 --- a/morphlib/exts/vdaboot.configure +++ b/morphlib/exts/vdaboot.configure @@ -1,5 +1,5 @@ #!/bin/sh -# Copyright (C) 2013 Codethink Limited +# Copyright (C) 2013,2015 Codethink Limited # # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by @@ -11,8 +11,7 @@ # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License along -# with this program; if not, write to the Free Software Foundation, Inc., -# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. +# with this program. If not, see <http://www.gnu.org/licenses/>. # Change the "/" mount point to /dev/vda to use virtio disks. diff --git a/morphlib/exts/virtualbox-ssh.check b/morphlib/exts/virtualbox-ssh.check index 57d54db1..a97f3294 100755 --- a/morphlib/exts/virtualbox-ssh.check +++ b/morphlib/exts/virtualbox-ssh.check @@ -1,5 +1,5 @@ #!/usr/bin/python -# Copyright (C) 2014 Codethink Limited +# Copyright (C) 2014-2015 Codethink Limited # # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by @@ -11,8 +11,7 @@ # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License along -# with this program; if not, write to the Free Software Foundation, Inc., -# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. +# with this program. If not, see <http://www.gnu.org/licenses/>. '''Preparatory checks for Morph 'virtualbox-ssh' write extension''' diff --git a/morphlib/exts/virtualbox-ssh.write b/morphlib/exts/virtualbox-ssh.write index 39ea8f86..774f2b4f 100755 --- a/morphlib/exts/virtualbox-ssh.write +++ b/morphlib/exts/virtualbox-ssh.write @@ -1,5 +1,5 @@ #!/usr/bin/python -# Copyright (C) 2012-2014 Codethink Limited +# Copyright (C) 2012-2015 Codethink Limited # # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by @@ -11,8 +11,7 @@ # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License along -# with this program; if not, write to the Free Software Foundation, Inc., -# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. +# with this program. If not, see <http://www.gnu.org/licenses/>. '''A Morph deployment write extension for deploying to VirtualBox via ssh. @@ -20,6 +19,8 @@ VirtualBox is assumed to be running on a remote machine, which is accessed over ssh. The machine gets created, but not started. +See file virtualbox-ssh.write.help for documentation + ''' @@ -36,30 +37,10 @@ import morphlib.writeexts class VirtualBoxPlusSshWriteExtension(morphlib.writeexts.WriteExtension): - '''Create a VirtualBox virtual machine during Morph's deployment. - - The location command line argument is the pathname of the disk image - to be created. The user is expected to provide the location argument - using the following syntax: - - vbox+ssh://HOST/GUEST/PATH - - where: - - * HOST is the host on which VirtualBox is running - * GUEST is the name of the guest virtual machine on that host - * PATH is the path to the disk image that should be created, - on that host - - The extension will connect to HOST via ssh to run VirtualBox's - command line management tools. - - ''' - def process_args(self, args): if len(args) != 2: raise cliapp.AppException('Wrong number of command line args') - + temp_root, location = args ssh_host, vm_name, vdi_path = self.parse_location(location) autostart = self.get_environment_boolean('AUTOSTART') @@ -88,7 +69,7 @@ class VirtualBoxPlusSshWriteExtension(morphlib.writeexts.WriteExtension): def parse_location(self, location): '''Parse the location argument to get relevant data.''' - + x = urlparse.urlparse(location) if x.scheme != 'vbox+ssh': raise cliapp.AppException( @@ -169,11 +150,11 @@ class VirtualBoxPlusSshWriteExtension(morphlib.writeexts.WriteExtension): attach_disks = self.parse_attach_disks() for device_no, disk in enumerate(attach_disks, 1): - cmd = ['storageattach', vm_name, + cmd = ['storageattach', vm_name, '--storagectl', 'SATA Controller', '--port', str(device_no), '--device', '0', - '--type', 'hdd', + '--type', 'hdd', '--medium', disk] commands.append(cmd) @@ -187,20 +168,6 @@ class VirtualBoxPlusSshWriteExtension(morphlib.writeexts.WriteExtension): def get_host_interface(self, ssh_host): host_ipaddr = os.environ.get('HOST_IPADDR') netmask = os.environ.get('NETMASK') - network_config = os.environ.get("NETWORK_CONFIG") - - if network_config is None: - raise cliapp.AppException('NETWORK_CONFIG was not given') - - if "eth0:" not in network_config: - raise cliapp.AppException( - 'NETWORK_CONFIG does not contain ' - 'the eth0 configuration') - - if "eth1:" not in network_config: - raise cliapp.AppException( - 'NETWORK_CONFIG does not contain ' - 'the eth1 configuration') if host_ipaddr is None: raise cliapp.AppException('HOST_IPADDR was not given') @@ -242,4 +209,3 @@ class VirtualBoxPlusSshWriteExtension(morphlib.writeexts.WriteExtension): return iface VirtualBoxPlusSshWriteExtension().run() - diff --git a/morphlib/exts/virtualbox-ssh.write.help b/morphlib/exts/virtualbox-ssh.write.help index 8b5053a5..2dbf988c 100644 --- a/morphlib/exts/virtualbox-ssh.write.help +++ b/morphlib/exts/virtualbox-ssh.write.help @@ -1,4 +1,135 @@ +# Copyright (C) 2014, 2015 Codethink Limited +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation; version 2 of the License. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License along +# with this program; if not, see <http://www.gnu.org/licenses/>. + help: | - The INITRAMFS_PATH option can be used to specify the location of an - initramfs for syslinux to tell Linux to use, rather than booting - the rootfs directly. + + Deploy a Baserock system as a *new* VirtualBox virtual machine. + (Use the `ssh-rsync` write extension to deploy upgrades to an *existing* + VM) + + Connects to HOST via ssh to run VirtualBox's command line management tools. + + Parameters: + + * location: a custom URL scheme of the form `vbox+ssh://HOST/GUEST/PATH`, + where: + * HOST is the name of the host on which VirtualBox is running + * GUEST is the name of the guest VM on that host + * PATH is the path to the disk image that should be created, + on that host. For example, + `vbox+ssh://alice@192.168.122.1/testsys/home/alice/testys.img` where + * `alice@192.168.122.1` is the target host as given to ssh, + **from within the development host** (which may be + different from the target host's normal address); + * `testsys` is the name of the new guest VM'; + * `/home/alice/testys.img` is the pathname of the disk image files + on the target host. + + * HOSTNAME=name: the hostname of the **guest** VM within the network into + which it is being deployed. + + * DISK_SIZE=X: **(MANDATORY)** the size of the VM's primary virtual hard + disk. `X` should use a suffix of `K`, `M`, or `G` (in upper or lower + case) to indicate kilo-, mega-, or gigabytes. For example, + `DISK_SIZE=100G` would create a 100 gigabyte virtual hard disk. + + * RAM_SIZE=X: The amount of RAM that the virtual machine should allocate + for itself from the host. `X` is interpreted in the same as for + DISK_SIZE, and defaults to `1G`. + + * VCPUS=n: the number of virtual CPUs for the VM. Allowed values 1-32. Do + not use more CPU cores than you have available physically (real cores, + no hyperthreads). + + * INITRAMFS_PATH=path: the location of an initramfs for the bootloader to + tell Linux to use, rather than booting the rootfs directly. + + * DTB_PATH=path: **(MANDATORY)** for systems that require a device tree + binary - Give the full path (without a leading /) to the location of the + DTB in the built system image . The deployment will fail if `path` does + not exist. + + * BOOTLOADER_INSTALL=value: the bootloader to be installed + **(MANDATORY)** for non-x86 systems + + allowed values = + - 'extlinux' (default) - the extlinux bootloader will + be installed + - 'none' - no bootloader will be installed by `morph deploy`. A + bootloader must be installed manually. This value must be used when + deploying non-x86 systems such as ARM. + + * BOOTLOADER_CONFIG_FORMAT=value: the bootloader format to be used. + If not specified for x86-32 and x86-64 systems, 'extlinux' will be used + + allowed values = + - 'extlinux' + + * KERNEL_ARGS=args: optional additional kernel command-line parameters to + be appended to the default set. The default set is: + + 'rw init=/sbin/init rootfstype=btrfs \ + rootflags=subvol=systems/default/run \ + root=[name or UUID of root filesystem]' + + (See https://www.kernel.org/doc/Documentation/kernel-parameters.txt) + + * AUTOSTART=<VALUE> - boolean. If it is set, the VM will be started when + it has been deployed. + + * VAGRANT=<VALUE> - boolean. If it is set, then networking is configured + so that the VM will work with Vagrant. Otherwise networking is + configured to run directly in VirtualBox. + + * HOST_IPADDR=<ip_address> - the IP address of the VM host. + + * NETMASK=<netmask> - the netmask of the VM host. + + * NETWORK_CONFIG=<net_config> - `net_config` is used to set up the VM's + network interfaces. It is a string containing semi-colon separated + 'stanzas' where each stanza provides information about a network + interface. Each stanza is of the form name:type[,arg=value] e.g. + + lo:loopback + eth0:dhcp + eth1:static,address=10.0.0.1,netmask=255.255.0.0 + + An example of the NETWORK_CONFIG parameter (It should be in one line) + + `"lo:loopback;eth0:static,address=192.168.100.2,netmask=255.255.255.0; + eth1:dhcp,hostname=$(hostname)"` + + It is useful to configure one interface to use NAT to give the VM access + to the outside world and another interface to use the Virtual Box host + adapter to allow you to access the Trove from the host machine. + + The NAT interface eth1 is set up to use dhcp, the host-only adapter + interface is configured statically. + + Note: you must give the host-only adapter interface an address that lies + **on the same network** as the host adapter. So if the host adapter has + an IP of 192.168.100.1 eth0 should have an address such as + 192.168.100.42. + + The settings of the host adapter, including its IP can be changed either + in the VirtualBox manager UI + (https://www.virtualbox.org/manual/ch03.html#settings-network) + or via the VBoxManage command line + (https://www.virtualbox.org/manual/ch08.html#idp57572192) + + See Chapter 6 of the VirtualBox User Manual for more information about + virtual networking (https://www.virtualbox.org/manual/ch06.html) + + (See `morph help deploy` for details of how to pass parameters to write + extensions) diff --git a/morphlib/fsutils.py b/morphlib/fsutils.py index 6d651171..a3b73bf6 100644 --- a/morphlib/fsutils.py +++ b/morphlib/fsutils.py @@ -1,4 +1,4 @@ -# Copyright (C) 2012-2014 Codethink Limited +# Copyright (C) 2012-2015 Codethink Limited # # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by @@ -10,8 +10,7 @@ # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License along -# with this program; if not, write to the Free Software Foundation, Inc., -# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. +# with this program. If not, see <http://www.gnu.org/licenses/>. import os import re @@ -97,48 +96,43 @@ def invert_paths(tree_walker, paths): ''' - def is_subpath(prefix, path): - prefix_components = prefix.split(os.sep) - path_components = path.split(os.sep) - return path_components[:len(prefix_components)] == prefix_components + def normpath(path): + if path == '.': + return path + path = os.path.normpath(path) + if not os.path.isabs(path): + path = os.path.join('.', path) + return path + def any_paths_are_subpath_of(prefix): + prefix = normpath(prefix) + norm_paths = (normpath(path) for path in paths) + return any(path[:len(prefix)] == prefix + for path in norm_paths) + + def path_is_listed(path): + return any(normpath(path) == normpath(other) + for other in paths) for dirpath, dirnames, filenames in tree_walker: - if any(p == dirpath for p in paths): # pragma: no cover - # Dir is an exact match for a path - # don't recurse any further - # Don't yield it, since we don't return listed paths - continue - dn_copy = list(dirnames) - for subdir in dn_copy: - subdirpath = os.path.join(dirpath, subdir) - - if any(p == subdirpath for p in paths): - # Subdir is an exact match for a path - # Don't recurse into it, so remove from list - # Don't yield it, since we don't return listed paths - dirnames.remove(subdir) - elif any(is_subpath(subdirpath, p) for p in paths): - # This directory is a parent directory of one - # of our paths - # Recurse into it, so don't remove it from the list - # Don't yield it, since we don't return listed paths - pass - else: - # This directory is neither one marked for writing, - # nor a parent of a file marked for writing - # Don't recurse, so remove it from the list - # Yield it, since we return listed paths - dirnames.remove(subdir) - yield subdirpath + if path_is_listed(dirpath): + # No subpaths need to be considered + del dirnames[:] + del filenames[:] + elif any_paths_are_subpath_of(dirpath): + # Subpaths may be marked, or may not, need to leave this + # writable, so don't yield, but we don't cull. + pass + else: + # not listed as a parent or an exact match, needs to be + # yielded, but we don't need to consider subdirs, so can cull + yield dirpath + del dirnames[:] + del filenames[:] for filename in filenames: fullpath = os.path.join(dirpath, filename) - if any(is_subpath(p, fullpath) for p in paths): - # The file path is a child of one of the paths - # or is equal. - # Don't yield because either it is one of the specified - # paths, or is a file in a directory specified by a path + if path_is_listed(fullpath): pass else: yield fullpath diff --git a/morphlib/fsutils_tests.py b/morphlib/fsutils_tests.py index 7b159665..f2772fb7 100644 --- a/morphlib/fsutils_tests.py +++ b/morphlib/fsutils_tests.py @@ -1,4 +1,4 @@ -# Copyright (C) 2013 Codethink Limited +# Copyright (C) 2013, 2014-2015 Codethink Limited # # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by @@ -10,8 +10,7 @@ # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License along -# with this program; if not, write to the Free Software Foundation, Inc., -# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. +# with this program. If not, see <http://www.gnu.org/licenses/>. import os @@ -51,11 +50,26 @@ class InvertPathsTests(unittest.TestCase): }, } - def test_flat_lists_single_files(self): + def test_flat_lists_top_dir(self): walker = dummy_top_down_walker('.', self.flat_tree) - self.assertEqual(sorted(["./foo", "./bar", "./baz"]), + self.assertEqual(["."], sorted(morphlib.fsutils.invert_paths(walker, []))) + def test_flat_skips_all_with_root_pased(self): + walker = dummy_top_down_walker('.', self.flat_tree) + self.assertEqual([], + list(morphlib.fsutils.invert_paths(walker, ['.']))) + + def test_flat_lists_top_dir(self): + walker = dummy_top_down_walker('.', self.nested_tree) + self.assertEqual(["."], + sorted(morphlib.fsutils.invert_paths(walker, []))) + + def test_flat_skips_all_with_root_pased(self): + walker = dummy_top_down_walker('.', self.nested_tree) + self.assertEqual([], + list(morphlib.fsutils.invert_paths(walker, ['.']))) + def test_flat_excludes_listed_files(self): walker = dummy_top_down_walker('.', self.flat_tree) self.assertTrue( @@ -95,5 +109,5 @@ class InvertPathsTests(unittest.TestCase): "./tmp/morph/staging/inst", "./tmp", ])) - expected = ("./bin",) - self.assertEqual(sorted(found), sorted(expected)) + expected = ["./bin"] + self.assertEqual(sorted(found), expected) diff --git a/morphlib/git.py b/morphlib/git.py index 6a5a9a47..acda6137 100644 --- a/morphlib/git.py +++ b/morphlib/git.py @@ -1,4 +1,4 @@ -# Copyright (C) 2011-2014 Codethink Limited +# Copyright (C) 2011-2015 Codethink Limited # # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by @@ -10,11 +10,9 @@ # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License along -# with this program; if not, write to the Free Software Foundation, Inc., -# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. +# with this program. If not, see <http://www.gnu.org/licenses/>. -import binascii import cliapp import ConfigParser import logging @@ -23,10 +21,6 @@ import re import string import StringIO import sys -import time - - -import cliapp import morphlib @@ -196,7 +190,7 @@ def get_user_name(runcmd): try: config = check_config_set(runcmd, keys={"user.name": "My Name"}) return config['user.name'] - except ConfigNotSetException, e: + except ConfigNotSetException as e: raise IdentityNotSetException(e.missing) @@ -207,7 +201,7 @@ def get_user_email(runcmd): try: cfg = check_config_set(runcmd, keys={"user.email": "me@example.com"}) return cfg['user.email'] - except ConfigNotSetException, e: + except ConfigNotSetException as e: raise IdentityNotSetException(e.missing) def check_config_set(runcmd, keys, cwd='.'): @@ -226,11 +220,6 @@ def check_config_set(runcmd, keys, cwd='.'): return found -def set_remote(runcmd, gitdir, name, url): - '''Set remote with name 'name' use a given url at gitdir''' - return gitcmd(runcmd, 'remote', 'set-url', name, url, cwd=gitdir) - - def copy_repository(runcmd, repo, destdir, is_mirror=True): '''Copies a cached repository into a directory using cp. @@ -275,25 +264,6 @@ def copy_repository(runcmd, repo, destdir, is_mirror=True): gitcmd(runcmd, 'remote', 'update', 'origin', '--prune', cwd=destdir) -def checkout_ref(runcmd, gitdir, ref): - '''Checks out a specific ref/SHA1 in a git working tree.''' - gitcmd(runcmd, 'checkout', ref, cwd=gitdir) - gd = morphlib.gitdir.GitDirectory(gitdir) - if gd.has_fat(): - gd.fat_init() - gd.fat_pull() - - -def index_has_changes(runcmd, gitdir): - '''Returns True if there are no staged changes to commit''' - try: - gitcmd(runcmd, 'diff-index', '--cached', '--quiet', - '--ignore-submodules', 'HEAD', cwd=gitdir) - except cliapp.AppException: - return True - return False - - def reset_workdir(runcmd, gitdir): '''Removes any differences between the current commit ''' '''and the status of the working directory''' @@ -321,10 +291,6 @@ def is_valid_sha1(ref): return len(ref) == 40 and all(x in string.hexdigits for x in ref) -def rev_parse(runcmd, gitdir, ref): - '''Find the sha1 for the given ref''' - return gitcmd(runcmd, 'rev-parse', '--verify', ref, cwd=gitdir)[0:40] - def gitcmd(runcmd, *args, **kwargs): '''Run git commands safely''' diff --git a/morphlib/gitdir.py b/morphlib/gitdir.py index 9fef4f1e..03640a22 100644 --- a/morphlib/gitdir.py +++ b/morphlib/gitdir.py @@ -1,4 +1,4 @@ -# Copyright (C) 2013-2014 Codethink Limited +# Copyright (C) 2013-2015 Codethink Limited # # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by @@ -10,8 +10,7 @@ # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License along -# with this program; if not, write to the Free Software Foundation, Inc., -# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. +# with this program. If not, see <http://www.gnu.org/licenses/>. # # =*= License: GPL-2 =*= @@ -24,6 +23,13 @@ import re import morphlib +class NoGitRepoError(cliapp.AppException): + + def __init__(self, dirname): + cliapp.AppException.__init__( + self, 'Directory %s is not a Git repository. ' % dirname) + + class NoWorkingTreeError(cliapp.AppException): def __init__(self, repo): @@ -343,23 +349,40 @@ class Remote(object): class GitDirectory(object): - '''Represent a git working tree + .git directory. + '''Represents a local Git repository. + + This class represents a directory that is the result of a `git clone` or + `git init`. It includes both the .git subdirectory and the working tree. + It is a thin abstraction, meant to make it easier to do certain git + operations. - This class represents a directory that is the result of a - "git clone". It includes both the .git subdirectory and - the working tree. It is a thin abstraction, meant to make - it easier to do certain git operations. + In the case of bare repositories, there is no working tree. These are + supported, but a NoWorkingTree exception will be raised if you try to + perform any operations that require a working tree. + + The repository must already exist on disk. Use gitdir.init() to create a + new repo, or gitdir.clone_from_cached_repo() if you want to clone a repo + that Morph has cached locally. ''' - def __init__(self, dirname): - self.dirname = morphlib.util.find_root(dirname, '.git') - # if we are in a bare repo, self.dirname will now be None - # so we just use the provided dirname - if not self.dirname: - self.dirname = dirname + def __init__(self, dirname, search_for_root=False): + '''Set up a GitDirectory instance for the repository at 'dirname'. + + If 'search_for_root' is set to True, 'dirname' may point to a + subdirectory inside the working tree of repository. Otherwise 'dirname' + must be the top directory. + + ''' + + if search_for_root: + dirname = morphlib.util.find_root(dirname, '.git') + + self.dirname = dirname self._config = {} + self._ensure_is_git_repo() + def _runcmd(self, argv, **kwargs): '''Run a command at the root of the git directory. @@ -375,6 +398,13 @@ class GitDirectory(object): def _runcmd_unchecked(self, *args, **kwargs): return cliapp.runcmd_unchecked(*args, cwd=self.dirname, **kwargs) + def _ensure_is_git_repo(self): + try: + self._runcmd(['git', 'rev-parse', '--git-dir']) + except cliapp.AppException as e: + # Exact error is logged already by the runcmd() function. + raise NoGitRepoError(self.dirname) + def checkout(self, branch_name): # pragma: no cover '''Check out a git branch.''' morphlib.git.gitcmd(self._runcmd, 'checkout', branch_name) @@ -429,8 +459,7 @@ class GitDirectory(object): def get_blob_contents(self, blob_id): # pragma: no cover '''Get file contents from git by ID''' - return morphlib.git.gitcmd(self._runcmd, 'cat-file', 'blob', - blob_id) + return morphlib.git.gitcmd(self._runcmd, 'cat-file', 'blob', blob_id) def get_commit_contents(self, commit_id): # pragma: no cover '''Get commit contents from git by ID''' @@ -454,11 +483,15 @@ class GitDirectory(object): self._config[key] = value def get_config(self, key): - '''Return value for a git repository configuration variable.''' + '''Return value for a git repository configuration variable. + + If the variable is unset, this will raise cliapp.AppException. + + ''' if key not in self._config: - value = morphlib.git.gitcmd(self._runcmd, 'config', '-z', key) - self._config[key] = value.rstrip('\0') + value = morphlib.git.gitcmd(self._runcmd, 'config', '-z', key) + self._config[key] = value.rstrip('\0') return self._config[key] def get_remote(self, *args, **kwargs): @@ -470,15 +503,16 @@ class GitDirectory(object): ''' return Remote(self, *args, **kwargs) - def update_remotes(self): # pragma: no cover + def update_remotes(self, echo_stderr=False): # pragma: no cover '''Run "git remote update --prune".''' - morphlib.git.gitcmd(self._runcmd, 'remote', 'update', '--prune') + morphlib.git.gitcmd(self._runcmd, 'remote', 'update', '--prune', + echo_stderr=echo_stderr) def is_bare(self): '''Determine whether the repository has no work tree (is bare)''' return self.get_config('core.bare') == 'true' - def list_files(self, ref=None): + def list_files(self, ref=None, recurse=True): '''Return an iterable of the files in the repository. If `ref` is specified, list files at that ref, otherwise @@ -491,9 +525,9 @@ class GitDirectory(object): if ref is None and self.is_bare(): raise NoWorkingTreeError(self) if ref is None: - return self._list_files_in_work_tree() + return self._list_files_in_work_tree(recurse) else: - return self._list_files_in_ref(ref) + return self._list_files_in_ref(ref, recurse) def _rev_parse(self, ref): try: @@ -537,31 +571,56 @@ class GitDirectory(object): def resolve_ref_to_tree(self, ref): return self._rev_parse('%s^{tree}' % ref) - def _list_files_in_work_tree(self): + def ref_exists(self, ref): + try: + self._rev_parse('%s^{commit}' % ref) + return True + except InvalidRefError: + return False + + def _list_files_in_work_tree(self, recurse=True): for dirpath, subdirs, filenames in os.walk(self.dirname): - if dirpath == self.dirname and '.git' in subdirs: + if not recurse: # pragma: no cover + subdirs[:] = [] + elif dirpath == self.dirname and '.git' in subdirs: subdirs.remove('.git') for filename in filenames: filepath = os.path.join(dirpath, filename) yield os.path.relpath(filepath, start=self.dirname) - def _list_files_in_ref(self, ref): + def _list_files_in_ref(self, ref, recurse=True): tree = self.resolve_ref_to_tree(ref) - output = morphlib.git.gitcmd(self._runcmd, 'ls-tree', - '--name-only', '-rz', tree) + + command = ['ls-tree', '--name-only', '-z'] + if recurse: + command.append('-r') + command.append(tree) + + output = morphlib.git.gitcmd(self._runcmd, *command) # ls-tree appends \0 instead of interspersing, so we need to # strip the trailing \0 before splitting paths = output.strip('\0').split('\0') return paths def read_file(self, filename, ref=None): + '''Attempts to read a file, from the working tree or a given ref. + + Raises an InvalidRefError if the ref is not found in the repository. + Raises an IOError if the requested file is not found in the ref. + + ''' + if ref is None and self.is_bare(): raise NoWorkingTreeError(self) if ref is None: with open(os.path.join(self.dirname, filename)) as f: return f.read() tree = self.resolve_ref_to_tree(ref) - return self.get_file_from_ref(tree, filename) + try: + return self.get_file_from_ref(tree, filename) + except cliapp.AppException: + raise IOError('File %s does not exist in ref %s of repo %s' % + (filename, ref, self)) def is_symlink(self, filename, ref=None): if ref is None and self.is_bare(): @@ -647,7 +706,7 @@ class GitDirectory(object): # this ensures it will fail if the branch already exists try: return self._update_ref((ref, sha1, '0' * 40), message) - except Exception, e: + except Exception as e: raise RefAddError(self, ref, sha1, e) def update_ref(self, ref, sha1, old_sha1, message=None): @@ -667,7 +726,7 @@ class GitDirectory(object): self._check_is_sha1(old_sha1) try: return self._update_ref((ref, sha1, old_sha1), message) - except Exception, e: + except Exception as e: raise RefUpdateError(self, ref, old_sha1, sha1, e) def delete_ref(self, ref, old_sha1, message=None): @@ -685,7 +744,7 @@ class GitDirectory(object): self._check_is_sha1(old_sha1) try: return self._update_ref(('-d', ref, old_sha1), message) - except Exception, e: + except Exception as e: raise RefDeleteError(self, ref, old_sha1, e) def describe(self): @@ -730,4 +789,3 @@ def clone_from_cached_repo(cached_repo, dirname, ref): # pragma: no cover cached_repo.clone_checkout(ref, dirname) return GitDirectory(dirname) - diff --git a/morphlib/gitdir_tests.py b/morphlib/gitdir_tests.py index 456e3716..a6e1921d 100644 --- a/morphlib/gitdir_tests.py +++ b/morphlib/gitdir_tests.py @@ -1,4 +1,4 @@ -# Copyright (C) 2013-2014 Codethink Limited +# Copyright (C) 2013-2015 Codethink Limited # # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by @@ -10,12 +10,12 @@ # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License along -# with this program; if not, write to the Free Software Foundation, Inc., -# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. +# with this program. If not, see <http://www.gnu.org/licenses/>. # # =*= License: GPL-2 =*= +import contextlib import datetime import os import shutil @@ -25,6 +25,26 @@ import unittest import morphlib +@contextlib.contextmanager +def monkeypatch(obj, attr, new_value): + old_value = getattr(obj, attr) + setattr(obj, attr, new_value) + yield + setattr(obj, attr, old_value) + + +def allow_nonexistant_git_repos(): + '''Disable the gitdir._ensure_is_git_repo() function. + + This is used in other unit tests to avoid needing to run 'git init' at the + start of each test. A library like 'mock' would be a better solution for + this problem. + + ''' + return monkeypatch( + morphlib.gitdir.GitDirectory, '_ensure_is_git_repo', lambda x: None) + + class GitDirectoryTests(unittest.TestCase): def setUp(self): @@ -34,30 +54,44 @@ class GitDirectoryTests(unittest.TestCase): def tearDown(self): shutil.rmtree(self.tempdir) - def fake_git_clone(self): + def empty_git_directory(self): os.mkdir(self.dirname) - os.mkdir(os.path.join(self.dirname, '.git')) + return morphlib.gitdir.init(self.dirname) + + def test_ensures_is_a_git_repo(self): + self.assertRaises(OSError, + morphlib.gitdir.GitDirectory, self.dirname) + + os.mkdir(self.dirname) + self.assertRaises(morphlib.gitdir.NoGitRepoError, + morphlib.gitdir.GitDirectory, self.dirname) def test_has_dirname_attribute(self): - self.fake_git_clone() - gitdir = morphlib.gitdir.GitDirectory(self.dirname) + gitdir = self.empty_git_directory() + self.assertEqual(gitdir.dirname, self.dirname) + + def test_can_search_for_top_directory(self): + self.empty_git_directory() + + path_inside_working_tree = os.path.join(self.dirname, 'a', 'b', 'c') + os.makedirs(path_inside_working_tree) + + gitdir = morphlib.gitdir.GitDirectory( + path_inside_working_tree, search_for_root=True) self.assertEqual(gitdir.dirname, self.dirname) def test_runs_command_in_right_directory(self): - self.fake_git_clone() - gitdir = morphlib.gitdir.GitDirectory(self.dirname) + gitdir = self.empty_git_directory() output = gitdir._runcmd(['pwd']) self.assertEqual(output.strip(), self.dirname) def test_sets_and_gets_configuration(self): - os.mkdir(self.dirname) - gitdir = morphlib.gitdir.init(self.dirname) + gitdir = self.empty_git_directory() gitdir.set_config('foo.bar', 'yoyo') self.assertEqual(gitdir.get_config('foo.bar'), 'yoyo') def test_gets_index(self): - os.mkdir(self.dirname) - gitdir = morphlib.gitdir.init(self.dirname) + gitdir = self.empty_git_directory() self.assertIsInstance(gitdir.get_index(), morphlib.gitindex.GitIndex) @@ -140,6 +174,12 @@ class GitDirectoryContentsTests(unittest.TestCase): self.assertRaises(morphlib.gitdir.InvalidRefError, gd.read_file, 'bar', 'no-such-ref') + def test_read_raises_io_error(self): + for gitdir in (self.dirname, self.mirror): + gd = morphlib.gitdir.GitDirectory(gitdir) + self.assertRaises(IOError, + gd.read_file, 'non-existant-file', 'HEAD') + def test_HEAD(self): gd = morphlib.gitdir.GitDirectory(self.dirname) self.assertEqual(gd.HEAD, 'master') @@ -162,6 +202,13 @@ class GitDirectoryContentsTests(unittest.TestCase): self.assertEqual(len(tree), 40) self.assertNotEqual(commit, tree) + def test_ref_exists(self): + gd = morphlib.gitdir.GitDirectory(self.dirname) + self.assertFalse(gd.ref_exists('non-existant-ref')) + self.assertTrue(gd.ref_exists('master')) + self.assertFalse( + gd.ref_exists('0000000000000000000000000000000000000000')) + def test_store_blob_with_string(self): gd = morphlib.gitdir.GitDirectory(self.dirname) sha1 = gd.store_blob('test string') diff --git a/morphlib/gitindex.py b/morphlib/gitindex.py index e22f6225..00098fbf 100644 --- a/morphlib/gitindex.py +++ b/morphlib/gitindex.py @@ -1,4 +1,4 @@ -# Copyright (C) 2013-2014 Codethink Limited +# Copyright (C) 2013-2015 Codethink Limited # # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by @@ -10,8 +10,7 @@ # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License along -# with this program; if not, write to the Free Software Foundation, Inc., -# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. +# with this program. If not, see <http://www.gnu.org/licenses/>. # # =*= License: GPL-2 =*= @@ -48,8 +47,16 @@ class GitIndex(object): def _run_git(self, *args, **kwargs): if self._index_file is not None: - kwargs['env'] = kwargs.get('env', dict(os.environ)) - kwargs['env']['GIT_INDEX_FILE'] = self._index_file + extra_env = kwargs.get('extra_env', {}) + extra_env['GIT_INDEX_FILE'] = self._index_file + kwargs['extra_env'] = extra_env + + if 'extra_env' in kwargs: + env = kwargs.get('env', dict(os.environ)) + env.update(kwargs['extra_env']) + kwargs['env'] = env + del kwargs['extra_env'] + return morphlib.git.gitcmd(self._gd._runcmd, *args, **kwargs) def _get_status(self): @@ -159,3 +166,11 @@ class GitIndex(object): def write_tree(self): '''Transform the index into a tree in the object store.''' return self._run_git('write-tree').strip() + + def checkout(self, working_tree=None): + '''Copy files from the index to the working tree.''' + if working_tree: + extra_env = {'GIT_WORK_TREE': working_tree} + else: + extra_env = {} + self._run_git('checkout-index', '--all', extra_env=extra_env) diff --git a/morphlib/gitindex_tests.py b/morphlib/gitindex_tests.py index 32d40a8c..a3196764 100644 --- a/morphlib/gitindex_tests.py +++ b/morphlib/gitindex_tests.py @@ -1,4 +1,4 @@ -# Copyright (C) 2013-2014 Codethink Limited +# Copyright (C) 2013-2015 Codethink Limited # # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by @@ -10,8 +10,7 @@ # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License along -# with this program; if not, write to the Free Software Foundation, Inc., -# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. +# with this program. If not, see <http://www.gnu.org/licenses/>. # # =*= License: GPL-2 =*= @@ -38,6 +37,8 @@ class GitIndexTests(unittest.TestCase): self.mirror = os.path.join(self.tempdir, 'mirror') morphlib.git.gitcmd(gd._runcmd, 'clone', '--mirror', self.dirname, self.mirror) + self.working_dir = os.path.join(self.tempdir, 'bar') + os.makedirs(self.working_dir) def tearDown(self): shutil.rmtree(self.tempdir) @@ -91,3 +92,15 @@ class GitIndexTests(unittest.TestCase): gd = morphlib.gitdir.GitDirectory(self.dirname) idx = gd.get_index() self.assertEqual(idx.write_tree(), gd.resolve_ref_to_tree(gd.HEAD)) + + def test_checkout(self): + gd = morphlib.gitdir.GitDirectory(self.dirname) + idx = gd.get_index() + idx.checkout(working_tree=self.working_dir) + self.assertTrue(os.path.exists(os.path.join(self.working_dir, 'foo'))) + + def test_checkout_without_working_dir(self): + gd = morphlib.gitdir.GitDirectory(self.dirname) + idx = gd.get_index() + idx.checkout() + self.assertTrue(os.path.exists(os.path.join(self.dirname, 'foo'))) diff --git a/morphlib/gitversion.py b/morphlib/gitversion.py index c593c330..7b4459a4 100644 --- a/morphlib/gitversion.py +++ b/morphlib/gitversion.py @@ -1,4 +1,4 @@ -# Copyright (C) 2013 - 2014 Codethink Limited +# Copyright (C) 2013-2015 Codethink Limited # # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by @@ -10,8 +10,7 @@ # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License along -# with this program; if not, write to the Free Software Foundation, Inc., -# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. +# with this program. If not, see <http://www.gnu.org/licenses/>. '''Version information retrieved either from the package data, or the @@ -34,7 +33,7 @@ try: commit = pkgutil.get_data('morphlib', 'commit') tree = pkgutil.get_data('morphlib', 'tree') ref = pkgutil.get_data('morphlib', 'ref') -except IOError, e: +except IOError as e: from os.path import dirname def run_git(*args): command = ['git'] + list(args) diff --git a/morphlib/localartifactcache.py b/morphlib/localartifactcache.py index 955ee97f..e6695c4e 100644 --- a/morphlib/localartifactcache.py +++ b/morphlib/localartifactcache.py @@ -1,4 +1,4 @@ -# Copyright (C) 2012, 2013, 2014 Codethink Limited +# Copyright (C) 2012, 2013, 2014-2015 Codethink Limited # # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by @@ -10,8 +10,7 @@ # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License along -# with this program; if not, write to the Free Software Foundation, Inc., -# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. +# with this program. If not, see <http://www.gnu.org/licenses/>. import collections diff --git a/morphlib/localartifactcache_tests.py b/morphlib/localartifactcache_tests.py index 4325cfbe..9483f9f6 100644 --- a/morphlib/localartifactcache_tests.py +++ b/morphlib/localartifactcache_tests.py @@ -1,4 +1,4 @@ -# Copyright (C) 2012,2014 Codethink Limited +# Copyright (C) 2012,2014-2015 Codethink Limited # # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by @@ -10,8 +10,7 @@ # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License along -# with this program; if not, write to the Free Software Foundation, Inc., -# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. +# with this program. If not, see <http://www.gnu.org/licenses/>. import unittest diff --git a/morphlib/localrepocache.py b/morphlib/localrepocache.py index 92c5e763..4fc3916a 100644 --- a/morphlib/localrepocache.py +++ b/morphlib/localrepocache.py @@ -1,4 +1,4 @@ -# Copyright (C) 2012-2014 Codethink Limited +# Copyright (C) 2012-2015 Codethink Limited # # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by @@ -10,14 +10,10 @@ # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License along -# with this program; if not, write to the Free Software Foundation, Inc., -# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. +# with this program. If not, see <http://www.gnu.org/licenses/>. -import logging import os -import re -import urllib2 import urlparse import string import sys @@ -176,7 +172,7 @@ class LocalRepoCache(object): self._git(['config', 'remote.origin.mirror', 'true'], cwd=path) self._git(['config', 'remote.origin.fetch', '+refs/*:refs/*'], cwd=path) - except BaseException, e: # pragma: no cover + except BaseException as e: # pragma: no cover if self.fs.exists(path): self.fs.removedir(path, force=True) return False, 'Unable to extract tarball %s: %s' % ( @@ -196,7 +192,7 @@ class LocalRepoCache(object): try: return self.get_repo(reponame) - except NotCached, e: + except NotCached as e: pass repourl = self._resolver.pull_url(reponame) @@ -204,7 +200,9 @@ class LocalRepoCache(object): if self._tarball_base_url: ok, error = self._clone_with_tarball(repourl, path) if ok: - return self.get_repo(reponame) + repo = self.get_repo(reponame) + repo.update() + return repo else: errors.append(error) self._app.status( @@ -215,7 +213,7 @@ class LocalRepoCache(object): try: self._git(['clone', '--mirror', '-n', repourl, target], echo_stderr=self._app.settings['verbose']) - except cliapp.AppException, e: + except cliapp.AppException as e: errors.append('Unable to clone from %s to %s: %s' % (repourl, target, e)) if self.fs.exists(target): @@ -225,6 +223,11 @@ class LocalRepoCache(object): self.fs.rename(target, path) return self.get_repo(reponame) + def _new_cached_repo_instance(self, reponame, repourl, + path): # pragma: no cover + return morphlib.cachedrepo.CachedRepo( + self._app, reponame, repourl, path) + def get_repo(self, reponame): '''Return an object representing a cached repository.''' @@ -234,21 +237,73 @@ class LocalRepoCache(object): repourl = self._resolver.pull_url(reponame) path = self._cache_name(repourl) if self.fs.exists(path): - repo = morphlib.cachedrepo.CachedRepo(self._app, reponame, - repourl, path) + repo = self._new_cached_repo_instance(reponame, repourl, path) self._cached_repo_objects[reponame] = repo return repo raise NotCached(reponame) - def get_updated_repo(self, reponame): # pragma: no cover - '''Return object representing cached repository, which is updated.''' + def get_updated_repo(self, repo_name, ref=None): # pragma: no cover + '''Return object representing cached repository. - if not self._app.settings['no-git-update']: - cached_repo = self.cache_repo(reponame) - self._app.status( - msg='Updating git repository %s in cache' % reponame) - cached_repo.update() - else: - cached_repo = self.get_repo(reponame) - return cached_repo + If 'ref' is None, the repo will be updated unless + app.settings['no-git-update'] is set. + + If 'ref' is set to a SHA1, the repo will only be updated if 'ref' isn't + already available locally. + ''' + + if self._app.settings['no-git-update']: + self._app.status(msg='Not updating existing git repository ' + '%(repo_name)s ' + 'because of no-git-update being set', + chatty=True, + repo_name=repo_name) + return self.get_repo(repo_name) + + if self.has_repo(repo_name): + repo = self.get_repo(repo_name) + if ref and morphlib.git.is_valid_sha1(ref): + try: + repo.resolve_ref_to_commit(ref) + self._app.status(msg='Not updating git repository ' + '%(repo_name)s because it ' + 'already contains sha1 %(sha1)s', + chatty=True, repo_name=repo_name, + sha1=ref) + return repo + except morphlib.gitdir.InvalidRefError: + pass + + self._app.status(msg='Updating %(repo_name)s', + repo_name=repo_name) + repo.update() + return repo + else: + self._app.status(msg='Cloning %(repo_name)s', + repo_name=repo_name) + return self.cache_repo(repo_name) + + def ensure_submodules(self, toplevel_repo, + toplevel_ref): # pragma: no cover + '''Ensure any submodules of a given repo are cached and up to date.''' + + def submodules_for_repo(repo_path, ref): + try: + submodules = morphlib.git.Submodules(self._app, repo_path, ref) + submodules.load() + return [(submod.url, submod.commit) for submod in submodules] + except morphlib.git.NoModulesFileError: + return [] + + done = set() + subs_to_process = submodules_for_repo(toplevel_repo.path, toplevel_ref) + while subs_to_process: + url, ref = subs_to_process.pop() + done.add((url, ref)) + + cached_repo = self.get_updated_repo(url, ref=ref) + + for submod in submodules_for_repo(cached_repo.path, ref): + if (submod.url, submod.commit) not in done: + subs_to_process.add((submod.url, submod.commit)) diff --git a/morphlib/localrepocache_tests.py b/morphlib/localrepocache_tests.py index 3cc4f07f..aeef18f1 100644 --- a/morphlib/localrepocache_tests.py +++ b/morphlib/localrepocache_tests.py @@ -1,4 +1,4 @@ -# Copyright (C) 2012-2014 Codethink Limited +# Copyright (C) 2012-2015 Codethink Limited # # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by @@ -10,8 +10,7 @@ # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License along -# with this program; if not, write to the Free Software Foundation, Inc., -# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. +# with this program. If not, see <http://www.gnu.org/licenses/>. import unittest @@ -22,6 +21,7 @@ import cliapp import fs.memoryfs import morphlib +import morphlib.gitdir_tests class FakeApplication(object): @@ -56,6 +56,7 @@ class LocalRepoCacheTests(unittest.TestCase): self.lrc._git = self.fake_git self.lrc._fetch = self.not_found self.lrc._mkdtemp = self.fake_mkdtemp + self.lrc._new_cached_repo_instance = self.new_cached_repo_instance self._mkdtemp_count = 0 def fake_git(self, args, **kwargs): @@ -86,6 +87,11 @@ class LocalRepoCacheTests(unittest.TestCase): self.lrc.fs.makedir(dirname+"/"+thing) return thing + def new_cached_repo_instance(self, *args): + with morphlib.gitdir_tests.allow_nonexistant_git_repos(): + return morphlib.cachedrepo.CachedRepo( + FakeApplication(), *args) + def not_found(self, url, path): raise cliapp.AppException('Not found') @@ -132,7 +138,11 @@ class LocalRepoCacheTests(unittest.TestCase): self.lrc._fetch = lambda url, path: self.fetched.append(url) self.unpacked_tar = "" self.mkdir_path = "" - self.lrc.cache_repo(self.repourl) + + with morphlib.gitdir_tests.monkeypatch( + morphlib.cachedrepo.CachedRepo, 'update', lambda self: None): + self.lrc.cache_repo(self.repourl) + self.assertEqual(self.fetched, [self.tarball_url]) self.assertFalse(self.lrc.fs.exists(self.cache_path + '.tar')) self.assertEqual(self.remotes['origin']['url'], self.repourl) diff --git a/morphlib/morphloader.py b/morphlib/morphloader.py index 8289b01e..47cb03d7 100644 --- a/morphlib/morphloader.py +++ b/morphlib/morphloader.py @@ -1,4 +1,4 @@ -# Copyright (C) 2013-2014 Codethink Limited +# Copyright (C) 2013-2015 Codethink Limited # # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by @@ -10,8 +10,7 @@ # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License along -# with this program; if not, write to the Free Software Foundation, Inc., -# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. +# with this program. If not, see <http://www.gnu.org/licenses/>. # # =*= License: GPL-2 =*= @@ -111,14 +110,6 @@ class UnknownArchitectureError(MorphologyValidationError): % (arch, morph_filename)) -class NoBuildDependenciesError(MorphologyValidationError): - - def __init__(self, stratum_name, chunk_name, morph_filename): - self.msg = ( - 'Stratum %s has no build dependencies for chunk %s in %s' % - (stratum_name, chunk_name, morph_filename)) - - class NoStratumBuildDependenciesError(MorphologyValidationError): def __init__(self, stratum_name, morph_filename): @@ -398,13 +389,17 @@ class MorphologyLoader(object): return morphlib.morphology.Morphology(obj) - def load_from_string(self, string, filename='string'): + def load_from_string(self, string, + filename='string'): # pragma: no cover '''Load a morphology from a string. Return the Morphology object. ''' + if string is None: + return None + m = self.parse_morphology_text(string, filename) m.filename = filename self.validate(m) @@ -552,7 +547,7 @@ class MorphologyLoader(object): # Validate build-dependencies if specified self._validate_stratum_specs_fields(morph, 'build-depends') - # Require build-dependencies for each chunk. + # Check build-dependencies for each chunk. for spec in morph['chunks']: chunk_name = spec.get('alias', spec['name']) if 'build-depends' in spec: @@ -560,9 +555,6 @@ class MorphologyLoader(object): raise InvalidTypeError( '%s.build-depends' % chunk_name, list, type(spec['build-depends']), morph['name']) - else: - raise NoBuildDependenciesError( - morph['name'], chunk_name, morph.filename) @classmethod def _validate_chunk(cls, morphology): diff --git a/morphlib/morphloader_tests.py b/morphlib/morphloader_tests.py index dd70c824..f9fa2a34 100644 --- a/morphlib/morphloader_tests.py +++ b/morphlib/morphloader_tests.py @@ -1,4 +1,4 @@ -# Copyright (C) 2013-2014 Codethink Limited +# Copyright (C) 2013-2015 Codethink Limited # # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by @@ -10,8 +10,7 @@ # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License along -# with this program; if not, write to the Free Software Foundation, Inc., -# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. +# with this program. If not, see <http://www.gnu.org/licenses/>. # # =*= License: GPL-2 =*= @@ -344,26 +343,6 @@ build-system: dummy self.loader.validate(m) self.assertEqual(m['arch'], 'armv7l') - def test_validate_requires_build_deps_for_chunks_in_strata(self): - m = morphlib.morphology.Morphology( - { - "kind": "stratum", - "name": "foo", - "chunks": [ - { - "name": "foo", - "repo": "foo", - "ref": "foo", - "morph": "foo", - "build-mode": "bootstrap", - } - ], - }) - - self.assertRaises( - morphlib.morphloader.NoBuildDependenciesError, - self.loader.validate, m) - def test_validate_requires_build_deps_or_bootstrap_mode_for_strata(self): m = morphlib.morphology.Morphology( { diff --git a/morphlib/morphology.py b/morphlib/morphology.py index 009ed044..53a8ff39 100644 --- a/morphlib/morphology.py +++ b/morphlib/morphology.py @@ -1,4 +1,4 @@ -# Copyright (C) 2013-2014 Codethink Limited +# Copyright (C) 2013-2015 Codethink Limited # # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by @@ -10,8 +10,7 @@ # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License along -# with this program; if not, write to the Free Software Foundation, Inc., -# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. +# with this program. If not, see <http://www.gnu.org/licenses/>. # # =*= License: GPL-2 =*= diff --git a/morphlib/morphology_tests.py b/morphlib/morphology_tests.py index 385f62ee..d3d99f15 100644 --- a/morphlib/morphology_tests.py +++ b/morphlib/morphology_tests.py @@ -1,4 +1,4 @@ -# Copyright (C) 2013-2014 Codethink Limited +# Copyright (C) 2013-2015 Codethink Limited # # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by @@ -10,8 +10,7 @@ # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License along -# with this program; if not, write to the Free Software Foundation, Inc., -# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. +# with this program. If not, see <http://www.gnu.org/licenses/>. # # =*= License: GPL-2 =*= diff --git a/morphlib/morphologyfactory.py b/morphlib/morphologyfactory.py deleted file mode 100644 index 606ca1ea..00000000 --- a/morphlib/morphologyfactory.py +++ /dev/null @@ -1,86 +0,0 @@ -# Copyright (C) 2012-2014 Codethink Limited -# -# This program is free software; you can redistribute it and/or modify -# it under the terms of the GNU General Public License as published by -# the Free Software Foundation; version 2 of the License. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License along -# with this program; if not, write to the Free Software Foundation, Inc., -# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. - - -import os - -import morphlib -import cliapp - - -class MorphologyNotFoundError(cliapp.AppException): - def __init__(self, filename): - cliapp.AppException.__init__( - self, "Couldn't find morphology: %s" % filename) - - -class NotcachedError(cliapp.AppException): - def __init__(self, repo_name): - cliapp.AppException.__init__( - self, "Repository %s is not cached locally and there is no " - "remote cache specified" % repo_name) - - -class MorphologyFactory(object): - - '''A way of creating morphologies which will provide a default''' - - def __init__(self, local_repo_cache, remote_repo_cache=None, app=None): - self._lrc = local_repo_cache - self._rrc = remote_repo_cache - self._app = app - - def status(self, *args, **kwargs): # pragma: no cover - if self._app is not None: - self._app.status(*args, **kwargs) - - def get_morphology(self, reponame, sha1, filename): - morph_name = os.path.splitext(os.path.basename(filename))[0] - loader = morphlib.morphloader.MorphologyLoader() - if self._lrc.has_repo(reponame): - self.status(msg="Looking for %s in local repo cache" % filename, - chatty=True) - try: - repo = self._lrc.get_repo(reponame) - morph = loader.load_from_string(repo.cat(sha1, filename)) - except IOError: - morph = None - file_list = repo.ls_tree(sha1) - elif self._rrc is not None: - self.status(msg="Retrieving %(reponame)s %(sha1)s %(filename)s" - " from the remote git cache.", - reponame=reponame, sha1=sha1, filename=filename, - chatty=True) - try: - text = self._rrc.cat_file(reponame, sha1, filename) - morph = loader.load_from_string(text) - except morphlib.remoterepocache.CatFileError: - morph = None - file_list = self._rrc.ls_tree(reponame, sha1) - else: - raise NotcachedError(reponame) - - if morph is None: - self.status(msg="File %s doesn't exist: attempting to infer " - "chunk morph from repo's build system" - % filename, chatty=True) - bs = morphlib.buildsystem.detect_build_system(file_list) - if bs is None: - raise MorphologyNotFoundError(filename) - morph = bs.get_morphology(morph_name) - loader.validate(morph) - loader.set_commands(morph) - loader.set_defaults(morph) - return morph diff --git a/morphlib/morphologyfinder.py b/morphlib/morphologyfinder.py index 87c0de1a..708c86a9 100644 --- a/morphlib/morphologyfinder.py +++ b/morphlib/morphologyfinder.py @@ -1,4 +1,4 @@ -# Copyright (C) 2013-2014 Codethink Limited +# Copyright (C) 2013-2015 Codethink Limited # # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by @@ -10,8 +10,7 @@ # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License along -# with this program; if not, write to the Free Software Foundation, Inc., -# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. +# with this program. If not, see <http://www.gnu.org/licenses/>. # # =*= License: GPL-2 =*= diff --git a/morphlib/morphologyfinder_tests.py b/morphlib/morphologyfinder_tests.py index 67161f9b..a83f89a5 100644 --- a/morphlib/morphologyfinder_tests.py +++ b/morphlib/morphologyfinder_tests.py @@ -1,4 +1,4 @@ -# Copyright (C) 2013-2014 Codethink Limited +# Copyright (C) 2013-2015 Codethink Limited # # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by @@ -10,8 +10,7 @@ # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License along -# with this program; if not, write to the Free Software Foundation, Inc., -# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. +# with this program. If not, see <http://www.gnu.org/licenses/>. # # =*= License: GPL-2 =*= diff --git a/morphlib/morphset.py b/morphlib/morphset.py index bf061f94..a4cb5dae 100644 --- a/morphlib/morphset.py +++ b/morphlib/morphset.py @@ -1,4 +1,4 @@ -# Copyright (C) 2013-2014 Codethink Limited +# Copyright (C) 2013-2015 Codethink Limited # # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by @@ -10,8 +10,7 @@ # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License along -# with this program; if not, write to the Free Software Foundation, Inc., -# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. +# with this program. If not, see <http://www.gnu.org/licenses/>. # # =*= License: GPL-2 =*= diff --git a/morphlib/morphset_tests.py b/morphlib/morphset_tests.py index 81b5810f..bf726d4c 100644 --- a/morphlib/morphset_tests.py +++ b/morphlib/morphset_tests.py @@ -1,4 +1,4 @@ -# Copyright (C) 2013, 2014 Codethink Limited +# Copyright (C) 2013, 2014-2015 Codethink Limited # # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by @@ -10,8 +10,7 @@ # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License along -# with this program; if not, write to the Free Software Foundation, Inc., -# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. +# with this program. If not, see <http://www.gnu.org/licenses/>. # # =*= License: GPL-2 =*= diff --git a/morphlib/mountableimage.py b/morphlib/mountableimage.py index f767228a..dcc496e7 100644 --- a/morphlib/mountableimage.py +++ b/morphlib/mountableimage.py @@ -1,4 +1,4 @@ -# Copyright (C) 2012-2013 Codethink Limited +# Copyright (C) 2012-2013,2015 Codethink Limited # # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by @@ -10,8 +10,7 @@ # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License along -# with this program; if not, write to the Free Software Foundation, Inc., -# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. +# with this program. If not, see <http://www.gnu.org/licenses/>. import cliapp @@ -46,7 +45,7 @@ class MountableImage(object): # pragma: no cover infh = gzip.open(path, "rb") morphlib.util.copyfileobj(infh, outfh) infh.close() - except BaseException, e: + except BaseException as e: logging.error('Caught exception: %s' % str(e)) logging.info('Removing temporary file %s' % self.temp_path) os.unlink(self.temp_path) @@ -65,17 +64,17 @@ class MountableImage(object): # pragma: no cover chatty=True) try: morphlib.fsutils.unmount(self.app.runcmd, mount_point) - except BaseException, e: + except BaseException as e: logging.info('Ignoring error when unmounting: %s' % str(e)) try: morphlib.fsutils.undo_device_mapping(self.app.runcmd, path) - except BaseException, e: + except BaseException as e: logging.info( 'Ignoring error when undoing device mapping: %s' % str(e)) try: os.rmdir(mount_point) os.unlink(path) - except BaseException, e: + except BaseException as e: logging.info( 'Ignoring error when removing temporary files: %s' % str(e)) diff --git a/morphlib/plugins/add_binary_plugin.py b/morphlib/plugins/add_binary_plugin.py index a192f792..45edae4c 100644 --- a/morphlib/plugins/add_binary_plugin.py +++ b/morphlib/plugins/add_binary_plugin.py @@ -1,4 +1,4 @@ -# Copyright (C) 2014 Codethink Limited +# Copyright (C) 2014-2015 Codethink Limited # # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by @@ -10,8 +10,7 @@ # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License along -# with this program; if not, write to the Free Software Foundation, Inc., -# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. +# with this program. If not, see <http://www.gnu.org/licenses/>. import cliapp @@ -56,7 +55,7 @@ class AddBinaryPlugin(cliapp.Plugin): if not binaries: raise morphlib.Error('add-binary must get at least one argument') - gd = morphlib.gitdir.GitDirectory(os.getcwd()) + gd = morphlib.gitdir.GitDirectory(os.getcwd(), search_for_root=True) gd.fat_init() if not gd.has_fat(): self._make_gitfat(gd) diff --git a/morphlib/plugins/artifact_inspection_plugin.py b/morphlib/plugins/artifact_inspection_plugin.py index 74645f41..b16f393c 100644 --- a/morphlib/plugins/artifact_inspection_plugin.py +++ b/morphlib/plugins/artifact_inspection_plugin.py @@ -1,4 +1,4 @@ -# Copyright (C) 2012-2014 Codethink Limited +# Copyright (C) 2012-2015 Codethink Limited # # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by @@ -10,8 +10,7 @@ # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License along -# with this program; if not, write to the Free Software Foundation, Inc., -# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. +# with this program. If not, see <http://www.gnu.org/licenses/>. import cliapp @@ -51,7 +50,7 @@ class ProjectVersionGuesser(object): if self.lrc.has_repo(repo): repository = self.lrc.get_repo(repo) for filename in filenames: - yield filename, repository.cat(ref, filename) + yield filename, repository.read_file(filename, ref) elif self.rrc: for filename in filenames: yield filename, self.rrc.cat_file(repo, ref, filename) @@ -153,7 +152,7 @@ class VersionGuesser(object): repository = self.lrc.get_repo(repo) if not self.app.settings['no-git-update']: repository.update() - tree = repository.ls_tree(ref) + tree = repository.list_files(ref=ref, recurse=False) elif self.rrc: repository = None tree = self.rrc.ls_tree(repo, ref) @@ -163,7 +162,7 @@ class VersionGuesser(object): version = guesser.guess_version(repo, ref, tree) if version: break - except cliapp.AppException, err: + except cliapp.AppException as err: self.app.status(msg='%(repo)s: Failed to list files in %(ref)s', repo=repo, ref=ref, chatty=True) return version diff --git a/morphlib/plugins/branch_and_merge_plugin.py b/morphlib/plugins/branch_and_merge_plugin.py index 5531f7f6..08589ea6 100644 --- a/morphlib/plugins/branch_and_merge_plugin.py +++ b/morphlib/plugins/branch_and_merge_plugin.py @@ -1,4 +1,4 @@ -# Copyright (C) 2012,2013,2014 Codethink Limited +# Copyright (C) 2012-2015 Codethink Limited # # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by @@ -10,8 +10,7 @@ # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License along -# with this program; if not, write to the Free Software Foundation, Inc., -# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. +# with this program. If not, see <http://www.gnu.org/licenses/>. import cliapp @@ -174,7 +173,7 @@ class BranchAndMergePlugin(cliapp.Plugin): cached_repo = lrc.get_updated_repo(root_url) # Check the git branch exists. - cached_repo.resolve_ref(system_branch) + cached_repo.resolve_ref_to_commit(system_branch) with self._initializing_system_branch( ws, root_url, system_branch, cached_repo, base_ref) as (sb, gd): @@ -233,7 +232,7 @@ class BranchAndMergePlugin(cliapp.Plugin): (system_branch, root_url)) # Make sure the base_ref exists. - cached_repo.resolve_ref(base_ref) + cached_repo.resolve_ref_to_commit(base_ref) with self._initializing_system_branch( ws, root_url, system_branch, cached_repo, base_ref) as (sb, gd): diff --git a/morphlib/plugins/build_plugin.py b/morphlib/plugins/build_plugin.py index 218bd819..2cc395fc 100644 --- a/morphlib/plugins/build_plugin.py +++ b/morphlib/plugins/build_plugin.py @@ -1,4 +1,4 @@ -# Copyright (C) 2012,2013,2014 Codethink Limited +# Copyright (C) 2012-2015 Codethink Limited # # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by @@ -10,13 +10,13 @@ # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License along -# with this program; if not, write to the Free Software Foundation, Inc., -# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. +# with this program. If not, see <http://www.gnu.org/licenses/>. import cliapp import contextlib import uuid +import logging import morphlib @@ -135,11 +135,13 @@ class BuildPlugin(cliapp.Plugin): The location of the resulting system image artifact is printed at the end of the build output. - You do not need to commit your changes before building, Morph - does that for you, in a temporary branch for each build. However, - note that Morph does not untracked files to the temporary branch, - only uncommitted changes to files git already knows about. You - need to `git add` and commit each new file yourself. + If the 'local-changes' setting is set to 'include', you do not need + to commit your changes before building. Morph does that for you, in a + temporary branch for each build. Note that any system produced this way + will not be reproducible later on as the branch it is built from will + have been deleted. Also note that Morph does not add untracked files to + the temporary branch, only uncommitted changes to files git already + knows about. You need to `git add` and commit each new file yourself. Example: @@ -158,12 +160,13 @@ class BuildPlugin(cliapp.Plugin): self.app.settings['cachedir'], self.app.settings['cachedir-min-space']) - system_filename = morphlib.util.sanitise_morphology_path(args[0]) - ws = morphlib.workspace.open('.') sb = morphlib.sysbranchdir.open_from_within('.') - build_uuid = uuid.uuid4().hex + system_filename = morphlib.util.sanitise_morphology_path(args[0]) + system_filename = sb.relative_to_root_repo(system_filename) + + logging.debug('System branch is %s' % sb.root_directory) if self.use_distbuild: addr = self.app.settings['controller-initiator-address'] @@ -174,6 +177,22 @@ class BuildPlugin(cliapp.Plugin): else: build_command = morphlib.buildcommand.BuildCommand(self.app) + if self.app.settings['local-changes'] == 'include': + self._build_with_local_changes(build_command, sb, system_filename) + else: + self._build_local_commit(build_command, sb, system_filename) + + def _build_with_local_changes(self, build_command, sb, system_filename): + '''Construct a branch including user's local changes, and build that. + + It is often a slow process to check all repos in the system branch for + local changes. However, when using a distributed build cluster, all + code being built must be pushed to the associated Trove, and it can be + helpful to have this automated as part of the `morph build` command. + + ''' + build_uuid = uuid.uuid4().hex + loader = morphlib.morphloader.MorphologyLoader() push = self.app.settings['push-build-branches'] name = morphlib.git.get_user_name(self.app.runcmd) @@ -194,3 +213,33 @@ class BuildPlugin(cliapp.Plugin): with pbb as (repo, commit, original_ref): build_command.build(repo, commit, system_filename, original_ref=original_ref) + + def _build_local_commit(self, build_command, sb, system_filename): + '''Build whatever commit the user has checked-out locally. + + This ignores any uncommitted changes. Also, if the user has a commit + checked out locally that hasn't been pushed to the Trove that Morph is + configured to work with, the build will fail in this sort of way: + + ERROR: Ref c55b853d92a52a5b5fe62edbfbf351169eb79c0a is an invalid + reference for repo + git://git.baserock.org/baserock/baserock/definitions + + The build process doesn't use the checked-out definitions repo at all, + except to resolve the checked-out commit (HEAD). Instead, it uses the + cached version of the definitions repo, updating the cache if + necessary. + + We don't detect and warn the user about any uncommitted changes because + doing so is slow when there are no changes (around 5 seconds on my + machine for Baserock's definitions.git). + + ''' + root_repo_url = sb.get_config('branch.root') + ref = sb.get_config('branch.name') + + definitions_repo_path = sb.get_git_directory_name(root_repo_url) + definitions_repo = morphlib.gitdir.GitDirectory(definitions_repo_path) + commit = definitions_repo.resolve_ref_to_commit(ref) + + build_command.build(root_repo_url, commit, system_filename) diff --git a/morphlib/plugins/cross-bootstrap_plugin.py b/morphlib/plugins/cross-bootstrap_plugin.py index 7b53a4a5..79609cb5 100644 --- a/morphlib/plugins/cross-bootstrap_plugin.py +++ b/morphlib/plugins/cross-bootstrap_plugin.py @@ -1,4 +1,4 @@ -# Copyright (C) 2013-2014 Codethink Limited +# Copyright (C) 2013-2015 Codethink Limited # # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by @@ -10,8 +10,7 @@ # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License along -# with this program; if not, write to the Free Software Foundation, Inc., -# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. +# with this program. If not, see <http://www.gnu.org/licenses/>. import cliapp import logging @@ -46,7 +45,7 @@ def escape_source_name(source): # Most of this is ripped from RootfsTarballBuilder, and should be reconciled # with it. -class BootstrapSystemBuilder(morphlib.builder2.BuilderBase): +class BootstrapSystemBuilder(morphlib.builder.BuilderBase): '''Build a bootstrap system tarball The bootstrap system image contains a minimal cross-compiled toolchain @@ -66,7 +65,7 @@ class BootstrapSystemBuilder(morphlib.builder2.BuilderBase): self.unpack_sources(fs_root) self.write_build_script(fs_root) self.create_tarball(handle, fs_root, system_name) - except BaseException, e: + except BaseException as e: logging.error(traceback.format_exc()) self.app.status(msg='Error while building bootstrap image', error=True) @@ -85,7 +84,7 @@ class BootstrapSystemBuilder(morphlib.builder2.BuilderBase): with cache.get(chunk_artifact) as chunk_file: try: morphlib.bins.unpack_binary_from_file(chunk_file, dest) - except BaseException, e: + except BaseException as e: self.app.status( msg='Error unpacking binary chunk %(name)s', name=chunk_artifact.name, @@ -104,14 +103,14 @@ class BootstrapSystemBuilder(morphlib.builder2.BuilderBase): source_dir = os.path.join(path, 'src', escaped_source) if not os.path.exists(source_dir): os.makedirs(source_dir) - morphlib.builder2.extract_sources( + morphlib.builder.extract_sources( self.app, self.repo_cache, s.repo, s.sha1, source_dir) name = s.name chunk_script = os.path.join(path, 'src', 'build-%s' % name) with morphlib.savefile.SaveFile(chunk_script, 'w') as f: self.write_chunk_build_script(s, f) - os.chmod(chunk_script, 0777) + os.chmod(chunk_script, 0o777) def write_build_script(self, path): '''Output a script to run build on the bootstrap target''' @@ -136,7 +135,7 @@ class BootstrapSystemBuilder(morphlib.builder2.BuilderBase): f.write('if [ -e /sbin/ldconfig ]; then /sbin/ldconfig; fi\n') f.write(driver_footer) - os.chmod(driver_script, 0777) + os.chmod(driver_script, 0o777) def write_chunk_build_script(self, source, f): m = source.morphology diff --git a/morphlib/plugins/deploy_plugin.py b/morphlib/plugins/deploy_plugin.py index e795e637..7635a7b4 100644 --- a/morphlib/plugins/deploy_plugin.py +++ b/morphlib/plugins/deploy_plugin.py @@ -1,4 +1,4 @@ -# Copyright (C) 2013, 2014 Codethink Limited +# Copyright (C) 2013-2015 Codethink Limited # # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by @@ -10,8 +10,7 @@ # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License along -# with this program; if not, write to the Free Software Foundation, Inc., -# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. +# with this program. If not, see <http://www.gnu.org/licenses/>. import json @@ -124,7 +123,7 @@ class DeployPlugin(cliapp.Plugin): Each system defined in a cluster morphology can be deployed in multiple ways (`type` in a cluster morphology). Morph provides - five types of deployment: + the following types of deployment: * `tar` where Morph builds a tar archive of the root file system. @@ -144,6 +143,35 @@ class DeployPlugin(cliapp.Plugin): * `nfsboot` where Morph creates a system to be booted over a network. + * `ssh-rsync` where Morph copies a binary delta over to the target + system and arranges for it to be bootable. This requires + `system-version-manager` from the tbdiff chunk + + * `initramfs`, where Morph turns the system into an initramfs image, + suitable for being used as the early userland environment for a + system to be able to locate more complicated storage for its root + file-system, or on its own for diskless deployments. + + There are additional extensions that currently live in the Baserock + definitions repo (baserock:baserock/definitions). These include: + + * `image-package` where Morph creates a tarball that includes scripts + that can be used to make disk images outside of a Baserock + environment. The example in definitions.git will create scripts for + generating disk images and installing to existing disks. + + * `sdk` where Morph generates something resembing a BitBake SDK, which + provides a toolchain for building software to target a system built + by Baserock, from outside of a Baserock environment. This creates a + self-extracting shell archive which you pass a directory to extract + to, and inside that has a shell snippet called + environment-setup-$TARGET which can be used to set environment + variables to use the toolchain. + + * `pxeboot` where Morph temporarily network-boots the system you are + deploying, so it can install a more permanent system onto local + storage. + In addition to the deployment type, the user must also give a value for `location`. Its syntax depends on the deployment types. The deployment types provided by Morph use the @@ -177,6 +205,31 @@ class DeployPlugin(cliapp.Plugin): the _address_ of the trove, _not_ `user@...`, since `root@` will automatically be prepended to the server address.) + In addition to the `location`parameter, deployments can take additional + `KEY=VALUE` parameters. These can be provided in the following ways: + + 1. In the cluster definition file, e.g. + + ... + systems: + - morph: systems/foo-system.morph + deploy: + foo: + HOSTNAME: foo + + 2. In the environment before running e.g. + + `HOSTNAME=foo morph deploy ...` + + 3. On the command-line e.g. + `morph deploy clusters/foo.morph foo.HOSTNAME=foo` + + For any boolean `KEY=VALUE` parameters, allowed values are: + + +ve `yes`, `1`, `true`; + + -ve `no`, `0`, `false`; + The following `KEY=VALUE` parameters are supported for `rawdisk`, `virtualbox-ssh` and `kvm` and deployment types: @@ -281,12 +334,12 @@ class DeployPlugin(cliapp.Plugin): self.app.settings['tempdir-min-space'], '/', 0) - self.app.settings['no-git-update'] = True - cluster_filename = morphlib.util.sanitise_morphology_path(args[0]) - ws = morphlib.workspace.open('.') sb = morphlib.sysbranchdir.open_from_within('.') + cluster_filename = morphlib.util.sanitise_morphology_path(args[0]) + cluster_filename = sb.relative_to_root_repo(cluster_filename) + build_uuid = uuid.uuid4().hex build_command = morphlib.buildcommand.BuildCommand(self.app) @@ -298,6 +351,7 @@ class DeployPlugin(cliapp.Plugin): build_ref_prefix = self.app.settings['build-ref-prefix'] root_repo_dir = morphlib.gitdir.GitDirectory( sb.get_git_directory_name(sb.root_repository_url)) + cluster_text = root_repo_dir.read_file(cluster_filename) cluster_morphology = loader.load_from_string(cluster_text, filename=cluster_filename) @@ -323,23 +377,24 @@ class DeployPlugin(cliapp.Plugin): self.validate_deployment_options( env_vars, all_deployments, all_subsystems) - bb = morphlib.buildbranch.BuildBranch(sb, build_ref_prefix) - pbb = morphlib.buildbranch.pushed_build_branch( - bb, loader=loader, changes_need_pushing=False, - name=name, email=email, build_uuid=build_uuid, - status=self.app.status) - with pbb as (repo, commit, original_ref): - # Create a tempdir for this deployment to work in - deploy_tempdir = tempfile.mkdtemp( - dir=os.path.join(self.app.settings['tempdir'], 'deployments')) - try: - for system in cluster_morphology['systems']: - self.deploy_system(build_command, deploy_tempdir, - root_repo_dir, repo, commit, system, - env_vars, deployments, - parent_location='') - finally: - shutil.rmtree(deploy_tempdir) + if self.app.settings['local-changes'] == 'include': + bb = morphlib.buildbranch.BuildBranch(sb, build_ref_prefix) + pbb = morphlib.buildbranch.pushed_build_branch( + bb, loader=loader, changes_need_pushing=False, + name=name, email=email, build_uuid=build_uuid, + status=self.app.status) + with pbb as (repo, commit, original_ref): + self.deploy_cluster(build_command, cluster_morphology, + root_repo_dir, repo, commit, env_vars, + deployments) + else: + repo = sb.get_config('branch.root') + ref = sb.get_config('branch.name') + commit = root_repo_dir.resolve_ref_to_commit(ref) + + self.deploy_cluster(build_command, cluster_morphology, + root_repo_dir, repo, commit, env_vars, + deployments) self.app.status(msg='Finished deployment') @@ -359,6 +414,20 @@ class DeployPlugin(cliapp.Plugin): 'Variable referenced a non-existent deployment ' 'name: %s' % var) + def deploy_cluster(self, build_command, cluster_morphology, root_repo_dir, + repo, commit, env_vars, deployments): + # Create a tempdir for this deployment to work in + deploy_tempdir = tempfile.mkdtemp( + dir=os.path.join(self.app.settings['tempdir'], 'deployments')) + try: + for system in cluster_morphology['systems']: + self.deploy_system(build_command, deploy_tempdir, + root_repo_dir, repo, commit, system, + env_vars, deployments, + parent_location='') + finally: + shutil.rmtree(deploy_tempdir) + def deploy_system(self, build_command, deploy_tempdir, root_repo_dir, build_repo, ref, system, env_vars, deployment_filter, parent_location): diff --git a/morphlib/plugins/distbuild_plugin.py b/morphlib/plugins/distbuild_plugin.py index 66d86dcf..1900b1bd 100644 --- a/morphlib/plugins/distbuild_plugin.py +++ b/morphlib/plugins/distbuild_plugin.py @@ -1,6 +1,6 @@ # distbuild_plugin.py -- Morph distributed build plugin # -# Copyright (C) 2014 Codethink Limited +# Copyright (C) 2014-2015 Codethink Limited # # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by @@ -12,8 +12,7 @@ # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License along -# with this program; if not, write to the Free Software Foundation, Inc., -# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.. +# with this program. If not, see <http://www.gnu.org/licenses/>. import cliapp @@ -72,7 +71,9 @@ class SerialiseArtifactPlugin(cliapp.Plugin): srcpool = build_command.create_source_pool( repo_name, ref, filename, original_ref=original_ref) artifact = build_command.resolve_artifacts(srcpool) - self.app.output.write(distbuild.serialise_artifact(artifact)) + self.app.output.write(distbuild.serialise_artifact(artifact, + repo_name, + ref)) self.app.output.write('\n') @@ -96,9 +97,14 @@ class WorkerBuild(cliapp.Plugin): distbuild.add_crash_conditions(self.app.settings['crash-condition']) serialized = sys.stdin.readline() - artifact = distbuild.deserialise_artifact(serialized) - + artifact_reference = distbuild.deserialise_artifact(serialized) + bc = morphlib.buildcommand.BuildCommand(self.app) + source_pool = bc.create_source_pool(artifact_reference.repo, + artifact_reference.ref, + artifact_reference.root_filename) + + root = bc.resolve_artifacts(source_pool) # Now, before we start the build, we garbage collect the caches # to ensure we have room. First we remove all system artifacts @@ -111,8 +117,21 @@ class WorkerBuild(cliapp.Plugin): self.app.subcommands['gc']([]) - arch = artifact.arch - bc.build_source(artifact.source, bc.new_build_env(arch)) + source = self.find_source(source_pool, artifact_reference) + build_env = bc.new_build_env(artifact_reference.arch) + bc.build_source(source, build_env) + + def find_source(self, source_pool, artifact_reference): + for s in source_pool.lookup(artifact_reference.source_repo, + artifact_reference.source_ref, + artifact_reference.filename): + if s.cache_key == artifact_reference.cache_key: + return s + for s in source_pool.lookup(artifact_reference.source_repo, + artifact_reference.source_sha1, + artifact_reference.filename): + if s.cache_key == artifact_reference.cache_key: + return s def is_system_artifact(self, filename): return re.match(r'^[0-9a-fA-F]{64}\.system\.', filename) @@ -180,7 +199,6 @@ class ControllerDaemon(cliapp.Plugin): self.app.settings.string( ['initiator-step-output-dir'], 'write build output to files in DIR', - default='.', group=group_distbuild) self.app.settings.string( @@ -238,6 +256,11 @@ class ControllerDaemon(cliapp.Plugin): distbuild.add_crash_conditions(self.app.settings['crash-condition']) + if not self.app.settings['worker']: + raise cliapp.AppException( + 'Distbuild controller has no workers configured. Refusing to ' + 'start.') + artifact_cache_server = ( self.app.settings['artifact-cache-server'] or self.app.settings['cache-server']) diff --git a/morphlib/plugins/expand_repo_plugin.py b/morphlib/plugins/expand_repo_plugin.py index 721287ca..bbee891d 100644 --- a/morphlib/plugins/expand_repo_plugin.py +++ b/morphlib/plugins/expand_repo_plugin.py @@ -1,4 +1,4 @@ -# Copyright (C) 2012-2013 Codethink Limited +# Copyright (C) 2012-2013,2015 Codethink Limited # # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by @@ -10,8 +10,7 @@ # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License along -# with this program; if not, write to the Free Software Foundation, Inc., -# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. +# with this program. If not, see <http://www.gnu.org/licenses/>. import cliapp diff --git a/morphlib/plugins/gc_plugin.py b/morphlib/plugins/gc_plugin.py index 68f386eb..71522b04 100644 --- a/morphlib/plugins/gc_plugin.py +++ b/morphlib/plugins/gc_plugin.py @@ -1,4 +1,4 @@ -# Copyright (C) 2013-2014 Codethink Limited +# Copyright (C) 2013-2015 Codethink Limited # # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by @@ -10,8 +10,7 @@ # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License along -# with this program; if not, write to the Free Software Foundation, Inc., -# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. +# with this program. If not, see <http://www.gnu.org/licenses/>. import logging diff --git a/morphlib/plugins/graphing_plugin.py b/morphlib/plugins/graphing_plugin.py index 57166e51..0db63ffc 100644 --- a/morphlib/plugins/graphing_plugin.py +++ b/morphlib/plugins/graphing_plugin.py @@ -1,4 +1,4 @@ -# Copyright (C) 2012, 2013 Codethink Limited +# Copyright (C) 2012, 2013,2015 Codethink Limited # # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by @@ -10,8 +10,7 @@ # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License along -# with this program; if not, write to the Free Software Foundation, Inc., -# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. +# with this program. If not, see <http://www.gnu.org/licenses/>. import cliapp diff --git a/morphlib/plugins/list_artifacts_plugin.py b/morphlib/plugins/list_artifacts_plugin.py index 0f14a579..b7a5e080 100644 --- a/morphlib/plugins/list_artifacts_plugin.py +++ b/morphlib/plugins/list_artifacts_plugin.py @@ -1,4 +1,4 @@ -# Copyright (C) 2014 Codethink Limited +# Copyright (C) 2014-2015 Codethink Limited # # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by @@ -10,13 +10,13 @@ # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License along -# with this program; if not, write to the Free Software Foundation, Inc., -# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. +# with this program. If not, see <http://www.gnu.org/licenses/>. # This plugin is used as part of the Baserock automated release process. # # See: <http://wiki.baserock.org/guides/release-process> for more information. +from __future__ import print_function import cliapp import morphlib @@ -68,7 +68,7 @@ class ListArtifactsPlugin(cliapp.Plugin): artifact_files.update(system_artifact_files) for artifact_file in sorted(artifact_files): - print artifact_file + print(artifact_file) def list_artifacts_for_system(self, repo, ref, system_filename): '''List all artifact files in the build graph of a single system.''' @@ -84,8 +84,11 @@ class ListArtifactsPlugin(cliapp.Plugin): self.app.status( msg='Creating source pool for %s' % system_filename, chatty=True) - source_pool = self.app.create_source_pool( - self.lrc, self.rrc, repo, ref, system_filename) + source_pool = morphlib.sourceresolver.create_source_pool( + self.lrc, self.rrc, repo, ref, system_filename, + cachedir=self.app.settings['cachedir'], + update_repos = not self.app.settings['no-git-update'], + status_cb=self.app.status) self.app.status( msg='Resolving artifacts for %s' % system_filename, chatty=True) diff --git a/morphlib/plugins/print_architecture_plugin.py b/morphlib/plugins/print_architecture_plugin.py index 08f500d0..264334d9 100644 --- a/morphlib/plugins/print_architecture_plugin.py +++ b/morphlib/plugins/print_architecture_plugin.py @@ -1,4 +1,4 @@ -# Copyright (C) 2013 Codethink Limited +# Copyright (C) 2013,2015 Codethink Limited # # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by @@ -10,8 +10,7 @@ # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License along -# with this program; if not, write to the Free Software Foundation, Inc., -# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. +# with this program. If not, see <http://www.gnu.org/licenses/>. import cliapp diff --git a/morphlib/plugins/push_pull_plugin.py b/morphlib/plugins/push_pull_plugin.py index 843de1a6..8546e2bf 100644 --- a/morphlib/plugins/push_pull_plugin.py +++ b/morphlib/plugins/push_pull_plugin.py @@ -1,4 +1,4 @@ -# Copyright (C) 2014 Codethink Limited +# Copyright (C) 2014-2015 Codethink Limited # # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by @@ -10,8 +10,7 @@ # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License along -# with this program; if not, write to the Free Software Foundation, Inc., -# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. +# with this program. If not, see <http://www.gnu.org/licenses/>. import cliapp import logging @@ -52,7 +51,7 @@ class PushPullPlugin(cliapp.Plugin): if len(args) != 2: raise morphlib.Error('push must get exactly two arguments') - gd = morphlib.gitdir.GitDirectory(os.getcwd()) + gd = morphlib.gitdir.GitDirectory(os.getcwd(), search_for_root=True) remote, branch = args rs = morphlib.gitdir.RefSpec(branch) gd.get_remote(remote).push(rs) @@ -81,7 +80,7 @@ class PushPullPlugin(cliapp.Plugin): if len(args) > 1: raise morphlib.Error('pull takes at most one argument') - gd = morphlib.gitdir.GitDirectory(os.getcwd()) + gd = morphlib.gitdir.GitDirectory(os.getcwd(), search_for_root=True) remote = gd.get_remote('origin') if args: branch = args[0] diff --git a/morphlib/plugins/show_dependencies_plugin.py b/morphlib/plugins/show_dependencies_plugin.py index e70f6bfb..ee96cb40 100644 --- a/morphlib/plugins/show_dependencies_plugin.py +++ b/morphlib/plugins/show_dependencies_plugin.py @@ -1,4 +1,4 @@ -# Copyright (C) 2012-2014 Codethink Limited +# Copyright (C) 2012-2015 Codethink Limited # # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by @@ -10,8 +10,7 @@ # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License along -# with this program; if not, write to the Free Software Foundation, Inc., -# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. +# with this program. If not, see <http://www.gnu.org/licenses/>. import cliapp diff --git a/morphlib/plugins/trovectl_plugin.py b/morphlib/plugins/trovectl_plugin.py index 80f3b4cf..ea780b65 100644 --- a/morphlib/plugins/trovectl_plugin.py +++ b/morphlib/plugins/trovectl_plugin.py @@ -1,4 +1,4 @@ -# Copyright (C) 2013 Codethink Limited +# Copyright (C) 2013,2015 Codethink Limited # # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by @@ -10,8 +10,7 @@ # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License along -# with this program; if not, write to the Free Software Foundation, Inc., -# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. +# with this program. If not, see <http://www.gnu.org/licenses/>. import cliapp diff --git a/morphlib/pylru.py b/morphlib/pylru.py new file mode 100644 index 00000000..28d55d50 --- /dev/null +++ b/morphlib/pylru.py @@ -0,0 +1,532 @@ + +# Cache implementaion with a Least Recently Used (LRU) replacement policy and +# a basic dictionary interface. + +# Copyright (C) 2006, 2009, 2010, 2011 Jay Hutchinson + +# This program is free software; you can redistribute it and/or modify it +# under the terms of the GNU General Public License as published by the Free +# Software Foundation; either version 2 of the License, or (at your option) +# any later version. + +# This program is distributed in the hope that it will be useful, but WITHOUT +# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or +# FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for +# more details. + +# You should have received a copy of the GNU General Public License along +# with this program; if not, write to the Free Software Foundation, Inc., 51 +# Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. + + + +# The cache is implemented using a combination of a python dictionary (hash +# table) and a circular doubly linked list. Items in the cache are stored in +# nodes. These nodes make up the linked list. The list is used to efficiently +# maintain the order that the items have been used in. The front or head of +# the list contains the most recently used item, the tail of the list +# contains the least recently used item. When an item is used it can easily +# (in a constant amount of time) be moved to the front of the list, thus +# updating its position in the ordering. These nodes are also placed in the +# hash table under their associated key. The hash table allows efficient +# lookup of values by key. + +# Class for the node objects. +class _dlnode(object): + def __init__(self): + self.empty = True + + +class lrucache(object): + + def __init__(self, size, callback=None): + + self.callback = callback + + # Create an empty hash table. + self.table = {} + + # Initialize the doubly linked list with one empty node. This is an + # invariant. The cache size must always be greater than zero. Each + # node has a 'prev' and 'next' variable to hold the node that comes + # before it and after it respectively. Initially the two variables + # each point to the head node itself, creating a circular doubly + # linked list of size one. Then the size() method is used to adjust + # the list to the desired size. + + self.head = _dlnode() + self.head.next = self.head + self.head.prev = self.head + + self.listSize = 1 + + # Adjust the size + self.size(size) + + + def __len__(self): + return len(self.table) + + def clear(self): + for node in self.dli(): + node.empty = True + node.key = None + node.value = None + + self.table.clear() + + + def __contains__(self, key): + return key in self.table + + # Looks up a value in the cache without affecting cache order. + def peek(self, key): + # Look up the node + node = self.table[key] + return node.value + + + def __getitem__(self, key): + # Look up the node + node = self.table[key] + + # Update the list ordering. Move this node so that is directly + # proceeds the head node. Then set the 'head' variable to it. This + # makes it the new head of the list. + self.mtf(node) + self.head = node + + # Return the value. + return node.value + + def get(self, key, default=None): + """Get an item - return default (None) if not present""" + try: + return self[key] + except KeyError: + return default + + def __setitem__(self, key, value): + # First, see if any value is stored under 'key' in the cache already. + # If so we are going to replace that value with the new one. + if key in self.table: + + # Lookup the node + node = self.table[key] + + # Replace the value. + node.value = value + + # Update the list ordering. + self.mtf(node) + self.head = node + + return + + # Ok, no value is currently stored under 'key' in the cache. We need + # to choose a node to place the new item in. There are two cases. If + # the cache is full some item will have to be pushed out of the + # cache. We want to choose the node with the least recently used + # item. This is the node at the tail of the list. If the cache is not + # full we want to choose a node that is empty. Because of the way the + # list is managed, the empty nodes are always together at the tail + # end of the list. Thus, in either case, by chooseing the node at the + # tail of the list our conditions are satisfied. + + # Since the list is circular, the tail node directly preceeds the + # 'head' node. + node = self.head.prev + + # If the node already contains something we need to remove the old + # key from the dictionary. + if not node.empty: + if self.callback is not None: + self.callback(node.key, node.value) + del self.table[node.key] + + # Place the new key and value in the node + node.empty = False + node.key = key + node.value = value + + # Add the node to the dictionary under the new key. + self.table[key] = node + + # We need to move the node to the head of the list. The node is the + # tail node, so it directly preceeds the head node due to the list + # being circular. Therefore, the ordering is already correct, we just + # need to adjust the 'head' variable. + self.head = node + + + def __delitem__(self, key): + + # Lookup the node, then remove it from the hash table. + node = self.table[key] + del self.table[key] + + node.empty = True + + # Not strictly necessary. + node.key = None + node.value = None + + # Because this node is now empty we want to reuse it before any + # non-empty node. To do that we want to move it to the tail of the + # list. We move it so that it directly preceeds the 'head' node. This + # makes it the tail node. The 'head' is then adjusted. This + # adjustment ensures correctness even for the case where the 'node' + # is the 'head' node. + self.mtf(node) + self.head = node.next + + def __iter__(self): + + # Return an iterator that returns the keys in the cache in order from + # the most recently to least recently used. Does not modify the cache + # order. + for node in self.dli(): + yield node.key + + def items(self): + + # Return an iterator that returns the (key, value) pairs in the cache + # in order from the most recently to least recently used. Does not + # modify the cache order. + for node in self.dli(): + yield (node.key, node.value) + + def keys(self): + + # Return an iterator that returns the keys in the cache in order from + # the most recently to least recently used. Does not modify the cache + # order. + for node in self.dli(): + yield node.key + + def values(self): + + # Return an iterator that returns the values in the cache in order + # from the most recently to least recently used. Does not modify the + # cache order. + for node in self.dli(): + yield node.value + + def size(self, size=None): + + if size is not None: + assert size > 0 + if size > self.listSize: + self.addTailNode(size - self.listSize) + elif size < self.listSize: + self.removeTailNode(self.listSize - size) + + return self.listSize + + # Increases the size of the cache by inserting n empty nodes at the tail + # of the list. + def addTailNode(self, n): + for i in range(n): + node = _dlnode() + node.next = self.head + node.prev = self.head.prev + + self.head.prev.next = node + self.head.prev = node + + self.listSize += n + + # Decreases the size of the list by removing n nodes from the tail of the + # list. + def removeTailNode(self, n): + assert self.listSize > n + for i in range(n): + node = self.head.prev + if not node.empty: + if self.callback is not None: + self.callback(node.key, node.value) + del self.table[node.key] + + # Splice the tail node out of the list + self.head.prev = node.prev + node.prev.next = self.head + + # The next four lines are not strictly necessary. + node.prev = None + node.next = None + + node.key = None + node.value = None + + self.listSize -= n + + + # This method adjusts the ordering of the doubly linked list so that + # 'node' directly precedes the 'head' node. Because of the order of + # operations, if 'node' already directly precedes the 'head' node or if + # 'node' is the 'head' node the order of the list will be unchanged. + def mtf(self, node): + node.prev.next = node.next + node.next.prev = node.prev + + node.prev = self.head.prev + node.next = self.head.prev.next + + node.next.prev = node + node.prev.next = node + + # This method returns an iterator that iterates over the non-empty nodes + # in the doubly linked list in order from the most recently to the least + # recently used. + def dli(self): + node = self.head + for i in range(len(self.table)): + yield node + node = node.next + + + + +class WriteThroughCacheManager(object): + def __init__(self, store, size): + self.store = store + self.cache = lrucache(size) + + def __len__(self): + return len(self.store) + + # Returns/sets the size of the managed cache. + def size(self, size=None): + return self.cache.size(size) + + def clear(self): + self.cache.clear() + self.store.clear() + + def __contains__(self, key): + # Check the cache first. If it is there we can return quickly. + if key in self.cache: + return True + + # Not in the cache. Might be in the underlying store. + if key in self.store: + return True + + return False + + def __getitem__(self, key): + # First we try the cache. If successful we just return the value. If + # not we catch KeyError and ignore it since that just means the key + # was not in the cache. + try: + return self.cache[key] + except KeyError: + pass + + # It wasn't in the cache. Look it up in the store, add the entry to + # the cache, and return the value. + value = self.store[key] + self.cache[key] = value + return value + + def get(self, key, default=None): + """Get an item - return default (None) if not present""" + try: + return self[key] + except KeyError: + return default + + def __setitem__(self, key, value): + # Add the key/value pair to the cache and store. + self.cache[key] = value + self.store[key] = value + + def __delitem__(self, key): + # Write-through behavior cache and store should be consistent. Delete + # it from the store. + del self.store[key] + try: + # Ok, delete from the store was successful. It might also be in + # the cache, try and delete it. If not we catch the KeyError and + # ignore it. + del self.cache[key] + except KeyError: + pass + + def __iter__(self): + return self.keys() + + def keys(self): + return self.store.keys() + + def values(self): + return self.store.values() + + def items(self): + return self.store.items() + + + +class WriteBackCacheManager(object): + def __init__(self, store, size): + self.store = store + + # Create a set to hold the dirty keys. + self.dirty = set() + + # Define a callback function to be called by the cache when a + # key/value pair is about to be ejected. This callback will check to + # see if the key is in the dirty set. If so, then it will update the + # store object and remove the key from the dirty set. + def callback(key, value): + if key in self.dirty: + self.store[key] = value + self.dirty.remove(key) + + # Create a cache and give it the callback function. + self.cache = lrucache(size, callback) + + # Returns/sets the size of the managed cache. + def size(self, size=None): + return self.cache.size(size) + + def clear(self): + self.cache.clear() + self.dirty.clear() + self.store.clear() + + def __contains__(self, key): + # Check the cache first, since if it is there we can return quickly. + if key in self.cache: + return True + + # Not in the cache. Might be in the underlying store. + if key in self.store: + return True + + return False + + def __getitem__(self, key): + # First we try the cache. If successful we just return the value. If + # not we catch KeyError and ignore it since that just means the key + # was not in the cache. + try: + return self.cache[key] + except KeyError: + pass + + # It wasn't in the cache. Look it up in the store, add the entry to + # the cache, and return the value. + value = self.store[key] + self.cache[key] = value + return value + + def get(self, key, default=None): + """Get an item - return default (None) if not present""" + try: + return self[key] + except KeyError: + return default + + def __setitem__(self, key, value): + # Add the key/value pair to the cache. + self.cache[key] = value + self.dirty.add(key) + + def __delitem__(self, key): + + found = False + try: + del self.cache[key] + found = True + self.dirty.remove(key) + except KeyError: + pass + + try: + del self.store[key] + found = True + except KeyError: + pass + + if not found: # If not found in cache or store, raise error. + raise KeyError + + + def __iter__(self): + return self.keys() + + def keys(self): + for key in self.store.keys(): + if key not in self.dirty: + yield key + + for key in self.dirty: + yield key + + + def values(self): + for key, value in self.items(): + yield value + + + def items(self): + for key, value in self.store.items(): + if key not in self.dirty: + yield (key, value) + + for key in self.dirty: + value = self.cache.peek(key) + yield (key, value) + + + + def sync(self): + # For each dirty key, peek at its value in the cache and update the + # store. Doesn't change the cache's order. + for key in self.dirty: + self.store[key] = self.cache.peek(key) + # There are no dirty keys now. + self.dirty.clear() + + def flush(self): + self.sync() + self.cache.clear() + + def __enter__(self): + return self + + def __exit__(self, exc_type, exc_val, exc_tb): + self.sync() + return False + + + + + +def lruwrap(store, size, writeback=False): + if writeback: + return WriteBackCacheManager(store, size) + else: + return WriteThroughCacheManager(store, size) + + + + +class lrudecorator(object): + def __init__(self, size): + self.cache = lrucache(size) + + def __call__(self, func): + def wrapped(*args, **kwargs): + kwtuple = tuple((key, kwargs[key]) for key in sorted(kwargs.keys())) + key = (args, kwtuple) + try: + return self.cache[key] + except KeyError: + pass + + value = func(*args, **kwargs) + self.cache[key] = value + return value + return wrapped diff --git a/morphlib/recv-hole b/morphlib/recv-hole index d6504bf6..fe69f304 100755 --- a/morphlib/recv-hole +++ b/morphlib/recv-hole @@ -1,6 +1,6 @@ #!/bin/sh # -# Copyright (C) 2014 Codethink Limited +# Copyright (C) 2014-2015 Codethink Limited # # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by @@ -12,8 +12,7 @@ # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License along -# with this program; if not, write to the Free Software Foundation, Inc., -# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. +# with this program. If not, see <http://www.gnu.org/licenses/>. # # =*= License: GPL-2 =*= diff --git a/morphlib/remoteartifactcache.py b/morphlib/remoteartifactcache.py index 4e09ce34..427e4cbb 100644 --- a/morphlib/remoteartifactcache.py +++ b/morphlib/remoteartifactcache.py @@ -1,4 +1,4 @@ -# Copyright (C) 2012-2014 Codethink Limited +# Copyright (C) 2012-2015 Codethink Limited # # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by @@ -10,8 +10,7 @@ # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License along -# with this program; if not, write to the Free Software Foundation, Inc., -# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. +# with this program. If not, see <http://www.gnu.org/licenses/>. import cliapp @@ -72,14 +71,14 @@ class RemoteArtifactCache(object): def get(self, artifact, log=logging.error): try: return self._get_file(artifact.basename()) - except urllib2.URLError, e: + except urllib2.URLError as e: log(str(e)) raise GetError(self, artifact) def get_artifact_metadata(self, artifact, name, log=logging.error): try: return self._get_file(artifact.metadata_basename(name)) - except urllib2.URLError, e: + except urllib2.URLError as e: log(str(e)) raise GetArtifactMetadataError(self, artifact, name) diff --git a/morphlib/remoteartifactcache_tests.py b/morphlib/remoteartifactcache_tests.py index 788882c2..18bef13f 100644 --- a/morphlib/remoteartifactcache_tests.py +++ b/morphlib/remoteartifactcache_tests.py @@ -1,4 +1,4 @@ -# Copyright (C) 2012-2014 Codethink Limited +# Copyright (C) 2012-2015 Codethink Limited # # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by @@ -10,8 +10,7 @@ # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License along -# with this program; if not, write to the Free Software Foundation, Inc., -# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. +# with this program. If not, see <http://www.gnu.org/licenses/>. import StringIO diff --git a/morphlib/remoterepocache.py b/morphlib/remoterepocache.py index 004ba86e..4a6d9fe9 100644 --- a/morphlib/remoterepocache.py +++ b/morphlib/remoterepocache.py @@ -1,4 +1,4 @@ -# Copyright (C) 2012-2014 Codethink Limited +# Copyright (C) 2012-2015 Codethink Limited # # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by @@ -10,8 +10,7 @@ # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License along -# with this program; if not, write to the Free Software Foundation, Inc., -# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. +# with this program. If not, see <http://www.gnu.org/licenses/>. import cliapp @@ -55,7 +54,7 @@ class RemoteRepoCache(object): repo_url = self._resolver.pull_url(repo_name) try: return self._resolve_ref_for_repo_url(repo_url, ref) - except BaseException, e: + except BaseException as e: logging.error('Caught exception: %s' % str(e)) raise ResolveRefError(repo_name, ref) @@ -74,7 +73,7 @@ class RemoteRepoCache(object): try: info = json.loads(self._ls_tree_for_repo_url(repo_url, ref)) return info['tree'].keys() - except BaseException, e: + except BaseException as e: logging.error('Caught exception: %s' % str(e)) raise LsTreeError(repo_name, ref) diff --git a/morphlib/remoterepocache_tests.py b/morphlib/remoterepocache_tests.py index ef81506f..966e74d5 100644 --- a/morphlib/remoterepocache_tests.py +++ b/morphlib/remoterepocache_tests.py @@ -1,4 +1,4 @@ -# Copyright (C) 2012-2014 Codethink Limited +# Copyright (C) 2012-2015 Codethink Limited # # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by @@ -10,8 +10,7 @@ # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License along -# with this program; if not, write to the Free Software Foundation, Inc., -# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. +# with this program. If not, see <http://www.gnu.org/licenses/>. import json diff --git a/morphlib/repoaliasresolver.py b/morphlib/repoaliasresolver.py index bc759dd4..de57628c 100644 --- a/morphlib/repoaliasresolver.py +++ b/morphlib/repoaliasresolver.py @@ -1,4 +1,4 @@ -# Copyright (C) 2012-2013 Codethink Limited +# Copyright (C) 2012-2013,2015 Codethink Limited # # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by @@ -10,8 +10,7 @@ # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License along -# with this program; if not, write to the Free Software Foundation, Inc., -# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. +# with this program. If not, see <http://www.gnu.org/licenses/>. import logging diff --git a/morphlib/repoaliasresolver_tests.py b/morphlib/repoaliasresolver_tests.py index c4ea16b0..a10c4017 100644 --- a/morphlib/repoaliasresolver_tests.py +++ b/morphlib/repoaliasresolver_tests.py @@ -1,4 +1,4 @@ -# Copyright (C) 2012-2013 Codethink Limited +# Copyright (C) 2012-2013,2015 Codethink Limited # # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by @@ -10,8 +10,7 @@ # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License along -# with this program; if not, write to the Free Software Foundation, Inc., -# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. +# with this program. If not, see <http://www.gnu.org/licenses/>. import morphlib diff --git a/morphlib/savefile.py b/morphlib/savefile.py index 2d87a54f..7deb2b89 100644 --- a/morphlib/savefile.py +++ b/morphlib/savefile.py @@ -1,4 +1,4 @@ -# Copyright (C) 2012 Codethink Limited +# Copyright (C) 2012,2015 Codethink Limited # # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by @@ -10,8 +10,7 @@ # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License along -# with this program; if not, write to the Free Software Foundation, Inc., -# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. +# with this program. If not, see <http://www.gnu.org/licenses/>. import logging diff --git a/morphlib/savefile_tests.py b/morphlib/savefile_tests.py index 7ae2360d..dc11086c 100644 --- a/morphlib/savefile_tests.py +++ b/morphlib/savefile_tests.py @@ -1,4 +1,4 @@ -# Copyright (C) 2012 Codethink Limited +# Copyright (C) 2012,2015 Codethink Limited # # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by @@ -10,8 +10,7 @@ # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License along -# with this program; if not, write to the Free Software Foundation, Inc., -# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. +# with this program. If not, see <http://www.gnu.org/licenses/>. import os diff --git a/morphlib/source.py b/morphlib/source.py index 4ad54ed9..2c96bcbd 100644 --- a/morphlib/source.py +++ b/morphlib/source.py @@ -1,4 +1,4 @@ -# Copyright (C) 2012-2014 Codethink Limited +# Copyright (C) 2012-2015 Codethink Limited # # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by @@ -10,8 +10,7 @@ # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License along -# with this program; if not, write to the Free Software Foundation, Inc., -# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. +# with this program. If not, see <http://www.gnu.org/licenses/>. import morphlib diff --git a/morphlib/source_tests.py b/morphlib/source_tests.py index 695041d3..4f6006e9 100644 --- a/morphlib/source_tests.py +++ b/morphlib/source_tests.py @@ -1,4 +1,4 @@ -# Copyright (C) 2012-2014 Codethink Limited +# Copyright (C) 2012-2015 Codethink Limited # # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by @@ -10,8 +10,7 @@ # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License along -# with this program; if not, write to the Free Software Foundation, Inc., -# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. +# with this program. If not, see <http://www.gnu.org/licenses/>. import unittest diff --git a/morphlib/sourcepool.py b/morphlib/sourcepool.py index 6dfcb2c3..b340a9de 100644 --- a/morphlib/sourcepool.py +++ b/morphlib/sourcepool.py @@ -1,4 +1,4 @@ -# Copyright (C) 2012-2014 Codethink Limited +# Copyright (C) 2012-2015 Codethink Limited # # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by @@ -10,8 +10,7 @@ # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License along -# with this program; if not, write to the Free Software Foundation, Inc., -# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. +# with this program. If not, see <http://www.gnu.org/licenses/>. import collections diff --git a/morphlib/sourcepool_tests.py b/morphlib/sourcepool_tests.py index f3740049..85833901 100644 --- a/morphlib/sourcepool_tests.py +++ b/morphlib/sourcepool_tests.py @@ -1,4 +1,4 @@ -# Copyright (C) 2012-2014 Codethink Limited +# Copyright (C) 2012-2015 Codethink Limited # # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by @@ -10,8 +10,7 @@ # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License along -# with this program; if not, write to the Free Software Foundation, Inc., -# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. +# with this program. If not, see <http://www.gnu.org/licenses/>. import unittest diff --git a/morphlib/sourceresolver.py b/morphlib/sourceresolver.py new file mode 100644 index 00000000..d2b47d35 --- /dev/null +++ b/morphlib/sourceresolver.py @@ -0,0 +1,584 @@ +# Copyright (C) 2014-2015 Codethink Limited +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation; version 2 of the License. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License along +# with this program. If not, see <http://www.gnu.org/licenses/>. + + +import collections +import cPickle +import logging +import os +import pylru +import shutil +import tempfile +import yaml + +import cliapp + +import morphlib + +tree_cache_size = 10000 +tree_cache_filename = 'trees.cache.pickle' +buildsystem_cache_size = 10000 +buildsystem_cache_filename = 'detected-chunk-buildsystems.cache.pickle' + +not_supported_versions = [] + +class PickleCacheManager(object): # pragma: no cover + '''Cache manager for PyLRU that reads and writes to Pickle files. + + The 'pickle' format is less than ideal in many ways and is actually + slower than JSON in Python. However, the data we need to cache is keyed + by tuples and in JSON a dict can only be keyed with strings. For now, + using 'pickle' seems to be the least worst option. + + ''' + + def __init__(self, filename, size): + self.filename = filename + self.size = size + + def _populate_cache_from_file(self, filename, cache): + try: + with open(filename, 'r') as f: + data = cPickle.load(f) + for key, value in data.iteritems(): + cache[key] = value + except (EOFError, IOError, cPickle.PickleError) as e: + logging.warning('Failed to load cache %s: %s', self.filename, e) + + def load_cache(self): + '''Create a pylru.lrucache object prepopulated with saved data.''' + cache = pylru.lrucache(self.size) + # There should be a more efficient way to do this, by hooking into + # the json module directly. + self._populate_cache_from_file(self.filename, cache) + return cache + + def save_cache(self, cache): + '''Save the data from a pylru.lrucache object to disk. + + Any changes that have been made by other instances or processes since + load_cache() was called will be overwritten. + + ''' + data = {} + for key, value in cache.items(): + data[key] = value + try: + with morphlib.savefile.SaveFile(self.filename, 'w') as f: + cPickle.dump(data, f) + except (IOError, cPickle.PickleError) as e: + logging.warning('Failed to save cache to %s: %s', self.filename, e) + + +class SourceResolverError(cliapp.AppException): + pass + + +class MorphologyNotFoundError(SourceResolverError): # pragma: no cover + def __init__(self, filename): + SourceResolverError.__init__( + self, "Couldn't find morphology: %s" % filename) + +class UnknownVersionError(SourceResolverError): # pragma: no cover + def __init__(self, version): + SourceResolverError.__init__( + self, "Definitions format version %s is not supported" % version) + + +class SourceResolver(object): + '''Provides a way of resolving the set of sources for a given system. + + There are three levels of caching involved in resolving the sources to + build. + + The canonical repo for each source is specified in the build-command + (for strata and systems) or in the stratum morphology (for chunks). It will + be either a normal URL, or a keyed URL using a repo-alias like + 'baserock:baserock/definitions'. + + The 'remote repo cache' is a Baserock Trove system. It functions as a + normal Git server, but in addition it runs a service on port 8080 called + 'morph-cache-server' which can resolve refs, list their contents and read + specific files from the repos it holds. This allows the SourceResolver to + work out how to build something without cloning the whole repo. (If a local + build of that source ends up being necessary then it will get cloned into + the local cache later on). + + The second layer of caching is the local repository cache, which mirrors + entire repositories in $cachedir/gits. If a repo is not in the remote repo + cache then it must be present in the local repo cache. + + The third layer of caching is a simple commit SHA1 -> tree SHA mapping. It + turns out that even if all repos are available locally, running + 'git rev-parse' on hundreds of repos requires a lot of IO and can take + several minutes. Likewise, on a slow network connection it is time + consuming to keep querying the remote repo cache. This third layer of + caching works around both of those issues. + + The need for 3 levels of caching highlights design inconsistencies in + Baserock, but for now it is worth the effort to maintain this code to save + users from waiting 7 minutes each time that they want to build. The level 3 + cache is fairly simple because commits are immutable, so there is no danger + of this cache being stale as long as it is indexed by commit SHA1. Due to + the policy in Baserock of always using a commit SHA1 (rather than a named + ref) in the system definitions, it makes repeated builds of a system very + fast as no resolution needs to be done at all. + + ''' + + def __init__(self, local_repo_cache, remote_repo_cache, + tree_cache_manager, buildsystem_cache_manager, update_repos, + status_cb=None): + self.lrc = local_repo_cache + self.rrc = remote_repo_cache + self.tree_cache_manager = tree_cache_manager + self.buildsystem_cache_manager = buildsystem_cache_manager + + self.update = update_repos + self.status = status_cb + + self._resolved_trees = {} + self._resolved_morphologies = {} + self._resolved_buildsystems = {} + + self._definitions_checkout_dir = None + + def cache_repo_locally(self, reponame): + if self.update: + self.status(msg='Caching git repository %(reponame)s', + reponame=reponame) + repo = self.lrc.cache_repo(reponame) + else: # pragma: no cover + # This is likely to raise a morphlib.localrepocache.NotCached + # exception, because the caller should have checked if the + # localrepocache already had the repo. But we may as well try. + repo = self.lrc.get_repo(reponame) + return repo + + def _resolve_ref(self, reponame, ref): # pragma: no cover + '''Resolves commit and tree sha1s of the ref in a repo and returns it. + + If update is True then this has the side-effect of updating or cloning + the repository into the local repo cache. + + This function is complex due to the 3 layers of caching described in + the SourceResolver docstring. + + ''' + + # The Baserock reference definitions use absolute refs so, and, if the + # absref is cached, we can short-circuit all this code. + if (reponame, ref) in self._resolved_trees: + logging.debug('Returning tree (%s, %s) from tree cache', + reponame, ref) + return ref, self._resolved_trees[(reponame, ref)] + + logging.debug('tree (%s, %s) not in cache', reponame, ref) + + absref = None + if self.lrc.has_repo(reponame): + repo = self.lrc.get_repo(reponame) + if self.update and repo.requires_update_for_ref(ref): + self.status(msg='Updating cached git repository %(reponame)s ' + 'for ref %(ref)s', reponame=reponame, ref=ref) + repo.update() + # If the user passed --no-git-update, and the ref is a SHA1 not + # available locally, this call will raise an exception. + absref = repo.resolve_ref_to_commit(ref) + tree = repo.resolve_ref_to_tree(absref) + elif self.rrc is not None: + try: + absref, tree = self.rrc.resolve_ref(reponame, ref) + if absref is not None: + self.status(msg='Resolved %(reponame)s %(ref)s via remote ' + 'repo cache', + reponame=reponame, + ref=ref, + chatty=True) + except BaseException as e: + logging.warning('Caught (and ignored) exception: %s' % str(e)) + + if absref is None: + repo = self.cache_repo_locally(reponame) + absref = repo.resolve_ref_to_commit(ref) + tree = repo.resolve_ref_to_tree(absref) + + logging.debug('Writing tree to cache with ref (%s, %s)', + reponame, absref) + self._resolved_trees[(reponame, absref)] = tree + + return absref, tree + + def _get_file_contents_from_definitions(self, + filename): # pragma: no cover + if os.path.exists(filename): + with open(filename) as f: + return f.read() + else: + return None + + def _get_file_contents_from_repo(self, reponame, + sha1, filename): # pragma: no cover + if self.lrc.has_repo(reponame): + self.status(msg="Looking for %(reponame)s:%(filename)s in the " + "local repo cache.", + reponame=reponame, filename=filename, chatty=True) + try: + repo = self.lrc.get_repo(reponame) + text = repo.read_file(filename, sha1) + except IOError: + text = None + elif self.rrc is not None: + self.status(msg="Looking for %(reponame)s:%(filename)s in the " + "remote repo cache.", + reponame=reponame, filename=filename, chatty=True) + try: + text = self.rrc.cat_file(reponame, sha1, filename) + except morphlib.remoterepocache.CatFileError: + text = None + else: # pragma: no cover + repo = self.cache_repo_locally(reponame) + text = repo.read_file(filename, sha1) + + return text + + def _get_file_contents(self, reponame, sha1, filename): # pragma: no cover + '''Read the file at the specified location. + + Returns None if the file does not exist in the specified commit. + + ''' + text = None + + if reponame == self._definitions_repo and \ + sha1 == self._definitions_absref: # pragma: no cover + # There is a temporary local checkout of the definitions repo which + # we can quickly read definitions files from. + defs_filename = os.path.join(self._definitions_checkout_dir, + filename) + text = self._get_file_contents_from_definitions(defs_filename) + else: + text = self._get_file_contents_from_repo(reponame, sha1, filename) + + return text + + def _get_morphology(self, reponame, sha1, filename): # pragma: no cover + '''Read the morphology at the specified location. + + Returns None if the file does not exist in the specified commit. + + ''' + key = (reponame, sha1, filename) + if key in self._resolved_morphologies: + return self._resolved_morphologies[key] + + loader = morphlib.morphloader.MorphologyLoader() + + text = self._get_file_contents(reponame, sha1, filename) + morph = loader.load_from_string(text) + + if morph is not None: + self._resolved_morphologies[key] = morph + + return morph + + def _detect_build_system(self, reponame, sha1, expected_filename): + '''Attempt to detect buildsystem of the given commit. + + Returns None if no known build system was detected. + + ''' + self.status(msg="File %s doesn't exist: attempting to infer " + "chunk morph from repo's build system" % + expected_filename, chatty=True) + + file_list = None + + if self.lrc.has_repo(reponame): + repo = self.lrc.get_repo(reponame) + try: + file_list = repo.list_files(ref=sha1, recurse=False) + except morphlib.gitdir.InvalidRefError: # pragma: no cover + pass + elif self.rrc is not None: + try: + # This may or may not succeed; if the is repo not + # hosted on the same Git server as the cache server then + # it'll definitely fail. + file_list = self.rrc.ls_tree(reponame, sha1) + except morphlib.remoterepocache.LsTreeError: + pass + + if not file_list: + repo = self.cache_repo_locally(reponame) + file_list = repo.list_files(ref=sha1, recurse=False) + + buildsystem = morphlib.buildsystem.detect_build_system(file_list) + + if buildsystem is None: + # It might surprise you to discover that if we can't autodetect a + # build system, we raise MorphologyNotFoundError. Users are + # required to provide a morphology for any chunk where Morph can't + # infer the build instructions automatically, so this is the right + # error. + raise MorphologyNotFoundError(expected_filename) + + return buildsystem.name + + def _create_morphology_for_build_system(self, buildsystem_name, + morph_name): # pragma: no cover + bs = morphlib.buildsystem.lookup_build_system(buildsystem_name) + loader = morphlib.morphloader.MorphologyLoader() + morph = bs.get_morphology(morph_name) + loader.validate(morph) + loader.set_commands(morph) + loader.set_defaults(morph) + return morph + + def _check_version_file(self,definitions_repo, + definitions_absref): # pragma: no cover + version_file = self._get_file_contents( + definitions_repo, definitions_absref, 'VERSION') + + if version_file is None: + return + + try: + version = yaml.safe_load(version_file)['version'] + except (yaml.error.YAMLError, KeyError, TypeError): + version = 0 + + if version in not_supported_versions: + raise UnknownVersionError(version) + + def _process_definitions_with_children(self, system_filenames, + definitions_repo, + definitions_ref, + definitions_absref, + definitions_tree, + visit): # pragma: no cover + definitions_queue = collections.deque(system_filenames) + chunk_queue = set() + + self._check_version_file(definitions_repo, definitions_absref) + + while definitions_queue: + filename = definitions_queue.popleft() + + morphology = self._get_morphology( + definitions_repo, definitions_absref, filename) + + if morphology is None: + raise MorphologyNotFoundError(filename) + + visit(definitions_repo, definitions_ref, filename, + definitions_absref, definitions_tree, morphology) + + if morphology['kind'] == 'cluster': + raise cliapp.AppException( + "Cannot build a morphology of type 'cluster'.") + elif morphology['kind'] == 'system': + definitions_queue.extend( + morphlib.util.sanitise_morphology_path(s['morph']) + for s in morphology['strata']) + elif morphology['kind'] == 'stratum': + if morphology['build-depends']: + definitions_queue.extend( + morphlib.util.sanitise_morphology_path(s['morph']) + for s in morphology['build-depends']) + for c in morphology['chunks']: + if 'morph' not in c: + # Autodetect a path if one is not given. This is to + # support the deprecated approach of putting the chunk + # .morph file in the toplevel directory of the chunk + # repo, instead of putting it in the definitions.git + # repo. + # + # All users should be specifying a full path to the + # chunk morph file, using the 'morph' field, and this + # code path should be removed. + path = morphlib.util.sanitise_morphology_path( + c.get('morph', c['name'])) + chunk_queue.add((c['repo'], c['ref'], path)) + else: + chunk_queue.add((c['repo'], c['ref'], c['morph'])) + + return chunk_queue + + def _generate_morph_and_cache_buildsystem(self, + definition_key, chunk_key, + buildsystem, + morph_name): # pragma: no cover + logging.debug('Caching build system for chunk with key %s', chunk_key) + + self._resolved_buildsystems[chunk_key] = buildsystem + + morphology = self._create_morphology_for_build_system(buildsystem, + morph_name) + self._resolved_morphologies[definition_key] = morphology + return morphology + + def process_chunk(self, definition_repo, definition_ref, chunk_repo, + chunk_ref, filename, visit): # pragma: no cover + absref = None + tree = None + chunk_key = None + buildsystem = None + + morph_name = os.path.splitext(os.path.basename(filename))[0] + + # Get morphology from definitions repo + definition_key = (definition_repo, definition_ref, filename) + morphology = self._get_morphology(*definition_key) + + if morphology: + absref, tree = self._resolve_ref(chunk_repo, chunk_ref) + visit(chunk_repo, chunk_ref, filename, absref, tree, morphology) + return + + absref, tree = self._resolve_ref(chunk_repo, chunk_ref) + chunk_key = (chunk_repo, absref, filename) + + if chunk_key in self._resolved_buildsystems: + logging.debug('Build system for %s is cached', str(chunk_key)) + self.status(msg='Build system for %(chunk)s is cached', + chunk=str(chunk_key), + chatty=True) + buildsystem = self._resolved_buildsystems[chunk_key] + + # If the build system for this chunk is cached then: + # * the chunk does not have a chunk morph + # (so we don't need to look for one) + # + # * a suitable (generated) morphology may already be cached. + # + # If the morphology is not already cached we can generate it + # from the build-system and cache it. + if definition_key in self._resolved_morphologies: + morphology = self._resolved_morphologies[definition_key] + else: + morphology = self._generate_morph_and_cache_buildsystem( + definition_key, chunk_key, buildsystem, morph_name) + else: + logging.debug('Build system for %s is NOT cached', str(chunk_key)) + # build-system not cached, look for morphology in chunk repo + # this can be slow (we may need to clone the repo from a remote) + morphology = self._get_morphology(*chunk_key) + + if morphology != None: + self._resolved_morphologies[definition_key] = morphology + else: + # This chunk doesn't have a chunk morph + buildsystem = self._detect_build_system(*chunk_key) + + if buildsystem is None: + raise MorphologyNotFoundError(filename) + else: + morphology = self._generate_morph_and_cache_buildsystem( + definition_key, chunk_key, buildsystem, morph_name) + + visit(chunk_repo, chunk_ref, filename, absref, tree, morphology) + + def traverse_morphs(self, definitions_repo, definitions_ref, + system_filenames, + visit=lambda rn, rf, fn, arf, m: None, + definitions_original_ref=None): # pragma: no cover + self._resolved_trees = self.tree_cache_manager.load_cache() + self._resolved_buildsystems = \ + self.buildsystem_cache_manager.load_cache() + + # Resolve the (repo, ref) pair for the definitions repo, cache result. + definitions_absref, definitions_tree = self._resolve_ref( + definitions_repo, definitions_ref) + + if definitions_original_ref: + definitions_ref = definitions_original_ref + + self._definitions_checkout_dir = tempfile.mkdtemp() + + try: + # FIXME: not an ideal way of passing this info across + self._definitions_repo = definitions_repo + self._definitions_absref = definitions_absref + try: + definitions_cached_repo = self.lrc.get_repo(definitions_repo) + except morphlib.localrepocache.NotCached: + definitions_cached_repo = self.cache_repo_locally( + definitions_repo) + definitions_cached_repo.extract_commit( + definitions_absref, self._definitions_checkout_dir) + + # First, process the system and its stratum morphologies. These + # will all live in the same Git repository, and will point to + # various chunk morphologies. + chunk_queue = self._process_definitions_with_children( + system_filenames, definitions_repo, definitions_ref, + definitions_absref, definitions_tree, visit) + + # Now process all the chunks involved in the build. + for repo, ref, filename in chunk_queue: + self.process_chunk(definitions_repo, definitions_absref, repo, + ref, filename, visit) + finally: + shutil.rmtree(self._definitions_checkout_dir) + self._definitions_checkout_dir = None + + logging.debug('Saving contents of resolved tree cache') + self.tree_cache_manager.save_cache(self._resolved_trees) + + logging.debug('Saving contents of build systems cache') + self.buildsystem_cache_manager.save_cache( + self._resolved_buildsystems) + + +def create_source_pool(lrc, rrc, repo, ref, filename, cachedir, + original_ref=None, update_repos=True, + status_cb=None): # pragma: no cover + '''Find all the sources involved in building a given system. + + Given a system morphology, this function will traverse the tree of stratum + and chunk morphologies that the system points to and create appropriate + Source objects. These are added to a new SourcePool object, which is + returned. + + Note that Git submodules are not considered 'sources' in the current + implementation, and so they must be handled separately. + + The 'lrc' and 'rrc' parameters specify the local and remote Git repository + caches used for resolving the sources. + + ''' + pool = morphlib.sourcepool.SourcePool() + + def add_to_pool(reponame, ref, filename, absref, tree, morphology): + sources = morphlib.source.make_sources(reponame, ref, + filename, absref, + tree, morphology) + for source in sources: + pool.add(source) + + tree_cache_manager = PickleCacheManager( + os.path.join(cachedir, tree_cache_filename), tree_cache_size) + + buildsystem_cache_manager = PickleCacheManager( + os.path.join(cachedir, buildsystem_cache_filename), + buildsystem_cache_size) + + resolver = SourceResolver(lrc, rrc, tree_cache_manager, + buildsystem_cache_manager, update_repos, + status_cb) + resolver.traverse_morphs(repo, ref, [filename], + visit=add_to_pool, + definitions_original_ref=original_ref) + return pool diff --git a/morphlib/morphologyfactory_tests.py b/morphlib/sourceresolver_tests.py index 52d5f598..298dae5d 100644 --- a/morphlib/morphologyfactory_tests.py +++ b/morphlib/sourceresolver_tests.py @@ -1,4 +1,4 @@ -# Copyright (C) 2012-2014 Codethink Limited +# Copyright (C) 2015 Codethink Limited # # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by @@ -10,17 +10,19 @@ # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License along -# with this program; if not, write to the Free Software Foundation, Inc., -# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. +# with this program. If not, see <http://www.gnu.org/licenses/>. +import os +import shutil +import tempfile import unittest import morphlib -from morphlib.morphologyfactory import (MorphologyFactory, - MorphologyNotFoundError, - NotcachedError) -from morphlib.remoterepocache import CatFileError +from morphlib.sourceresolver import (SourceResolver, + PickleCacheManager, + MorphologyNotFoundError) +from morphlib.remoterepocache import CatFileError, LsTreeError class FakeRemoteRepoCache(object): @@ -37,6 +39,7 @@ class FakeRemoteRepoCache(object): def ls_tree(self, reponame, sha1): return [] + class FakeLocalRepo(object): morphologies = { @@ -65,15 +68,6 @@ class FakeLocalRepo(object): build-mode: bootstrap build-depends: [] ''', - 'stratum-no-chunk-bdeps.morph': ''' - name: stratum-no-chunk-bdeps - kind: stratum - chunks: - - name: chunk - repo: test:repo - ref: sha1 - build-mode: bootstrap - ''', 'stratum-no-bdeps-no-bootstrap.morph': ''' name: stratum-no-bdeps-no-bootstrap kind: stratum @@ -115,23 +109,25 @@ class FakeLocalRepo(object): def __init__(self): self.arch = 'x86_64' - def cat(self, sha1, filename): + def read_file(self, filename, ref): if filename in self.morphologies: values = { 'arch': self.arch, } return self.morphologies[filename] % values elif filename.endswith('.morph'): - return '''{ - "name": "%s", - "kind": "chunk", - "build-system": "dummy" - }''' % filename[:-len('.morph')] + return '''name: %s + kind: chunk + build-system: dummy''' % filename[:-len('.morph')] return 'text' - def ls_tree(self, sha1): + def list_files(self, ref, recurse): return self.morphologies.keys() + def update(self): + pass + + class FakeLocalRepoCache(object): def __init__(self, lr): @@ -143,21 +139,43 @@ class FakeLocalRepoCache(object): def get_repo(self, reponame): return self.lr - -class FakeApp(object): - - def status(self, **kwargs): - pass + def cache_repo(self, reponame): + return self.lr -class MorphologyFactoryTests(unittest.TestCase): +class SourceResolverTests(unittest.TestCase): def setUp(self): + # create temp "definitions" repo + # set self.sr._definitions_repo to that + # trick it into presenting temp repo using FakeLocalRepoCache + # magic self.lr = FakeLocalRepo() self.lrc = FakeLocalRepoCache(self.lr) self.rrc = FakeRemoteRepoCache() - self.mf = MorphologyFactory(self.lrc, self.rrc, app=FakeApp()) - self.lmf = MorphologyFactory(self.lrc, None) + + self.cachedir = tempfile.mkdtemp() + buildsystem_cache_file = os.path.join(self.cachedir, + 'detected-chunk-buildsystems.cache.pickle') + buildsystem_cache_manager = PickleCacheManager( + buildsystem_cache_file, 1000) + + tree_cache_file = os.path.join(self.cachedir, 'trees.cache.pickle') + tree_cache_manager = PickleCacheManager(tree_cache_file, 1000) + + def status(msg='', **kwargs): + pass + + self.sr = SourceResolver(self.lrc, self.rrc, tree_cache_manager, + buildsystem_cache_manager, True, status) + self.lsr = SourceResolver(self.lrc, None, tree_cache_manager, + buildsystem_cache_manager, True, status) + + self.sr._definitions_repo = None + self.lsr._definitions_repo = None + + def tearDown(self): + shutil.rmtree(self.cachedir) def nolocalfile(self, *args): raise IOError('File not found') @@ -165,18 +183,24 @@ class MorphologyFactoryTests(unittest.TestCase): def noremotefile(self, *args): raise CatFileError('reponame', 'ref', 'filename') + def noremoterepo(self, *args): + raise LsTreeError('reponame', 'ref') + def localmorph(self, *args): return ['chunk.morph'] def nolocalmorph(self, *args): - if args[-1].endswith('.morph'): + if args[0].endswith('.morph'): raise IOError('File not found') return 'text' - def autotoolsbuildsystem(self, *args): + def autotoolsbuildsystem(self, *args, **kwargs): return ['configure.in'] - def remotemorph(self, *args): + def emptytree(self, *args, **kwargs): + return [] + + def remotemorph(self, *args, **kwargs): return ['remote-chunk.morph'] def noremotemorph(self, *args): @@ -188,98 +212,125 @@ class MorphologyFactoryTests(unittest.TestCase): return False def test_gets_morph_from_local_repo(self): - self.lr.ls_tree = self.localmorph - morph = self.mf.get_morphology('reponame', 'sha1', + self.lr.list_files = self.localmorph + morph = self.sr._get_morphology('reponame', 'sha1', 'chunk.morph') self.assertEqual('chunk', morph['name']) + def test_gets_morph_from_cache(self): + self.lr.list_files = self.localmorph + morph_from_repo = self.sr._get_morphology('reponame', 'sha1', + 'chunk.morph') + morph_from_cache = self.sr._get_morphology('reponame', 'sha1', + 'chunk.morph') + self.assertEqual(morph_from_repo, morph_from_cache) + def test_gets_morph_from_remote_repo(self): self.rrc.ls_tree = self.remotemorph self.lrc.has_repo = self.doesnothaverepo - morph = self.mf.get_morphology('reponame', 'sha1', + morph = self.sr._get_morphology('reponame', 'sha1', 'remote-chunk.morph') self.assertEqual('remote-chunk', morph['name']) def test_autodetects_local_morphology(self): - self.lr.cat = self.nolocalmorph - self.lr.ls_tree = self.autotoolsbuildsystem - morph = self.mf.get_morphology('reponame', 'sha1', - 'assumed-local.morph') - self.assertEqual('assumed-local', morph['name']) + self.lr.read_file = self.nolocalmorph + self.lr.list_files = self.autotoolsbuildsystem + name = self.sr._detect_build_system('reponame', 'sha1', + 'assumed-local.morph') + self.assertEqual('autotools', name) + + def test_cache_repo_if_not_in_either_cache(self): + self.lrc.has_repo = self.doesnothaverepo + self.lr.read_file = self.nolocalmorph + self.lr.list_files = self.autotoolsbuildsystem + self.rrc.ls_tree = self.noremoterepo + name = self.sr._detect_build_system('reponame', 'sha1', + 'assumed-local.morph') + self.assertEqual('autotools', name) def test_autodetects_remote_morphology(self): self.lrc.has_repo = self.doesnothaverepo self.rrc.cat_file = self.noremotemorph self.rrc.ls_tree = self.autotoolsbuildsystem - morph = self.mf.get_morphology('reponame', 'sha1', - 'assumed-remote.morph') - self.assertEqual('assumed-remote', morph['name']) + name = self.sr._detect_build_system('reponame', 'sha1', + 'assumed-remote.morph') + self.assertEqual('autotools', name) - def test_raises_error_when_no_local_morph(self): - self.lr.cat = self.nolocalfile - self.assertRaises(MorphologyNotFoundError, self.mf.get_morphology, - 'reponame', 'sha1', 'unreached.morph') + def test_returns_none_when_no_local_morph(self): + self.lr.read_file = self.nolocalfile + morph = self.sr._get_morphology('reponame', 'sha1', 'unreached.morph') + self.assertEqual(morph, None) - def test_raises_error_when_fails_no_remote_morph(self): + def test_returns_none_when_fails_no_remote_morph(self): self.lrc.has_repo = self.doesnothaverepo self.rrc.cat_file = self.noremotefile - self.assertRaises(MorphologyNotFoundError, self.mf.get_morphology, - 'reponame', 'sha1', 'unreached.morph') + morph = self.sr._get_morphology('reponame', 'sha1', 'unreached.morph') + self.assertEqual(morph, None) + + def test_raises_error_when_repo_does_not_exist(self): + self.lrc.has_repo = self.doesnothaverepo + self.assertRaises(MorphologyNotFoundError, + self.lsr._detect_build_system, + 'reponame', 'sha1', 'non-existent.morph') + + def test_raises_error_when_failed_to_detect_build_system(self): + self.lr.read_file = self.nolocalfile + self.lr.list_files = self.emptytree + self.assertRaises(MorphologyNotFoundError, + self.sr._detect_build_system, + 'reponame', 'sha1', 'undetected.morph') def test_raises_error_when_name_mismatches(self): - self.assertRaises(morphlib.Error, self.mf.get_morphology, + self.assertRaises(morphlib.Error, self.sr._get_morphology, 'reponame', 'sha1', 'name-mismatch.morph') def test_looks_locally_with_no_remote(self): - self.lr.ls_tree = self.localmorph - morph = self.lmf.get_morphology('reponame', 'sha1', - 'chunk.morph') + self.lr.list_files = self.localmorph + morph = self.lsr._get_morphology('reponame', 'sha1', + 'chunk.morph') self.assertEqual('chunk', morph['name']) def test_autodetects_locally_with_no_remote(self): - self.lr.cat = self.nolocalmorph - self.lr.ls_tree = self.autotoolsbuildsystem - morph = self.mf.get_morphology('reponame', 'sha1', - 'assumed-local.morph') - self.assertEqual('assumed-local', morph['name']) + self.lr.read_file = self.nolocalmorph + self.lr.list_files = self.autotoolsbuildsystem + name = self.sr._detect_build_system('reponame', 'sha1', + 'assumed-local.morph') + self.assertEqual('autotools', name) - def test_fails_when_local_not_cached_and_no_remote(self): + def test_succeeds_when_local_not_cached_and_no_remote(self): self.lrc.has_repo = self.doesnothaverepo - self.assertRaises(NotcachedError, self.lmf.get_morphology, - 'reponame', 'sha1', 'unreached.morph') + self.lr.list_files = self.localmorph + morph = self.sr._get_morphology('reponame', 'sha1', + 'chunk.morph') + self.assertEqual('chunk', morph['name']) def test_arch_is_validated(self): self.lr.arch = 'unknown' - self.assertRaises(morphlib.Error, self.mf.get_morphology, + self.assertRaises(morphlib.Error, self.sr._get_morphology, 'reponame', 'sha1', 'system.morph') def test_arch_arm_defaults_to_le(self): self.lr.arch = 'armv7' - morph = self.mf.get_morphology('reponame', 'sha1', 'system.morph') + morph = self.sr._get_morphology('reponame', 'sha1', 'system.morph') self.assertEqual(morph['arch'], 'armv7l') def test_fails_on_parse_error(self): - self.assertRaises(morphlib.Error, self.mf.get_morphology, + self.assertRaises(morphlib.Error, self.sr._get_morphology, 'reponame', 'sha1', 'parse-error.morph') - def test_fails_on_no_chunk_bdeps(self): - self.assertRaises(morphlib.morphloader.NoBuildDependenciesError, - self.mf.get_morphology, 'reponame', 'sha1', - 'stratum-no-chunk-bdeps.morph') - def test_fails_on_no_bdeps_or_bootstrap(self): self.assertRaises( morphlib.morphloader.NoStratumBuildDependenciesError, - self.mf.get_morphology, 'reponame', 'sha1', + self.sr._get_morphology, 'reponame', 'sha1', 'stratum-no-bdeps-no-bootstrap.morph') def test_succeeds_on_bdeps_no_bootstrap(self): - self.mf.get_morphology( + self.sr._get_morphology( 'reponame', 'sha1', 'stratum-bdeps-no-bootstrap.morph') def test_fails_on_empty_stratum(self): self.assertRaises( morphlib.morphloader.EmptyStratumError, - self.mf.get_morphology, 'reponame', 'sha1', 'stratum-empty.morph') + self.sr._get_morphology, 'reponame', 'sha1', 'stratum-empty.morph') diff --git a/morphlib/stagingarea.py b/morphlib/stagingarea.py index b676d4db..8c2781aa 100644 --- a/morphlib/stagingarea.py +++ b/morphlib/stagingarea.py @@ -1,4 +1,4 @@ -# Copyright (C) 2012-2014 Codethink Limited +# Copyright (C) 2012-2015 Codethink Limited # # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by @@ -10,8 +10,7 @@ # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License along -# with this program; if not, write to the Free Software Foundation, Inc., -# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. +# with this program. If not, see <http://www.gnu.org/licenses/>. import logging @@ -166,7 +165,7 @@ class StagingArea(object): try: morphlib.bins.unpack_binary_from_file( handle, savedir + '/') - except BaseException, e: # pragma: no cover + except BaseException as e: # pragma: no cover shutil.rmtree(savedir) raise # TODO: This rename is not concurrency safe if two builds are @@ -190,9 +189,10 @@ class StagingArea(object): shutil.rmtree(self.dirname) - to_mount = ( + to_mount_in_staging = ( ('dev/shm', 'tmpfs', 'none'), ) + to_mount_in_bootstrap = () def ccache_dir(self, source): #pragma: no cover ccache_dir = self._app.settings['compiler-cache-dir'] @@ -266,7 +266,13 @@ class StagingArea(object): do_not_mount_dirs += [temp_dir] logging.debug("Not mounting dirs %r" % do_not_mount_dirs) + if self.use_chroot: + mounts = self.to_mount_in_staging + else: + mounts = [(os.path.join(self.dirname, target), type, source) + for target, type, source in self.to_mount_in_bootstrap] mount_proc = self.use_chroot + if ccache_dir and not self._app.settings['no-ccache']: ccache_target = os.path.join( self.dirname, kwargs['env']['CCACHE_DIR'].lstrip('/')) @@ -274,22 +280,38 @@ class StagingArea(object): else: binds = () + container_config=dict( + cwd=kwargs.pop('cwd', '/'), + root=chroot_dir, + mounts=mounts, + mount_proc=mount_proc, + binds=binds, + writable_paths=do_not_mount_dirs) + cmdline = morphlib.util.containerised_cmdline( - argv, cwd=kwargs.pop('cwd', '/'), - root=chroot_dir, mounts=self.to_mount, - binds=binds, mount_proc=mount_proc, - writable_paths=do_not_mount_dirs) - try: - if kwargs.get('logfile') != None: - logfile = kwargs.pop('logfile') - teecmd = ['tee', '-a', logfile] - return self._app.runcmd(cmdline, teecmd, **kwargs) - else: - return self._app.runcmd(cmdline, **kwargs) - except cliapp.AppException as e: - raise cliapp.AppException('In staging area %s: running ' - 'command \'%s\' failed.' % - (self.dirname, ' '.join(argv))) + argv, **container_config) + + if kwargs.get('logfile') != None: + logfile = kwargs.pop('logfile') + teecmd = ['tee', '-a', logfile] + exit, out, err = self._app.runcmd_unchecked( + cmdline, teecmd, **kwargs) + else: + exit, out, err = self._app.runcmd_unchecked(cmdline, **kwargs) + + if exit == 0: + return out + else: + logging.debug('Command returned code %i', exit) + msg = morphlib.util.error_message_for_containerised_commandline( + argv, err, container_config) + raise cliapp.AppException( + 'In staging area %s: %s' % (self._failed_location(), msg)) + + def _failed_location(self): # pragma: no cover + '''Path this staging area will be moved to if an error occurs.''' + return os.path.join(self._app.settings['tempdir'], 'failed', + os.path.basename(self.dirname)) def abort(self): # pragma: no cover '''Handle what to do with a staging area in the case of failure. @@ -298,9 +320,7 @@ class StagingArea(object): # TODO: when we add the option to throw away failed builds, # hook it up here - - dest_dir = os.path.join(self._app.settings['tempdir'], - 'failed', os.path.basename(self.dirname)) + dest_dir = self._failed_location() os.rename(self.dirname, dest_dir) self.dirname = dest_dir diff --git a/morphlib/stagingarea_tests.py b/morphlib/stagingarea_tests.py index dc43e4f6..97d78236 100644 --- a/morphlib/stagingarea_tests.py +++ b/morphlib/stagingarea_tests.py @@ -1,4 +1,4 @@ -# Copyright (C) 2012-2014 Codethink Limited +# Copyright (C) 2012-2015 Codethink Limited # # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by @@ -10,8 +10,7 @@ # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License along -# with this program; if not, write to the Free Software Foundation, Inc., -# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. +# with this program. If not, see <http://www.gnu.org/licenses/>. import cliapp diff --git a/morphlib/stopwatch.py b/morphlib/stopwatch.py index 29e584bd..bb01d06f 100644 --- a/morphlib/stopwatch.py +++ b/morphlib/stopwatch.py @@ -1,4 +1,4 @@ -# Copyright (C) 2011-2012 Codethink Limited +# Copyright (C) 2011-2012,2015 Codethink Limited # # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by @@ -10,8 +10,7 @@ # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License along -# with this program; if not, write to the Free Software Foundation, Inc., -# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. +# with this program. If not, see <http://www.gnu.org/licenses/>. import operator diff --git a/morphlib/stopwatch_tests.py b/morphlib/stopwatch_tests.py index deb528d5..3c8376b1 100644 --- a/morphlib/stopwatch_tests.py +++ b/morphlib/stopwatch_tests.py @@ -1,4 +1,4 @@ -# Copyright (C) 2011-2012 Codethink Limited +# Copyright (C) 2011-2012,2015 Codethink Limited # # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by @@ -10,8 +10,7 @@ # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License along -# with this program; if not, write to the Free Software Foundation, Inc., -# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. +# with this program. If not, see <http://www.gnu.org/licenses/>. import datetime diff --git a/morphlib/sysbranchdir.py b/morphlib/sysbranchdir.py index 4351c6b3..69119f5b 100644 --- a/morphlib/sysbranchdir.py +++ b/morphlib/sysbranchdir.py @@ -1,4 +1,4 @@ -# Copyright (C) 2013-2014 Codethink Limited +# Copyright (C) 2013-2015 Codethink Limited # # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by @@ -10,8 +10,7 @@ # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License along -# with this program; if not, write to the Free Software Foundation, Inc., -# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. +# with this program. If not, see <http://www.gnu.org/licenses/>. # # =*= License: GPL-2 =*= @@ -72,7 +71,11 @@ class SystemBranchDirectory(object): def _find_git_directory(self, repo_url): for gd in self.list_git_directories(): - if gd.get_config('morph.repository') == repo_url: + try: + gd_repo_url = gd.get_config('morph.repository') + except cliapp.AppException: # pragma: no cover + continue + if gd_repo_url == repo_url: return gd.dirname return None @@ -103,6 +106,11 @@ class SystemBranchDirectory(object): return os.path.join(self.root_directory, relative) + def relative_to_root_repo(self, path): # pragma: no cover + gitdirpath = self.get_git_directory_name(self.root_repository_url) + + return os.path.relpath(os.path.abspath(path), gitdirpath) + def get_git_directory_name(self, repo_url): '''Return directory pathname for a given git repository. @@ -119,6 +127,7 @@ class SystemBranchDirectory(object): ''' found_repo = self._find_git_directory(repo_url) + if not found_repo: return self._fabricate_git_directory_name(repo_url) return found_repo diff --git a/morphlib/sysbranchdir_tests.py b/morphlib/sysbranchdir_tests.py index 1aca54e6..341fedca 100644 --- a/morphlib/sysbranchdir_tests.py +++ b/morphlib/sysbranchdir_tests.py @@ -1,4 +1,4 @@ -# Copyright (C) 2013-2014 Codethink Limited +# Copyright (C) 2013-2015 Codethink Limited # # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by @@ -10,8 +10,7 @@ # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License along -# with this program; if not, write to the Free Software Foundation, Inc., -# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. +# with this program. If not, see <http://www.gnu.org/licenses/>. # # =*= License: GPL-2 =*= diff --git a/morphlib/systemmetadatadir.py b/morphlib/systemmetadatadir.py index 7e89142c..573d8a20 100644 --- a/morphlib/systemmetadatadir.py +++ b/morphlib/systemmetadatadir.py @@ -1,4 +1,4 @@ -# Copyright (C) 2013-2014 Codethink Limited +# Copyright (C) 2013-2015 Codethink Limited # # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by @@ -10,8 +10,7 @@ # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License along -# with this program; if not, write to the Free Software Foundation, Inc., -# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. +# with this program. If not, see <http://www.gnu.org/licenses/>. # # =*= License: GPL-2 =*= diff --git a/morphlib/systemmetadatadir_tests.py b/morphlib/systemmetadatadir_tests.py index 0126f862..03496108 100644 --- a/morphlib/systemmetadatadir_tests.py +++ b/morphlib/systemmetadatadir_tests.py @@ -1,4 +1,4 @@ -# Copyright (C) 2013 Codethink Limited +# Copyright (C) 2013,2015 Codethink Limited # # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by @@ -10,8 +10,7 @@ # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License along -# with this program; if not, write to the Free Software Foundation, Inc., -# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. +# with this program. If not, see <http://www.gnu.org/licenses/>. # # =*= License: GPL-2 =*= diff --git a/morphlib/util.py b/morphlib/util.py index 6f735387..a3a07cce 100644 --- a/morphlib/util.py +++ b/morphlib/util.py @@ -1,4 +1,4 @@ -# Copyright (C) 2011-2014 Codethink Limited +# Copyright (C) 2011-2015 Codethink Limited # # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by @@ -10,12 +10,12 @@ # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License along -# with this program; if not, write to the Free Software Foundation, Inc., -# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. +# with this program. If not, see <http://www.gnu.org/licenses/>. import contextlib import itertools import os +import pipes import re import subprocess import textwrap @@ -41,7 +41,6 @@ try: from multiprocessing import cpu_count except NotImplementedError: # pragma: no cover cpu_count = lambda: 1 -import os def indent(string, spaces=4): @@ -465,6 +464,10 @@ def get_host_architecture(): # pragma: no cover 'i686': 'x86_32', 'armv7l': 'armv7l', 'armv7b': 'armv7b', + 'armv8l': 'armv8l', + 'armv8b': 'armv8b', + 'aarch64': 'armv8l64', + 'aarch64b': 'armv8b64', 'ppc64': 'ppc64' } @@ -626,3 +629,18 @@ def containerised_cmdline(args, cwd='.', root='/', binds=(), cmdargs.append(root) cmdargs.extend(args) return unshared_cmdline(cmdargs, root=root, **kwargs) + + +def error_message_for_containerised_commandline( + argv, err, container_kwargs): # pragma: no cover + '''Return a semi-readable error message for a containerised command.''' + + # This function should do some formatting of the container_kwargs dict, + # rather than just dumping it in the error message, but that is better than + # nothing. + + argv_string = ' '.join(map(pipes.quote, argv)) + return 'Command failed: %s:\n' \ + 'Containerisation settings: %s\n' \ + 'Error output:\n%s' \ + % (argv_string, container_kwargs, err) diff --git a/morphlib/util_tests.py b/morphlib/util_tests.py index 715892b6..20972a58 100644 --- a/morphlib/util_tests.py +++ b/morphlib/util_tests.py @@ -1,4 +1,4 @@ -# Copyright (C) 2011-2014 Codethink Limited +# Copyright (C) 2011-2015 Codethink Limited # # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by @@ -10,8 +10,7 @@ # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License along -# with this program; if not, write to the Free Software Foundation, Inc., -# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. +# with this program. If not, see <http://www.gnu.org/licenses/>. import os diff --git a/morphlib/workspace.py b/morphlib/workspace.py index 27ccbe65..26986655 100644 --- a/morphlib/workspace.py +++ b/morphlib/workspace.py @@ -1,4 +1,4 @@ -# Copyright (C) 2013 Codethink Limited +# Copyright (C) 2013,2015 Codethink Limited # # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by @@ -10,8 +10,7 @@ # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License along -# with this program; if not, write to the Free Software Foundation, Inc., -# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. +# with this program. If not, see <http://www.gnu.org/licenses/>. # # =*= License: GPL-2 =*= diff --git a/morphlib/workspace_tests.py b/morphlib/workspace_tests.py index 9eef1053..23d7ad5e 100644 --- a/morphlib/workspace_tests.py +++ b/morphlib/workspace_tests.py @@ -1,4 +1,4 @@ -# Copyright (C) 2013-2014 Codethink Limited +# Copyright (C) 2013-2015 Codethink Limited # # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by @@ -10,8 +10,7 @@ # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License along -# with this program; if not, write to the Free Software Foundation, Inc., -# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. +# with this program. If not, see <http://www.gnu.org/licenses/>. # # =*= License: GPL-2 =*= diff --git a/morphlib/writeexts.py b/morphlib/writeexts.py index 0fd0ad7b..129b2bc4 100644 --- a/morphlib/writeexts.py +++ b/morphlib/writeexts.py @@ -1,4 +1,4 @@ -# Copyright (C) 2012-2014 Codethink Limited +# Copyright (C) 2012-2015 Codethink Limited # # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by @@ -10,8 +10,7 @@ # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License along -# with this program; if not, write to the Free Software Foundation, Inc., -# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. +# with this program. If not, see <http://www.gnu.org/licenses/>. import cliapp @@ -22,6 +21,9 @@ import shutil import sys import time import tempfile +import errno +import stat +import contextlib import morphlib @@ -80,14 +82,14 @@ class Fstab(object): class WriteExtension(cliapp.Application): '''A base class for deployment write extensions. - + A subclass should subclass this class, and add a ``process_args`` method. - + Note that it is not necessary to subclass this class for write extensions. This class is here just to collect common code for write extensions. - + ''' def setup_logging(self): @@ -122,13 +124,13 @@ class WriteExtension(cliapp.Application): def status(self, **kwargs): '''Provide status output. - + The ``msg`` keyword argument is the actual message, the rest are values for fields in the message as interpolated by %. - + ''' - + self.output.write('%s\n' % (kwargs['msg'] % kwargs)) self.output.flush() @@ -145,34 +147,45 @@ class WriteExtension(cliapp.Application): def create_local_system(self, temp_root, raw_disk): '''Create a raw system image locally.''' + + with self.created_disk_image(raw_disk): + self.format_btrfs(raw_disk) + self.create_system(temp_root, raw_disk) + + @contextlib.contextmanager + def created_disk_image(self, location): size = self.get_disk_size() if not size: raise cliapp.AppException('DISK_SIZE is not defined') - self.create_raw_disk_image(raw_disk, size) + self.create_raw_disk_image(location, size) try: - self.mkfs_btrfs(raw_disk) - mp = self.mount(raw_disk) + yield except BaseException: - sys.stderr.write('Error creating disk image') - os.remove(raw_disk) + os.unlink(location) raise + + def format_btrfs(self, raw_disk): try: - self.create_btrfs_system_layout( - temp_root, mp, version_label='factory', - disk_uuid=self.get_uuid(raw_disk)) - except BaseException, e: - sys.stderr.write('Error creating Btrfs system layout') - self.unmount(mp) - os.remove(raw_disk) + self.mkfs_btrfs(raw_disk) + except BaseException: + sys.stderr.write('Error creating disk image') raise - else: - self.unmount(mp) + + def create_system(self, temp_root, raw_disk): + with self.mount(raw_disk) as mp: + try: + self.create_btrfs_system_layout( + temp_root, mp, version_label='factory', + disk_uuid=self.get_uuid(raw_disk)) + except BaseException as e: + sys.stderr.write('Error creating Btrfs system layout') + raise def _parse_size(self, size): '''Parse a size from a string. - + Return size in bytes. - + ''' m = re.match('^(\d+)([kmgKMG]?)$', size) @@ -223,8 +236,32 @@ class WriteExtension(cliapp.Application): def mkfs_btrfs(self, location): '''Create a btrfs filesystem on the disk.''' + self.status(msg='Creating btrfs filesystem') - cliapp.runcmd(['mkfs.btrfs', '-L', 'baserock', location]) + try: + # The following command disables some new filesystem features. We + # need to do this because at the time of writing, SYSLINUX has not + # been updated to understand these new features and will fail to + # boot if the kernel is on a filesystem where they are enabled. + cliapp.runcmd( + ['mkfs.btrfs','-f', '-L', 'baserock', + '--features', '^extref', + '--features', '^skinny-metadata', + '--features', '^mixed-bg', + '--nodesize', '4096', + location]) + except cliapp.AppException as e: + if 'unrecognized option \'--features\'' in e.msg: + # Old versions of mkfs.btrfs (including v0.20, present in many + # Baserock releases) don't support the --features option, but + # also don't enable the new features by default. So we can + # still create a bootable system in this situation. + logging.debug( + 'Assuming mkfs.btrfs failure was because the tool is too ' + 'old to have --features flag.') + cliapp.runcmd(['mkfs.btrfs','-f', '-L', 'baserock', location]) + else: + raise def get_uuid(self, location): '''Get the UUID of a block device's file system.''' @@ -232,31 +269,26 @@ class WriteExtension(cliapp.Application): # lies by exiting successfully. return cliapp.runcmd(['blkid', '-s', 'UUID', '-o', 'value', location]).strip() - - def mount(self, location): - '''Mount the filesystem so it can be tweaked. - - Return path to the mount point. - The mount point is a newly created temporary directory. - The caller must call self.unmount to unmount on the return value. - - ''' - self.status(msg='Mounting filesystem') - tempdir = tempfile.mkdtemp() - cliapp.runcmd(['mount', '-o', 'loop', location, tempdir]) - return tempdir - - def unmount(self, mount_point): - '''Unmount the filesystem mounted by self.mount. - - Also, remove the temporary directory. - - ''' - - self.status(msg='Unmounting filesystem') - cliapp.runcmd(['umount', mount_point]) - os.rmdir(mount_point) + @contextlib.contextmanager + def mount(self, location): + self.status(msg='Mounting filesystem') + try: + mount_point = tempfile.mkdtemp() + if self.is_device(location): + cliapp.runcmd(['mount', location, mount_point]) + else: + cliapp.runcmd(['mount', '-o', 'loop', location, mount_point]) + except BaseException as e: + sys.stderr.write('Error mounting filesystem') + os.rmdir(mount_point) + raise + try: + yield mount_point + finally: + self.status(msg='Unmounting filesystem') + cliapp.runcmd(['umount', mount_point]) + os.rmdir(mount_point) def create_btrfs_system_layout(self, temp_root, mountpoint, version_label, disk_uuid): @@ -327,7 +359,7 @@ class WriteExtension(cliapp.Application): self.status(msg='Creating %s subvolume' % state_subdir) subvolume = os.path.join(mountpoint, 'state', state_subdir) cliapp.runcmd(['btrfs', 'subvolume', 'create', subvolume]) - os.chmod(subvolume, 0755) + os.chmod(subvolume, 0o755) existing_state_dir = os.path.join(system_dir, state_subdir) files = [] @@ -465,6 +497,12 @@ class WriteExtension(cliapp.Application): self.status(msg='Creating extlinux.conf') config = os.path.join(real_root, 'extlinux.conf') + + ''' Please also update the documentation in the following files + if you change these default kernel args: + - kvm.write.help + - rawdisk.write.help + - virtualbox-ssh.write.help ''' kernel_args = ( 'rw ' # ro ought to work, but we don't test that regularly 'init=/sbin/init ' # default, but it doesn't hurt to be explicit @@ -536,9 +574,8 @@ class WriteExtension(cliapp.Application): '''Does the user want to generate a bootloader config? The user may set $BOOTLOADER_CONFIG_FORMAT to the desired - format (u-boot or extlinux). If not set, extlinux is the - default but will be generated on x86-32 and x86-64, but not - otherwise. + format. 'extlinux' is the only allowed value, and is the default + value for x86-32 and x86-64. ''' @@ -572,3 +609,12 @@ class WriteExtension(cliapp.Application): logging.error("Error checking SSH connectivity: %s", str(e)) raise cliapp.AppException( 'Unable to SSH to %s: %s' % (ssh_host, e)) + + def is_device(self, location): + try: + st = os.stat(location) + return stat.S_ISBLK(st.st_mode) + except OSError as e: + if e.errno == errno.ENOENT: + return False + raise diff --git a/morphlib/xfer-hole b/morphlib/xfer-hole index 22ee06bf..91f1be01 100755 --- a/morphlib/xfer-hole +++ b/morphlib/xfer-hole @@ -7,7 +7,7 @@ # SEEK_DATA and SEEK_HOLE. # # -# Copyright (C) 2014 Codethink Limited +# Copyright (C) 2014-2015 Codethink Limited # # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by @@ -19,8 +19,7 @@ # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License along -# with this program; if not, write to the Free Software Foundation, Inc., -# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. +# with this program. If not, see <http://www.gnu.org/licenses/>. # # =*= License: GPL-2 =*= diff --git a/morphlib/yamlparse.py b/morphlib/yamlparse.py index 6f139304..075fe3b1 100644 --- a/morphlib/yamlparse.py +++ b/morphlib/yamlparse.py @@ -1,4 +1,4 @@ -# Copyright (C) 2013-2014 Codethink Limited +# Copyright (C) 2013-2015 Codethink Limited # # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by @@ -10,8 +10,7 @@ # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License along -# with this program; if not, write to the Free Software Foundation, Inc., -# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. +# with this program. If not, see <http://www.gnu.org/licenses/>. import morphlib diff --git a/morphlib/yamlparse_tests.py b/morphlib/yamlparse_tests.py index 38815168..d5da0ca9 100644 --- a/morphlib/yamlparse_tests.py +++ b/morphlib/yamlparse_tests.py @@ -1,4 +1,4 @@ -# Copyright (C) 2013-2014 Codethink Limited +# Copyright (C) 2013-2015 Codethink Limited # # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by @@ -10,8 +10,7 @@ # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License along -# with this program; if not, write to the Free Software Foundation, Inc., -# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. +# with this program. If not, see <http://www.gnu.org/licenses/>. import unittest diff --git a/scripts/check-copyright-year b/scripts/check-copyright-year index e72eaeea..08bee0af 100755 --- a/scripts/check-copyright-year +++ b/scripts/check-copyright-year @@ -2,7 +2,7 @@ # # Does the copyright statement include the year of the latest git commit? # -# Copyright (C) 2012, 2014 Codethink Limited +# Copyright (C) 2012, 2014-2015 Codethink Limited # # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by @@ -14,8 +14,7 @@ # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License along -# with this program; if not, write to the Free Software Foundation, Inc., -# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. +# with this program. If not, see <http://www.gnu.org/licenses/>. from __future__ import print_function diff --git a/scripts/check-silliness b/scripts/check-silliness index 597eb664..0c761a69 100755 --- a/scripts/check-silliness +++ b/scripts/check-silliness @@ -2,7 +2,7 @@ # # Does the file contain any of the code constructs deemed silly? # -# Copyright (C) 2013, 2014 Codethink Limited +# Copyright (C) 2013, 2014-2015 Codethink Limited # # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by @@ -14,8 +14,7 @@ # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License along -# with this program; if not, write to the Free Software Foundation, Inc., -# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. +# with this program. If not, see <http://www.gnu.org/licenses/>. errors=0 diff --git a/scripts/clean-artifact-cache b/scripts/clean-artifact-cache index 2fdd5605..623f61f3 100755 --- a/scripts/clean-artifact-cache +++ b/scripts/clean-artifact-cache @@ -1,6 +1,6 @@ #!/bin/sh -# Copyright (C) 2012 Codethink Limited +# Copyright (C) 2012,2015 Codethink Limited # # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by @@ -12,8 +12,7 @@ # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License along -# with this program; if not, write to the Free Software Foundation, Inc., -# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. +# with this program. If not, see <http://www.gnu.org/licenses/>. # Remove all chunk artifacts in the cache except the newest. Morph does # not currently clean its caches at any point, so this script is necessary diff --git a/scripts/cmd-filter b/scripts/cmd-filter index d2f4f784..25511755 100755 --- a/scripts/cmd-filter +++ b/scripts/cmd-filter @@ -1,5 +1,5 @@ #!/bin/sh -# Copyright (C) 2012 Codethink Limited +# Copyright (C) 2012,2015 Codethink Limited # # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by @@ -11,8 +11,7 @@ # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License along -# with this program; if not, write to the Free Software Foundation, Inc., -# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. +# with this program. If not, see <http://www.gnu.org/licenses/>. # Wrapper around morph for use by cmdtest tests. This does several things: # diff --git a/scripts/convert-git-cache b/scripts/convert-git-cache index 33a8edf1..4a7f89ce 100755 --- a/scripts/convert-git-cache +++ b/scripts/convert-git-cache @@ -1,6 +1,6 @@ #!/bin/sh # -# Copyright (C) 2012,2013 Codethink Limited +# Copyright (C) 2012,2013,2015 Codethink Limited # # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by @@ -12,8 +12,7 @@ # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License along -# with this program; if not, write to the Free Software Foundation, Inc., -# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. +# with this program. If not, see <http://www.gnu.org/licenses/>. set -eu diff --git a/scripts/edit-morph b/scripts/edit-morph index 90679b23..8c0863c3 100755 --- a/scripts/edit-morph +++ b/scripts/edit-morph @@ -1,5 +1,5 @@ #!/usr/bin/env python -# Copyright (C) 2013-2014 Codethink Limited +# Copyright (C) 2013-2015 Codethink Limited # # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by @@ -11,8 +11,7 @@ # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License along -# with this program; if not, write to the Free Software Foundation, Inc., -# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. +# with this program. If not, see <http://www.gnu.org/licenses/>. import cliapp diff --git a/scripts/fix-committer-info b/scripts/fix-committer-info index 0bd85274..58276afc 100644 --- a/scripts/fix-committer-info +++ b/scripts/fix-committer-info @@ -1,5 +1,5 @@ #!/bin/sh -# Copyright (C) 2012 Codethink Limited +# Copyright (C) 2012,2015 Codethink Limited # # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by @@ -11,8 +11,7 @@ # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License along -# with this program; if not, write to the Free Software Foundation, Inc., -# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. +# with this program. If not, see <http://www.gnu.org/licenses/>. # Fix git committer info. By hardcoding all of this info we make sure that # all of the commits we make during testing have reproducible commit SHA1s. diff --git a/scripts/list-tree b/scripts/list-tree index a1e2e8cb..877c42a7 100755 --- a/scripts/list-tree +++ b/scripts/list-tree @@ -1,5 +1,5 @@ #!/bin/sh -# Copyright (C) 2012 Codethink Limited +# Copyright (C) 2012,2015 Codethink Limited # # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by @@ -11,8 +11,7 @@ # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License along -# with this program; if not, write to the Free Software Foundation, Inc., -# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. +# with this program. If not, see <http://www.gnu.org/licenses/>. # List contents of a directory tree in a reproducible manner: only include # details that we care about, and that won't be changing between test runs diff --git a/scripts/python-check b/scripts/python-check index ce3419d5..fc89023d 100644 --- a/scripts/python-check +++ b/scripts/python-check @@ -1,5 +1,5 @@ #!/bin/sh -# Copyright (C) 2012-2013 Codethink Limited +# Copyright (C) 2012-2013,2015 Codethink Limited # # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by @@ -11,8 +11,7 @@ # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License along -# with this program; if not, write to the Free Software Foundation, Inc., -# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. +# with this program. If not, see <http://www.gnu.org/licenses/>. # When sourced by scripts, if the python version is too old # fake the output and exit. diff --git a/scripts/review-gitmodules b/scripts/review-gitmodules index 89574833..16a416d6 100755 --- a/scripts/review-gitmodules +++ b/scripts/review-gitmodules @@ -1,5 +1,5 @@ #!/usr/bin/python -# Copyright (C) 2012-2013 Codethink Limited +# Copyright (C) 2012-2013,2015 Codethink Limited # # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by @@ -11,8 +11,7 @@ # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License along -# with this program; if not, write to the Free Software Foundation, Inc., -# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. +# with this program. If not, see <http://www.gnu.org/licenses/>. # Check every system morphology in a checked out system branch by editing # every chunk and recursively checking gitmodules. diff --git a/scripts/run-git-in b/scripts/run-git-in index 80b87d1a..ae3fca93 100755 --- a/scripts/run-git-in +++ b/scripts/run-git-in @@ -1,5 +1,5 @@ #!/bin/sh -# Copyright (C) 2012 Codethink Limited +# Copyright (C) 2012,2015 Codethink Limited # # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by @@ -11,8 +11,7 @@ # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License along -# with this program; if not, write to the Free Software Foundation, Inc., -# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. +# with this program. If not, see <http://www.gnu.org/licenses/>. # Run git in a given directory. diff --git a/scripts/setup-3rd-party-strata b/scripts/setup-3rd-party-strata index fc263f96..fd5aab9e 100644 --- a/scripts/setup-3rd-party-strata +++ b/scripts/setup-3rd-party-strata @@ -1,5 +1,5 @@ #!/bin/sh -# Copyright (C) 2012-2013 Codethink Limited +# Copyright (C) 2012-2013,2015 Codethink Limited # # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by @@ -11,8 +11,7 @@ # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License along -# with this program; if not, write to the Free Software Foundation, Inc., -# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. +# with this program. If not, see <http://www.gnu.org/licenses/>. # Create strata outside the main morphologies repository, which is useful diff --git a/scripts/test-morph b/scripts/test-morph index d8480d92..9a48d12a 100755 --- a/scripts/test-morph +++ b/scripts/test-morph @@ -1,5 +1,5 @@ #!/bin/sh -# Copyright (C) 2012,2014 Codethink Limited +# Copyright (C) 2012,2014-2015 Codethink Limited # # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by @@ -11,8 +11,7 @@ # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License along -# with this program; if not, write to the Free Software Foundation, Inc., -# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. +# with this program. If not, see <http://www.gnu.org/licenses/>. # Wrapper around morph for use by cmdtest tests. This does several things: # diff --git a/scripts/test-shell.c b/scripts/test-shell.c index 2f8ecbd8..1a6f1642 100644 --- a/scripts/test-shell.c +++ b/scripts/test-shell.c @@ -1,3 +1,18 @@ +/* Copyright (C) 2014-2015 Codethink Limited + + This program is free software; you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation; version 2 of the License. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License along + with this program. If not, see <http://www.gnu.org/licenses/>. +*/ + #include <fcntl.h> #include <sys/types.h> #include <sys/stat.h> @@ -10,6 +25,7 @@ #include <stdint.h> #include <ftw.h> #include <errno.h> +#include <err.h> char *readlinka(char const *path){ size_t buflen = BUFSIZ; @@ -113,8 +129,8 @@ int copy_file_objects(FILE *source, FILE *target) { do { read = fread(buffer, 1, sizeof(buffer), source); fwrite(buffer, 1, read, target); - } while (!feof(source)); - return ferror(source) ? -1 : 0; + } while (!feof(source) && !feof(target)); + return (ferror(source) || ferror(target)) ? -1 : 0; } int run_commands(FILE *cmdstream){ @@ -145,8 +161,14 @@ int run_commands(FILE *cmdstream){ } else if (strstr(line, "create file ") == line) { char const *filename = line + sizeof("create file ") -1; FILE *outfile = fopen(filename, "w"); + if (outfile == NULL){ + ret = 1; + err(errno, "Opening %s for write failed", filename); + break; + } if (copy_file_objects(cmdstream, outfile) < 0) { ret = 1; + err(errno, "Writing to %s failed", filename); fclose(outfile); break; } diff --git a/scripts/yaml-extract b/scripts/yaml-extract index 6f55e62f..4ca5551a 100755 --- a/scripts/yaml-extract +++ b/scripts/yaml-extract @@ -28,7 +28,7 @@ # values for missing fields (e.g., the morph field if name is given). # Construct your tests accordingly. -# Copyright (C) 2013-2014 Codethink Limited +# Copyright (C) 2013-2015 Codethink Limited # # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by @@ -40,8 +40,7 @@ # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License along -# with this program; if not, write to the Free Software Foundation, Inc., -# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. +# with this program. If not, see <http://www.gnu.org/licenses/>. import sys @@ -1,4 +1,4 @@ -# Copyright (C) 2011 - 2014 Codethink Limited +# Copyright (C) 2011,2014-2015 Codethink Limited # # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by @@ -10,8 +10,7 @@ # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License along -# with this program; if not, write to the Free Software Foundation, Inc., -# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. +# with this program. If not, see <http://www.gnu.org/licenses/>. '''Setup.py for morph.''' diff --git a/source-stats b/source-stats index 811fdc3b..08f7bf97 100755 --- a/source-stats +++ b/source-stats @@ -1,5 +1,5 @@ #!/usr/bin/python -# Copyright (C) 2012,2013 Codethink Limited +# Copyright (C) 2012,2013,2015 Codethink Limited # # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by @@ -11,8 +11,7 @@ # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License along -# with this program; if not, write to the Free Software Foundation, Inc., -# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. +# with this program. If not, see <http://www.gnu.org/licenses/>. import cliapp diff --git a/tests.branching/add-then-edit.script b/tests.branching/add-then-edit.script index be3315d9..bd1094e5 100755 --- a/tests.branching/add-then-edit.script +++ b/tests.branching/add-then-edit.script @@ -1,6 +1,6 @@ #!/bin/sh # -# Copyright (C) 2013-2014 Codethink Limited +# Copyright (C) 2013-2015 Codethink Limited # # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by @@ -12,8 +12,7 @@ # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License along -# with this program; if not, write to the Free Software Foundation, Inc., -# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. +# with this program. If not, see <http://www.gnu.org/licenses/>. ## Test the workflow of adding a new chunk to a stratum then editing it diff --git a/tests.branching/add-then-edit.setup b/tests.branching/add-then-edit.setup index bb58d05a..0dc3368d 100755 --- a/tests.branching/add-then-edit.setup +++ b/tests.branching/add-then-edit.setup @@ -1,5 +1,5 @@ #!/bin/sh -# Copyright (C) 2013 Codethink Limited +# Copyright (C) 2013,2015 Codethink Limited # # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by @@ -11,8 +11,7 @@ # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License along -# with this program; if not, write to the Free Software Foundation, Inc., -# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. +# with this program. If not, see <http://www.gnu.org/licenses/>. set -eu diff --git a/tests.branching/branch-cleans-up-on-failure.script b/tests.branching/branch-cleans-up-on-failure.script index 55666137..6dd8eac0 100755 --- a/tests.branching/branch-cleans-up-on-failure.script +++ b/tests.branching/branch-cleans-up-on-failure.script @@ -1,6 +1,6 @@ #!/bin/sh # -# Copyright (C) 2012 Codethink Limited +# Copyright (C) 2012,2015 Codethink Limited # # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by @@ -12,8 +12,7 @@ # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License along -# with this program; if not, write to the Free Software Foundation, Inc., -# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. +# with this program. If not, see <http://www.gnu.org/licenses/>. ## If a command fails, the state of the workspace should be as if the command diff --git a/tests.branching/branch-cleans-up-on-failure.stderr b/tests.branching/branch-cleans-up-on-failure.stderr index 37533408..959226d8 100644 --- a/tests.branching/branch-cleans-up-on-failure.stderr +++ b/tests.branching/branch-cleans-up-on-failure.stderr @@ -1 +1 @@ -ERROR: Ref invalid-ref is an invalid reference for repo file://TMP/morphs +ERROR: Git directory TMP/morphs has no commit at ref invalid-ref^{commit}. diff --git a/tests.branching/branch-creates-new-system-branch-not-from-master.script b/tests.branching/branch-creates-new-system-branch-not-from-master.script index c561f191..e4d815d1 100755 --- a/tests.branching/branch-creates-new-system-branch-not-from-master.script +++ b/tests.branching/branch-creates-new-system-branch-not-from-master.script @@ -1,6 +1,6 @@ #!/bin/sh # -# Copyright (C) 2012,2014 Codethink Limited +# Copyright (C) 2012,2014-2015 Codethink Limited # # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by @@ -12,8 +12,7 @@ # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License along -# with this program; if not, write to the Free Software Foundation, Inc., -# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. +# with this program. If not, see <http://www.gnu.org/licenses/>. ## Make sure "morph branch" creates a new system branch. diff --git a/tests.branching/branch-creates-new-system-branch.script b/tests.branching/branch-creates-new-system-branch.script index 784bed62..bc849f90 100755 --- a/tests.branching/branch-creates-new-system-branch.script +++ b/tests.branching/branch-creates-new-system-branch.script @@ -1,6 +1,6 @@ #!/bin/sh # -# Copyright (C) 2012,2014 Codethink Limited +# Copyright (C) 2012,2014-2015 Codethink Limited # # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by @@ -12,8 +12,7 @@ # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License along -# with this program; if not, write to the Free Software Foundation, Inc., -# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. +# with this program. If not, see <http://www.gnu.org/licenses/>. ## Make sure "morph branch" creates a new system branch. diff --git a/tests.branching/branch-fails-if-branch-exists.script b/tests.branching/branch-fails-if-branch-exists.script index 8a7da8ab..d396d4fa 100755 --- a/tests.branching/branch-fails-if-branch-exists.script +++ b/tests.branching/branch-fails-if-branch-exists.script @@ -1,6 +1,6 @@ #!/bin/sh # -# Copyright (C) 2012 Codethink Limited +# Copyright (C) 2012,2015 Codethink Limited # # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by @@ -12,8 +12,7 @@ # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License along -# with this program; if not, write to the Free Software Foundation, Inc., -# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. +# with this program. If not, see <http://www.gnu.org/licenses/>. ## Make sure "morph branch" fails if the system branch already exists in the diff --git a/tests.branching/branch-when-branchdir-exists-locally.script b/tests.branching/branch-when-branchdir-exists-locally.script index 66a116be..f8f93812 100755 --- a/tests.branching/branch-when-branchdir-exists-locally.script +++ b/tests.branching/branch-when-branchdir-exists-locally.script @@ -1,6 +1,6 @@ #!/bin/sh # -# Copyright (C) 2012 Codethink Limited +# Copyright (C) 2012,2015 Codethink Limited # # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by @@ -12,8 +12,7 @@ # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License along -# with this program; if not, write to the Free Software Foundation, Inc., -# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. +# with this program. If not, see <http://www.gnu.org/licenses/>. ## Make sure "morph branch" fails when the system branch directory already diff --git a/tests.branching/branch-works-anywhere.script b/tests.branching/branch-works-anywhere.script index 7f6156ce..3dfcffc2 100755 --- a/tests.branching/branch-works-anywhere.script +++ b/tests.branching/branch-works-anywhere.script @@ -1,6 +1,6 @@ #!/bin/bash # -# Copyright (C) 2012,2014 Codethink Limited +# Copyright (C) 2012,2014-2015 Codethink Limited # # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by @@ -12,8 +12,7 @@ # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License along -# with this program; if not, write to the Free Software Foundation, Inc., -# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. +# with this program. If not, see <http://www.gnu.org/licenses/>. ## Make sure "morph branch" works anywhere in a workspace or system branch. diff --git a/tests.branching/checkout-cleans-up-on-failure.script b/tests.branching/checkout-cleans-up-on-failure.script index a0b0411b..fa0d61fa 100755 --- a/tests.branching/checkout-cleans-up-on-failure.script +++ b/tests.branching/checkout-cleans-up-on-failure.script @@ -1,6 +1,6 @@ #!/bin/sh # -# Copyright (C) 2012 Codethink Limited +# Copyright (C) 2012,2015 Codethink Limited # # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by @@ -12,8 +12,7 @@ # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License along -# with this program; if not, write to the Free Software Foundation, Inc., -# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. +# with this program. If not, see <http://www.gnu.org/licenses/>. ## If a command fails, the state of the workspace should be as if the command diff --git a/tests.branching/checkout-cleans-up-on-failure.stderr b/tests.branching/checkout-cleans-up-on-failure.stderr index 5b6a5645..14ed130c 100644 --- a/tests.branching/checkout-cleans-up-on-failure.stderr +++ b/tests.branching/checkout-cleans-up-on-failure.stderr @@ -1 +1 @@ -ERROR: Ref i/do/not/exist is an invalid reference for repo file://TMP/morphs +ERROR: Git directory TMP/morphs has no commit at ref i/do/not/exist^{commit}. diff --git a/tests.branching/checkout-existing-branch.script b/tests.branching/checkout-existing-branch.script index b1740d9c..8ab2834d 100755 --- a/tests.branching/checkout-existing-branch.script +++ b/tests.branching/checkout-existing-branch.script @@ -1,6 +1,6 @@ #!/bin/sh # -# Copyright (C) 2012,2014 Codethink Limited +# Copyright (C) 2012,2014-2015 Codethink Limited # # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by @@ -12,8 +12,7 @@ # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License along -# with this program; if not, write to the Free Software Foundation, Inc., -# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. +# with this program. If not, see <http://www.gnu.org/licenses/>. ## Verify that "morph checkout test:morphs master" works. diff --git a/tests.branching/checkout-non-aliased-repos.script b/tests.branching/checkout-non-aliased-repos.script index b52f6675..4b2ac1c2 100755 --- a/tests.branching/checkout-non-aliased-repos.script +++ b/tests.branching/checkout-non-aliased-repos.script @@ -1,6 +1,6 @@ #!/bin/bash # -# Copyright (C) 2012 Codethink Limited +# Copyright (C) 2012,2015 Codethink Limited # # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by @@ -12,8 +12,7 @@ # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License along -# with this program; if not, write to the Free Software Foundation, Inc., -# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. +# with this program. If not, see <http://www.gnu.org/licenses/>. ## Verify that "morph checkout" works with repos that are not aliased. diff --git a/tests.branching/checkout-works-anywhere.script b/tests.branching/checkout-works-anywhere.script index 14d18842..ebd5afc4 100755 --- a/tests.branching/checkout-works-anywhere.script +++ b/tests.branching/checkout-works-anywhere.script @@ -1,6 +1,6 @@ #!/bin/bash # -# Copyright (C) 2012,2014 Codethink Limited +# Copyright (C) 2012,2014-2015 Codethink Limited # # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by @@ -12,8 +12,7 @@ # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License along -# with this program; if not, write to the Free Software Foundation, Inc., -# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. +# with this program. If not, see <http://www.gnu.org/licenses/>. ## Make sure "morph checkout" works anywhere in a workspace or system branch. diff --git a/tests.branching/edit-checkouts-existing-chunk.script b/tests.branching/edit-checkouts-existing-chunk.script index df2a7d85..55d73bb0 100755 --- a/tests.branching/edit-checkouts-existing-chunk.script +++ b/tests.branching/edit-checkouts-existing-chunk.script @@ -1,6 +1,6 @@ #!/bin/sh # -# Copyright (C) 2012-2014 Codethink Limited +# Copyright (C) 2012-2015 Codethink Limited # # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by @@ -12,8 +12,7 @@ # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License along -# with this program; if not, write to the Free Software Foundation, Inc., -# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. +# with this program. If not, see <http://www.gnu.org/licenses/>. ## Verify that "morph edit" clones a chunk repository into a system branch. diff --git a/tests.branching/edit-clones-chunk.script b/tests.branching/edit-clones-chunk.script index a6313ca6..710565fb 100755 --- a/tests.branching/edit-clones-chunk.script +++ b/tests.branching/edit-clones-chunk.script @@ -1,6 +1,6 @@ #!/bin/sh # -# Copyright (C) 2012-2014 Codethink Limited +# Copyright (C) 2012-2015 Codethink Limited # # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by @@ -12,8 +12,7 @@ # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License along -# with this program; if not, write to the Free Software Foundation, Inc., -# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. +# with this program. If not, see <http://www.gnu.org/licenses/>. ## Verify that "morph edit" clones a chunk repository into a system branch. diff --git a/tests.branching/edit-clones-chunk.stdout b/tests.branching/edit-clones-chunk.stdout index d0bcb565..9a62092d 100644 --- a/tests.branching/edit-clones-chunk.stdout +++ b/tests.branching/edit-clones-chunk.stdout @@ -13,9 +13,7 @@ test:hello * remote origin Fetch URL: file://TMP/hello Push URL: file://TMP/hello - HEAD branch (remote HEAD is ambiguous, may be one of the following): - alfred - master + HEAD branch: master Remote branches: alfred tracked master tracked diff --git a/tests.branching/edit-handles-submodules.script b/tests.branching/edit-handles-submodules.script index 09592f74..b7a57b42 100755 --- a/tests.branching/edit-handles-submodules.script +++ b/tests.branching/edit-handles-submodules.script @@ -1,6 +1,6 @@ #!/bin/sh # -# Copyright (C) 2012-2014 Codethink Limited +# Copyright (C) 2012-2015 Codethink Limited # # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by @@ -12,8 +12,7 @@ # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License along -# with this program; if not, write to the Free Software Foundation, Inc., -# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. +# with this program. If not, see <http://www.gnu.org/licenses/>. ## 'morph edit' should set up git URL rewriting correctly so that submodule diff --git a/tests.branching/edit-handles-submodules.setup b/tests.branching/edit-handles-submodules.setup index cb61ad66..922afbe3 100755 --- a/tests.branching/edit-handles-submodules.setup +++ b/tests.branching/edit-handles-submodules.setup @@ -1,5 +1,5 @@ #!/bin/sh -# Copyright (C) 2012 Codethink Limited +# Copyright (C) 2012,2015 Codethink Limited # # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by @@ -11,8 +11,7 @@ # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License along -# with this program; if not, write to the Free Software Foundation, Inc., -# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. +# with this program. If not, see <http://www.gnu.org/licenses/>. set -eu diff --git a/tests.branching/edit-updates-stratum.script b/tests.branching/edit-updates-stratum.script index b60c46e7..6d330016 100755 --- a/tests.branching/edit-updates-stratum.script +++ b/tests.branching/edit-updates-stratum.script @@ -1,6 +1,6 @@ #!/bin/sh # -# Copyright (C) 2012-2014 Codethink Limited +# Copyright (C) 2012-2015 Codethink Limited # # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by @@ -12,8 +12,7 @@ # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License along -# with this program; if not, write to the Free Software Foundation, Inc., -# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. +# with this program. If not, see <http://www.gnu.org/licenses/>. ## Verify that "morph edit" clones a chunk repository into a system branch. diff --git a/tests.branching/edit-works-after-branch-root-was-renamed.script b/tests.branching/edit-works-after-branch-root-was-renamed.script index e28ab7df..e2c253f8 100755 --- a/tests.branching/edit-works-after-branch-root-was-renamed.script +++ b/tests.branching/edit-works-after-branch-root-was-renamed.script @@ -1,6 +1,6 @@ #!/bin/sh # -# Copyright (C) 2012-2014 Codethink Limited +# Copyright (C) 2012-2015 Codethink Limited # # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by @@ -12,8 +12,7 @@ # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License along -# with this program; if not, write to the Free Software Foundation, Inc., -# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. +# with this program. If not, see <http://www.gnu.org/licenses/>. ## Verify that the branch root repository created by "morph branch" or diff --git a/tests.branching/foreach-handles-command-failure.script b/tests.branching/foreach-handles-command-failure.script index 4bc71c78..09674c03 100755 --- a/tests.branching/foreach-handles-command-failure.script +++ b/tests.branching/foreach-handles-command-failure.script @@ -1,6 +1,6 @@ #!/bin/sh # -# Copyright (C) 2012-2014 Codethink Limited +# Copyright (C) 2012-2015 Codethink Limited # # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by @@ -12,8 +12,7 @@ # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License along -# with this program; if not, write to the Free Software Foundation, Inc., -# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. +# with this program. If not, see <http://www.gnu.org/licenses/>. ## Verify that "morph foreach" deals with failure in a grown-up way diff --git a/tests.branching/foreach-handles-full-urls.script b/tests.branching/foreach-handles-full-urls.script index 6e0b14ec..8fb18d75 100755 --- a/tests.branching/foreach-handles-full-urls.script +++ b/tests.branching/foreach-handles-full-urls.script @@ -1,6 +1,6 @@ #!/bin/sh # -# Copyright (C) 2012 Codethink Limited +# Copyright (C) 2012,2015 Codethink Limited # # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by @@ -12,8 +12,7 @@ # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License along -# with this program; if not, write to the Free Software Foundation, Inc., -# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. +# with this program. If not, see <http://www.gnu.org/licenses/>. ## morph foreach: should not break if we used a full URL for a repo diff --git a/tests.branching/foreach-handles-full-urls.stdout b/tests.branching/foreach-handles-full-urls.stdout index cee2f70a..6b8728d0 100644 --- a/tests.branching/foreach-handles-full-urls.stdout +++ b/tests.branching/foreach-handles-full-urls.stdout @@ -1,4 +1,5 @@ file://TMP/morphs -# On branch master +On branch master +Your branch is up-to-date with 'origin/master'. nothing to commit, working directory clean diff --git a/tests.branching/init-cwd.script b/tests.branching/init-cwd.script index 10dd0cc7..9613bac8 100755 --- a/tests.branching/init-cwd.script +++ b/tests.branching/init-cwd.script @@ -1,6 +1,6 @@ #!/bin/sh # -# Copyright (C) 2012 Codethink Limited +# Copyright (C) 2012,2015 Codethink Limited # # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by @@ -12,8 +12,7 @@ # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License along -# with this program; if not, write to the Free Software Foundation, Inc., -# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. +# with this program. If not, see <http://www.gnu.org/licenses/>. ## Test that "morph init" works for the current working directory. diff --git a/tests.branching/init-default.script b/tests.branching/init-default.script index da67828f..328f9f84 100755 --- a/tests.branching/init-default.script +++ b/tests.branching/init-default.script @@ -1,6 +1,6 @@ #!/bin/sh # -# Copyright (C) 2012 Codethink Limited +# Copyright (C) 2012,2015 Codethink Limited # # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by @@ -12,8 +12,7 @@ # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License along -# with this program; if not, write to the Free Software Foundation, Inc., -# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. +# with this program. If not, see <http://www.gnu.org/licenses/>. ## Test that "morph init" works without an explicit argument. diff --git a/tests.branching/init-existing.script b/tests.branching/init-existing.script index 506e94bb..c2671ab7 100755 --- a/tests.branching/init-existing.script +++ b/tests.branching/init-existing.script @@ -1,6 +1,6 @@ #!/bin/sh # -# Copyright (C) 2012 Codethink Limited +# Copyright (C) 2012,2015 Codethink Limited # # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by @@ -12,8 +12,7 @@ # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License along -# with this program; if not, write to the Free Software Foundation, Inc., -# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. +# with this program. If not, see <http://www.gnu.org/licenses/>. ## Test that "morph init" works when given the name of an existing, diff --git a/tests.branching/init-newdir.script b/tests.branching/init-newdir.script index 1f505d92..2ebe4325 100755 --- a/tests.branching/init-newdir.script +++ b/tests.branching/init-newdir.script @@ -1,6 +1,6 @@ #!/bin/sh # -# Copyright (C) 2012 Codethink Limited +# Copyright (C) 2012,2015 Codethink Limited # # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by @@ -12,8 +12,7 @@ # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License along -# with this program; if not, write to the Free Software Foundation, Inc., -# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. +# with this program. If not, see <http://www.gnu.org/licenses/>. ## Test that "morph init" works if given a directory that diff --git a/tests.branching/init-nonempty.script b/tests.branching/init-nonempty.script index c5c1947c..a3aba9e6 100755 --- a/tests.branching/init-nonempty.script +++ b/tests.branching/init-nonempty.script @@ -1,6 +1,6 @@ #!/bin/sh # -# Copyright (C) 2012 Codethink Limited +# Copyright (C) 2012,2015 Codethink Limited # # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by @@ -12,8 +12,7 @@ # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License along -# with this program; if not, write to the Free Software Foundation, Inc., -# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. +# with this program. If not, see <http://www.gnu.org/licenses/>. ## Test that "morph init" fails when given the name of an existing, diff --git a/tests.branching/morph-repository-stored-in-cloned-repositories.script b/tests.branching/morph-repository-stored-in-cloned-repositories.script index f60b16ae..baa0868b 100755 --- a/tests.branching/morph-repository-stored-in-cloned-repositories.script +++ b/tests.branching/morph-repository-stored-in-cloned-repositories.script @@ -1,6 +1,6 @@ #!/bin/sh # -# Copyright (C) 2012,2014 Codethink Limited +# Copyright (C) 2012,2014-2015 Codethink Limited # # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by @@ -12,8 +12,7 @@ # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License along -# with this program; if not, write to the Free Software Foundation, Inc., -# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. +# with this program. If not, see <http://www.gnu.org/licenses/>. ## Verify that morph branch/checkout/edit create repositories that have diff --git a/tests.branching/setup b/tests.branching/setup index a2d23090..59979ca8 100755 --- a/tests.branching/setup +++ b/tests.branching/setup @@ -1,5 +1,5 @@ #!/bin/bash -# Copyright (C) 2012-2014 Codethink Limited +# Copyright (C) 2012-2015 Codethink Limited # # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by @@ -11,8 +11,7 @@ # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License along -# with this program; if not, write to the Free Software Foundation, Inc., -# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. +# with this program. If not, see <http://www.gnu.org/licenses/>. # Set up $DATADIR. diff --git a/tests.branching/setup-second-chunk b/tests.branching/setup-second-chunk index 24604ab8..c8baa946 100755 --- a/tests.branching/setup-second-chunk +++ b/tests.branching/setup-second-chunk @@ -1,5 +1,5 @@ #!/bin/sh -# Copyright (C) 2012-2014 Codethink Limited +# Copyright (C) 2012-2015 Codethink Limited # # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by @@ -11,8 +11,7 @@ # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License along -# with this program; if not, write to the Free Software Foundation, Inc., -# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. +# with this program. If not, see <http://www.gnu.org/licenses/>. # Add a second chunk to hello-stratum. diff --git a/tests.branching/show-system-branch-fails-outside-workspace.script b/tests.branching/show-system-branch-fails-outside-workspace.script index d227d5b0..95bdbc92 100755 --- a/tests.branching/show-system-branch-fails-outside-workspace.script +++ b/tests.branching/show-system-branch-fails-outside-workspace.script @@ -1,6 +1,6 @@ #!/bin/sh # -# Copyright (C) 2012 Codethink Limited +# Copyright (C) 2012,2015 Codethink Limited # # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by @@ -12,8 +12,7 @@ # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License along -# with this program; if not, write to the Free Software Foundation, Inc., -# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. +# with this program. If not, see <http://www.gnu.org/licenses/>. ## Check that 'morph show-system-branch' fails when being run diff --git a/tests.branching/show-system-branch-fails-when-branch-is-ambiguous.script b/tests.branching/show-system-branch-fails-when-branch-is-ambiguous.script index 12e23147..bb69eb1c 100755 --- a/tests.branching/show-system-branch-fails-when-branch-is-ambiguous.script +++ b/tests.branching/show-system-branch-fails-when-branch-is-ambiguous.script @@ -1,6 +1,6 @@ #!/bin/sh # -# Copyright (C) 2012 Codethink Limited +# Copyright (C) 2012,2015 Codethink Limited # # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by @@ -12,8 +12,7 @@ # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License along -# with this program; if not, write to the Free Software Foundation, Inc., -# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. +# with this program. If not, see <http://www.gnu.org/licenses/>. ## Check that 'morph show-system-branch' fails when the system branch diff --git a/tests.branching/show-system-branch-works-anywhere-with-a-single-branch.script b/tests.branching/show-system-branch-works-anywhere-with-a-single-branch.script index 800a8e5b..f7d8c7a9 100755 --- a/tests.branching/show-system-branch-works-anywhere-with-a-single-branch.script +++ b/tests.branching/show-system-branch-works-anywhere-with-a-single-branch.script @@ -1,6 +1,6 @@ #!/bin/sh # -# Copyright (C) 2012 Codethink Limited +# Copyright (C) 2012,2015 Codethink Limited # # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by @@ -12,8 +12,7 @@ # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License along -# with this program; if not, write to the Free Software Foundation, Inc., -# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. +# with this program. If not, see <http://www.gnu.org/licenses/>. ## Check that 'morph show-system-branch' works even outside a branch diff --git a/tests.branching/show-system-branch-works-in-different-directories-in-a-branch.script b/tests.branching/show-system-branch-works-in-different-directories-in-a-branch.script index d89e671c..ab46cb08 100755 --- a/tests.branching/show-system-branch-works-in-different-directories-in-a-branch.script +++ b/tests.branching/show-system-branch-works-in-different-directories-in-a-branch.script @@ -1,6 +1,6 @@ #!/bin/sh # -# Copyright (C) 2012 Codethink Limited +# Copyright (C) 2012,2015 Codethink Limited # # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by @@ -12,8 +12,7 @@ # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License along -# with this program; if not, write to the Free Software Foundation, Inc., -# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. +# with this program. If not, see <http://www.gnu.org/licenses/>. ## Check that 'morph show-system-branch' shows the name of the diff --git a/tests.branching/status-in-clean-branch.script b/tests.branching/status-in-clean-branch.script index 335db9f9..a7f38a61 100755 --- a/tests.branching/status-in-clean-branch.script +++ b/tests.branching/status-in-clean-branch.script @@ -1,6 +1,6 @@ #!/bin/sh # -# Copyright (C) 2011, 2012 Codethink Limited +# Copyright (C) 2011, 2012,2015 Codethink Limited # # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by @@ -12,8 +12,7 @@ # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License along -# with this program; if not, write to the Free Software Foundation, Inc., -# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. +# with this program. If not, see <http://www.gnu.org/licenses/>. ## 'morph status' within a branch diff --git a/tests.branching/status-in-dirty-branch.script b/tests.branching/status-in-dirty-branch.script index 37fca97b..5cc8b3a4 100755 --- a/tests.branching/status-in-dirty-branch.script +++ b/tests.branching/status-in-dirty-branch.script @@ -1,6 +1,6 @@ #!/bin/sh # -# Copyright (C) 2011-2014 Codethink Limited +# Copyright (C) 2011-2015 Codethink Limited # # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by @@ -12,8 +12,7 @@ # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License along -# with this program; if not, write to the Free Software Foundation, Inc., -# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. +# with this program. If not, see <http://www.gnu.org/licenses/>. ## 'morph status' within a branch diff --git a/tests.branching/status-in-workspace.script b/tests.branching/status-in-workspace.script index e998c097..9bfeb999 100755 --- a/tests.branching/status-in-workspace.script +++ b/tests.branching/status-in-workspace.script @@ -1,6 +1,6 @@ #!/bin/sh # -# Copyright (C) 2011, 2012 Codethink Limited +# Copyright (C) 2011, 2012,2015 Codethink Limited # # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by @@ -12,8 +12,7 @@ # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License along -# with this program; if not, write to the Free Software Foundation, Inc., -# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. +# with this program. If not, see <http://www.gnu.org/licenses/>. ## 'morph status' within a workspace diff --git a/tests.branching/teardown b/tests.branching/teardown index 94928416..d5d7b603 100755 --- a/tests.branching/teardown +++ b/tests.branching/teardown @@ -1,5 +1,5 @@ #!/bin/sh -# Copyright (C) 2012 Codethink Limited +# Copyright (C) 2012,2015 Codethink Limited # # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by @@ -11,8 +11,7 @@ # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License along -# with this program; if not, write to the Free Software Foundation, Inc., -# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. +# with this program. If not, see <http://www.gnu.org/licenses/>. # Clean up $DATADIR. diff --git a/tests.branching/workspace-not-found.script b/tests.branching/workspace-not-found.script index 9e9b5d75..629fd1e9 100755 --- a/tests.branching/workspace-not-found.script +++ b/tests.branching/workspace-not-found.script @@ -1,6 +1,6 @@ #!/bin/sh # -# Copyright (C) 2012 Codethink Limited +# Copyright (C) 2012,2015 Codethink Limited # # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by @@ -12,8 +12,7 @@ # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License along -# with this program; if not, write to the Free Software Foundation, Inc., -# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. +# with this program. If not, see <http://www.gnu.org/licenses/>. ## Handle being run from outside workspace directory. diff --git a/tests.branching/workspace.script b/tests.branching/workspace.script index e717873c..9da75173 100755 --- a/tests.branching/workspace.script +++ b/tests.branching/workspace.script @@ -1,6 +1,6 @@ #!/bin/sh # -# Copyright (C) 2012 Codethink Limited +# Copyright (C) 2012,2015 Codethink Limited # # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by @@ -12,8 +12,7 @@ # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License along -# with this program; if not, write to the Free Software Foundation, Inc., -# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. +# with this program. If not, see <http://www.gnu.org/licenses/>. ## morph init: Create a workspace. diff --git a/tests.build/ambiguous-refs.script b/tests.build/ambiguous-refs.script index e1eae59d..181af3cc 100755 --- a/tests.build/ambiguous-refs.script +++ b/tests.build/ambiguous-refs.script @@ -1,6 +1,6 @@ #!/bin/sh # -# Copyright (C) 2011-2013 Codethink Limited +# Copyright (C) 2011-2013,2015 Codethink Limited # # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by @@ -12,8 +12,7 @@ # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License along -# with this program; if not, write to the Free Software Foundation, Inc., -# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. +# with this program. If not, see <http://www.gnu.org/licenses/>. ## Guard against a bug that occurs if 'git show-ref' is used to resolve refs diff --git a/tests.build/build-chunk-failures-dump-log.script b/tests.build/build-chunk-failures-dump-log.script index 645fd59a..81361ad0 100755 --- a/tests.build/build-chunk-failures-dump-log.script +++ b/tests.build/build-chunk-failures-dump-log.script @@ -1,6 +1,6 @@ #!/bin/bash # -# Copyright (C) 2011-2014 Codethink Limited +# Copyright (C) 2011-2015 Codethink Limited # # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by @@ -12,8 +12,7 @@ # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License along -# with this program; if not, write to the Free Software Foundation, Inc., -# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. +# with this program. If not, see <http://www.gnu.org/licenses/>. ## Test building a chunk that fails. diff --git a/tests.build/build-chunk-writes-log.script b/tests.build/build-chunk-writes-log.script index 5bfb2ae3..e636924e 100755 --- a/tests.build/build-chunk-writes-log.script +++ b/tests.build/build-chunk-writes-log.script @@ -1,6 +1,6 @@ #!/bin/sh # -# Copyright (C) 2011-2013 Codethink Limited +# Copyright (C) 2011-2013,2015 Codethink Limited # # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by @@ -12,8 +12,7 @@ # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License along -# with this program; if not, write to the Free Software Foundation, Inc., -# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. +# with this program. If not, see <http://www.gnu.org/licenses/>. ## Build log should be saved when a chunk is built. diff --git a/tests.build/build-stratum-with-submodules.script b/tests.build/build-stratum-with-submodules.script index 015db3f2..bd6b97ce 100755 --- a/tests.build/build-stratum-with-submodules.script +++ b/tests.build/build-stratum-with-submodules.script @@ -1,6 +1,6 @@ #!/bin/sh # -# Copyright (C) 2011-2014 Codethink Limited +# Copyright (C) 2011-2015 Codethink Limited # # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by @@ -12,8 +12,7 @@ # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License along -# with this program; if not, write to the Free Software Foundation, Inc., -# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. +# with this program. If not, see <http://www.gnu.org/licenses/>. ## Test build a stratum that uses a chunk which needs a submodule. diff --git a/tests.build/build-system-autotools-fails-if-autogen-fails.script b/tests.build/build-system-autotools-fails-if-autogen-fails.script index d7fdd055..0b009b54 100755 --- a/tests.build/build-system-autotools-fails-if-autogen-fails.script +++ b/tests.build/build-system-autotools-fails-if-autogen-fails.script @@ -1,6 +1,6 @@ #!/bin/sh # -# Copyright (C) 2012-2013 Codethink Limited +# Copyright (C) 2012-2013,2015 Codethink Limited # # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by @@ -12,8 +12,7 @@ # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License along -# with this program; if not, write to the Free Software Foundation, Inc., -# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. +# with this program. If not, see <http://www.gnu.org/licenses/>. ## Test that the autotools build system fails if it runs autogen.sh and that diff --git a/tests.build/build-system-autotools.script b/tests.build/build-system-autotools.script index 2ea53174..710a8f98 100755 --- a/tests.build/build-system-autotools.script +++ b/tests.build/build-system-autotools.script @@ -1,6 +1,6 @@ #!/bin/sh # -# Copyright (C) 2011-2014 Codethink Limited +# Copyright (C) 2011-2015 Codethink Limited # # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by @@ -12,8 +12,7 @@ # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License along -# with this program; if not, write to the Free Software Foundation, Inc., -# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. +# with this program. If not, see <http://www.gnu.org/licenses/>. ## Convert the hello-chunk project to something autotools-like, then diff --git a/tests.build/build-system-cmake.script b/tests.build/build-system-cmake.script index 570a9af7..fe02f9dc 100755 --- a/tests.build/build-system-cmake.script +++ b/tests.build/build-system-cmake.script @@ -1,6 +1,6 @@ #!/bin/sh # -# Copyright (C) 2011-2014 Codethink Limited +# Copyright (C) 2011-2015 Codethink Limited # # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by @@ -12,8 +12,7 @@ # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License along -# with this program; if not, write to the Free Software Foundation, Inc., -# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. +# with this program. If not, see <http://www.gnu.org/licenses/>. ## Convert the hello-chunk project to something cmake-like, then diff --git a/tests.build/build-system-cpan.script b/tests.build/build-system-cpan.script index 735dac84..103d5466 100755 --- a/tests.build/build-system-cpan.script +++ b/tests.build/build-system-cpan.script @@ -1,6 +1,6 @@ #!/bin/sh # -# Copyright (C) 2011-2014 Codethink Limited +# Copyright (C) 2011-2015 Codethink Limited # # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by @@ -12,8 +12,7 @@ # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License along -# with this program; if not, write to the Free Software Foundation, Inc., -# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. +# with this program. If not, see <http://www.gnu.org/licenses/>. ## Convert the hello-chunk project to perl with CPAN and build. diff --git a/tests.build/build-system-python-distutils.script b/tests.build/build-system-python-distutils.script index 9a751491..e5c0ea74 100755 --- a/tests.build/build-system-python-distutils.script +++ b/tests.build/build-system-python-distutils.script @@ -1,6 +1,6 @@ #!/bin/sh # -# Copyright (C) 2011-2014 Codethink Limited +# Copyright (C) 2011-2015 Codethink Limited # # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by @@ -12,8 +12,7 @@ # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License along -# with this program; if not, write to the Free Software Foundation, Inc., -# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. +# with this program. If not, see <http://www.gnu.org/licenses/>. ## Convert the hello-chunk project to python with distutils and build. diff --git a/tests.build/build-system-qmake.script b/tests.build/build-system-qmake.script index b3861936..d430fba7 100755 --- a/tests.build/build-system-qmake.script +++ b/tests.build/build-system-qmake.script @@ -1,6 +1,6 @@ #!/bin/sh # -# Copyright (C) 2011-2014 Codethink Limited +# Copyright (C) 2011-2015 Codethink Limited # # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by @@ -12,8 +12,7 @@ # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License along -# with this program; if not, write to the Free Software Foundation, Inc., -# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. +# with this program. If not, see <http://www.gnu.org/licenses/>. ## Convert the hello-chunk project to something qmake-like, then diff --git a/tests.build/build-system.script b/tests.build/build-system.script index 56d80735..0180939a 100755 --- a/tests.build/build-system.script +++ b/tests.build/build-system.script @@ -1,6 +1,6 @@ #!/bin/sh # -# Copyright (C) 2011-2014 Codethink Limited +# Copyright (C) 2011-2015 Codethink Limited # # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by @@ -12,8 +12,7 @@ # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License along -# with this program; if not, write to the Free Software Foundation, Inc., -# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. +# with this program. If not, see <http://www.gnu.org/licenses/>. ## Test building a simple system. diff --git a/tests.build/cross-bootstrap-only-to-supported-archs.script b/tests.build/cross-bootstrap-only-to-supported-archs.script index 872acd9f..3b19b910 100755 --- a/tests.build/cross-bootstrap-only-to-supported-archs.script +++ b/tests.build/cross-bootstrap-only-to-supported-archs.script @@ -1,6 +1,6 @@ #!/bin/bash # -# Copyright (C) 2013 Codethink Limited +# Copyright (C) 2013,2015 Codethink Limited # # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by @@ -12,8 +12,7 @@ # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License along -# with this program; if not, write to the Free Software Foundation, Inc., -# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. +# with this program. If not, see <http://www.gnu.org/licenses/>. # Test that "morph cross-bootstrap" works only for the architectures that diff --git a/tests.build/cross-bootstrap.script b/tests.build/cross-bootstrap.script index 51d9ef1e..245c2a13 100755 --- a/tests.build/cross-bootstrap.script +++ b/tests.build/cross-bootstrap.script @@ -1,6 +1,6 @@ #!/bin/bash # -# Copyright (C) 2013 Codethink Limited +# Copyright (C) 2013,2015 Codethink Limited # # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by @@ -12,8 +12,7 @@ # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License along -# with this program; if not, write to the Free Software Foundation, Inc., -# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. +# with this program. If not, see <http://www.gnu.org/licenses/>. # Test "morph cross-bootstrap", up to the point of the tarball it generates diff --git a/tests.build/empty-stratum.exit b/tests.build/empty-stratum.exit deleted file mode 100644 index d00491fd..00000000 --- a/tests.build/empty-stratum.exit +++ /dev/null @@ -1 +0,0 @@ -1 diff --git a/tests.build/empty-stratum.script b/tests.build/empty-stratum.script deleted file mode 100755 index 19c36558..00000000 --- a/tests.build/empty-stratum.script +++ /dev/null @@ -1,36 +0,0 @@ -#!/bin/sh -# -# Copyright (C) 2013-2014 Codethink Limited -# -# This program is free software; you can redistribute it and/or modify -# it under the terms of the GNU General Public License as published by -# the Free Software Foundation; version 2 of the License. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License along -# with this program; if not, write to the Free Software Foundation, Inc., -# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. - -set -eu - -morphsrepo="$DATADIR/morphs-repo" -cd "$morphsrepo" - -git checkout --quiet -b empty-stratum - -# Create empty stratum to test S4585 -cat <<EOF > hello-stratum.morph -name: hello-stratum -kind: stratum -EOF -sed -i 's/master/empty-stratum/' hello-system.morph -git add hello-stratum.morph hello-system.morph - -git commit --quiet -m "add empty stratum" - -"$SRCDIR/scripts/test-morph" build-morphology \ - test:morphs-repo empty-stratum hello-system diff --git a/tests.build/empty-stratum.stderr b/tests.build/empty-stratum.stderr deleted file mode 100644 index 6a4ecb05..00000000 --- a/tests.build/empty-stratum.stderr +++ /dev/null @@ -1 +0,0 @@ -ERROR: Stratum hello-stratum has no chunks in string diff --git a/tests.build/missing-ref.script b/tests.build/missing-ref.script index a18ce2d1..83562035 100755 --- a/tests.build/missing-ref.script +++ b/tests.build/missing-ref.script @@ -1,6 +1,6 @@ #!/bin/sh # -# Copyright (C) 2011-2013 Codethink Limited +# Copyright (C) 2011-2013,2015 Codethink Limited # # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by @@ -12,8 +12,7 @@ # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License along -# with this program; if not, write to the Free Software Foundation, Inc., -# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. +# with this program. If not, see <http://www.gnu.org/licenses/>. ## Test building with a bad reference. diff --git a/tests.build/missing-ref.stderr b/tests.build/missing-ref.stderr index 5fa5456b..b5139e25 100644 --- a/tests.build/missing-ref.stderr +++ b/tests.build/missing-ref.stderr @@ -1 +1 @@ -ERROR: Ref non-existent-branch is an invalid reference for repo file://TMP/morphs-repo +ERROR: Git directory TMP/morphs-repo has no commit at ref non-existent-branch^{commit}. diff --git a/tests.build/morphless-chunks.script b/tests.build/morphless-chunks.script index 9a8b41dd..5b19bc4a 100755 --- a/tests.build/morphless-chunks.script +++ b/tests.build/morphless-chunks.script @@ -1,6 +1,6 @@ #!/bin/sh # -# Copyright (C) 2012-2014 Codethink Limited +# Copyright (C) 2012-2015 Codethink Limited # # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by @@ -12,8 +12,7 @@ # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License along -# with this program; if not, write to the Free Software Foundation, Inc., -# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. +# with this program. If not, see <http://www.gnu.org/licenses/>. ## Try to build a morphless chunk. diff --git a/tests.build/only-build-systems.script b/tests.build/only-build-systems.script index 699be942..ae664f7e 100755 --- a/tests.build/only-build-systems.script +++ b/tests.build/only-build-systems.script @@ -1,6 +1,6 @@ #!/bin/sh # -# Copyright (C) 2013 Codethink Limited +# Copyright (C) 2013,2015 Codethink Limited # # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by @@ -12,8 +12,7 @@ # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License along -# with this program; if not, write to the Free Software Foundation, Inc., -# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. +# with this program. If not, see <http://www.gnu.org/licenses/>. ## Morph should refuse build a chunk out of the context. Only diff --git a/tests.build/prefix.script b/tests.build/prefix.script index 209c1a54..140617e1 100755 --- a/tests.build/prefix.script +++ b/tests.build/prefix.script @@ -1,6 +1,6 @@ #!/bin/sh # -# Copyright (C) 2013-2014 Codethink Limited +# Copyright (C) 2013-2015 Codethink Limited # # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by @@ -12,8 +12,7 @@ # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License along -# with this program; if not, write to the Free Software Foundation, Inc., -# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. +# with this program. If not, see <http://www.gnu.org/licenses/>. ## Honour 'prefix' attribute for chunks within stratum morphs diff --git a/tests.build/rebuild-cached-stratum.script b/tests.build/rebuild-cached-stratum.script index 0014e545..e2e0face 100755 --- a/tests.build/rebuild-cached-stratum.script +++ b/tests.build/rebuild-cached-stratum.script @@ -1,6 +1,6 @@ #!/bin/sh # -# Copyright (C) 2011-2014 Codethink Limited +# Copyright (C) 2011-2015 Codethink Limited # # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by @@ -12,8 +12,7 @@ # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License along -# with this program; if not, write to the Free Software Foundation, Inc., -# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. +# with this program. If not, see <http://www.gnu.org/licenses/>. ## Does a cached stratum get rebuilt if its chunk changes? diff --git a/tests.build/setup b/tests.build/setup index 833f132d..4663ff1d 100755 --- a/tests.build/setup +++ b/tests.build/setup @@ -10,7 +10,7 @@ # The stratum repository contains a single branch, "master", with a # stratum and a system morphology that include the chunk above. # -# Copyright (C) 2011-2014 Codethink Limited +# Copyright (C) 2011-2015 Codethink Limited # # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by @@ -22,8 +22,7 @@ # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License along -# with this program; if not, write to the Free Software Foundation, Inc., -# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. +# with this program. If not, see <http://www.gnu.org/licenses/>. set -eu diff --git a/tests.build/setup-build-essential b/tests.build/setup-build-essential index 9ffb7774..281ff7ec 100755 --- a/tests.build/setup-build-essential +++ b/tests.build/setup-build-essential @@ -1,6 +1,6 @@ #!/bin/sh # -# Copyright (C) 2013-2014 Codethink Limited +# Copyright (C) 2013-2015 Codethink Limited # # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by @@ -12,8 +12,7 @@ # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License along -# with this program; if not, write to the Free Software Foundation, Inc., -# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. +# with this program. If not, see <http://www.gnu.org/licenses/>. # Set up a stratum which resembles Baserock's 'build-essential' slightly. Used # for testing 'morph cross-bootstrap' and the 'bootstrap' build mode. diff --git a/tests.build/uses-tempdir.script b/tests.build/uses-tempdir.script index 80c06d56..aa4563f8 100755 --- a/tests.build/uses-tempdir.script +++ b/tests.build/uses-tempdir.script @@ -1,6 +1,6 @@ #!/bin/sh # -# Copyright (C) 2011-2013 Codethink Limited +# Copyright (C) 2011-2013,2015 Codethink Limited # # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by @@ -12,8 +12,7 @@ # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License along -# with this program; if not, write to the Free Software Foundation, Inc., -# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. +# with this program. If not, see <http://www.gnu.org/licenses/>. ## Test that temporary directories are created in the dir specified diff --git a/tests/setup b/tests/setup index 6ebab880..9010921c 100755 --- a/tests/setup +++ b/tests/setup @@ -10,7 +10,7 @@ # The stratum repository contains a single branch, "master", with a # stratum and a system morphology that include the chunk above. # -# Copyright (C) 2011-2014 Codethink Limited +# Copyright (C) 2011-2015 Codethink Limited # # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by @@ -22,8 +22,7 @@ # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License along -# with this program; if not, write to the Free Software Foundation, Inc., -# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. +# with this program. If not, see <http://www.gnu.org/licenses/>. set -eu diff --git a/tests/show-dependencies.script b/tests/show-dependencies.script index 15b69e25..db76274d 100755 --- a/tests/show-dependencies.script +++ b/tests/show-dependencies.script @@ -1,6 +1,6 @@ #!/bin/bash # -# Copyright (C) 2012-2013 Codethink Limited +# Copyright (C) 2012-2013,2015 Codethink Limited # # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by @@ -12,8 +12,7 @@ # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License along -# with this program; if not, write to the Free Software Foundation, Inc., -# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. +# with this program. If not, see <http://www.gnu.org/licenses/>. ## Test "show-dependencies" subcommand. diff --git a/tests/show-dependencies.setup b/tests/show-dependencies.setup index 74d10c2b..38a76722 100755 --- a/tests/show-dependencies.setup +++ b/tests/show-dependencies.setup @@ -1,6 +1,6 @@ #!/bin/bash # -# Copyright (C) 2012-2014 Codethink Limited +# Copyright (C) 2012-2015 Codethink Limited # # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by @@ -12,8 +12,7 @@ # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License along -# with this program; if not, write to the Free Software Foundation, Inc., -# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. +# with this program. If not, see <http://www.gnu.org/licenses/>. set -e diff --git a/tests/trove-id.script b/tests/trove-id.script index 998bde44..ee382de1 100755 --- a/tests/trove-id.script +++ b/tests/trove-id.script @@ -1,6 +1,6 @@ #!/bin/sh # -# Copyright (C) 2012-2013 Codethink Limited +# Copyright (C) 2012-2013,2015 Codethink Limited # # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by @@ -12,8 +12,7 @@ # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License along -# with this program; if not, write to the Free Software Foundation, Inc., -# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. +# with this program. If not, see <http://www.gnu.org/licenses/>. ## Verify that trove-id (and by corollary trove-host) work properly. diff --git a/without-test-modules b/without-test-modules index 55e5291d..de9a377a 100644 --- a/without-test-modules +++ b/without-test-modules @@ -52,3 +52,6 @@ distbuild/timer_event_source.py distbuild/worker_build_scheduler.py # Not unit tested, since it needs a full system branch morphlib/buildbranch.py + +# External +morphlib/pylru.py diff --git a/yarns/branches-workspaces.yarn b/yarns/branches-workspaces.yarn index 34aa97e0..27bddff5 100644 --- a/yarns/branches-workspaces.yarn +++ b/yarns/branches-workspaces.yarn @@ -233,6 +233,7 @@ build branch is made to include that change. WHEN the user makes changes to test-chunk in branch master AND the user builds systems/test-system.morph of the master branch THEN the changes to test-chunk in branch master are included in the temporary build branch + FINALLY the git server is shut down ### When branches are created ### @@ -444,21 +445,8 @@ repositories referenced in the system branch. THEN morph reports no outstanding changes in foo FINALLY the git server is shut down -`morph foreach` --------------- - -`morph foreach` runs a shell command in each of the git repos in a system -branch checkout. - - SCENARIO morph foreach runs command in each git repo - GIVEN a workspace - AND a git server - WHEN the user creates a system branch called foo - AND the user edits the chunk test-chunk in branch foo - AND running shell command in each repo in foo - THEN morph ran command in test/morphs in foo - AND morph ran command in test/test-chunk in foo - FINALLY the git server is shut down +Manifests +--------- Generating a manifest works diff --git a/yarns/building.yarn b/yarns/building.yarn index 253b3b3c..b5e46b73 100644 --- a/yarns/building.yarn +++ b/yarns/building.yarn @@ -6,7 +6,35 @@ Morph Building Tests AND a git server WHEN the user checks out the system branch called master AND the user creates an uncommitted system morphology called systems/base-system.morph for our architecture in system branch master - THEN morph build the system systems/base-system.morph of the branch master + AND the user attempts to build the system systems/base-system.morph in branch master + THEN morph succeeded + FINALLY the git server is shut down + + SCENARIO build system with relative path + GIVEN a workspace + AND a git server + WHEN the user checks out the system branch called master + AND the user creates an uncommitted system morphology called systems/base-system.morph for our architecture in system branch master + AND from the directory workspace/master/test/morphs/systems the user attempts to morph build the system base-system.morph + THEN morph succeeded + FINALLY the git server is shut down + + SCENARIO build system with relative path (second variant) + GIVEN a workspace + AND a git server + WHEN the user checks out the system branch called master + AND the user creates an uncommitted system morphology called systems/base-system.morph for our architecture in system branch master + AND from the directory workspace/master/test/morphs/systems the user attempts to morph build the system ../systems/base-system.morph + THEN morph succeeded + FINALLY the git server is shut down + + SCENARIO build system with absolute path + GIVEN a workspace + AND a git server + WHEN the user checks out the system branch called master + AND the user creates an uncommitted system morphology called systems/base-system.morph for our architecture in system branch master + AND from the directory workspace/master/test/morphs/systems the user attempts to morph build the system using the absolute path to base-system.morph + THEN morph succeeded FINALLY the git server is shut down System integrations @@ -35,6 +63,7 @@ so when we deploy the system, we can check whether it exists. WHEN the user attempts to deploy the cluster test-cluster.morph in branch master with options test-system.location="$DATADIR/test.tar" THEN morph succeeded AND tarball test.tar contains etc/passwd + FINALLY the git server is shut down Distbuilding ------------ @@ -72,3 +101,14 @@ repos cached locally. AND the distbuild worker is terminated AND the communal cache server is terminated AND the git server is shut down + +Empty strata don't build +------------------------ + + SCENARIO empty-strata + GIVEN a workspace + AND a git server + WHEN the user checks out the system branch called empty-stratum + AND the user attempts to build the system systems/empty-stratum-system.morph in branch empty-stratum + THEN morph failed + FINALLY the git server is shut down diff --git a/yarns/deployment.yarn b/yarns/deployment.yarn index 0782c7c1..6ec8c0af 100644 --- a/yarns/deployment.yarn +++ b/yarns/deployment.yarn @@ -14,13 +14,35 @@ Morph Deployment Tests GIVEN a workspace AND a git server WHEN the user checks out the system branch called master - GIVEN a cluster called test-cluster.morph in system branch master - AND a system in cluster test-cluster.morph in branch master called test-system - AND system test-system in cluster test-cluster.morph in branch master builds systems/test-system.morph - AND system test-system in cluster test-cluster.morph in branch master has deployment type: tar - AND system test-system in cluster test-cluster.morph in branch master has deployment location: test.tar - WHEN the user builds the system systems/test-system.morph in branch master - AND the user attempts to deploy the cluster test-cluster.morph in branch master + AND the user builds the system systems/test-system.morph in branch master + AND the user attempts to deploy the cluster clusters/test-cluster.morph in branch master + THEN morph succeeded + FINALLY the git server is shut down + + SCENARIO deploying a cluster using a relative path + GIVEN a workspace + AND a git server + WHEN the user checks out the system branch called master + AND the user builds the system systems/test-system.morph in branch master + AND from directory workspace/master/test/morphs/clusters the user attempts to deploy the cluster test-cluster.morph in branch master + THEN morph succeeded + FINALLY the git server is shut down + + SCENARIO deploying a cluster using a relative path (second variant) + GIVEN a workspace + AND a git server + WHEN the user checks out the system branch called master + AND the user builds the system systems/test-system.morph in branch master + AND from directory workspace/master/test/morphs/clusters the user attempts to deploy the cluster ../clusters/test-cluster.morph in branch master + THEN morph succeeded + FINALLY the git server is shut down + + SCENARIO deploying a cluster using an absolute path + GIVEN a workspace + AND a git server + WHEN the user checks out the system branch called master + AND the user builds the system systems/test-system.morph in branch master + AND from directory workspace/master/test/morphs/clusters the user attempts to deploy the cluster using the absolute path to test-cluster.morph in branch master THEN morph succeeded FINALLY the git server is shut down @@ -31,13 +53,8 @@ this clear. GIVEN a workspace AND a git server WHEN the user checks out the system branch called master - GIVEN a cluster called test-cluster.morph in system branch master - AND a system in cluster test-cluster.morph in branch master called test-system - AND system test-system in cluster test-cluster.morph in branch master builds systems/test-system.morph - AND system test-system in cluster test-cluster.morph in branch master has deployment type: tar - AND system test-system in cluster test-cluster.morph in branch master has deployment location: test.tar - WHEN the user builds the system systems/test-system.morph in branch master - AND the user attempts to upgrade the cluster test-cluster.morph in branch master + AND the user builds the system systems/test-system.morph in branch master + AND the user attempts to upgrade the cluster clusters/test-cluster.morph in branch master THEN morph failed FINALLY the git server is shut down @@ -328,3 +345,4 @@ Once it is rebuilt, it can be deployed. WHEN the user attempts to deploy the cluster test-cluster.morph in branch mybranch THEN morph succeeded AND file workspace/mybranch/test/morphs/test-system.tar exists + FINALLY the git server is shut down diff --git a/yarns/implementations.yarn b/yarns/implementations.yarn index 6110148e..2bbb1f5c 100644 --- a/yarns/implementations.yarn +++ b/yarns/implementations.yarn @@ -34,7 +34,8 @@ we can test it later in a THEN step. case $(cat "$DATADIR/morph-exit") in 0) echo "Morph succeeded!" ;; - *) die "Morph should have succeeded, but didn't. Unexpected failure!" + *) cat "$DATADIR/result-latest" >&2 + die "Morph should have succeeded, but didn't. Unexpected failure!" ;; esac @@ -246,6 +247,19 @@ another to hold a chunk. morph: strata/core.morph EOF + mkdir "$DATADIR/gits/morphs/clusters" + cd "$DATADIR/gits/morphs" + install -m644 -D /dev/stdin << EOF "clusters/test-cluster.morph" + name: test-cluster + kind: cluster + systems: + - morph: systems/test-system.morph + deploy: + test-system: + type: tar + location: test.tar + EOF + install -m644 -D /dev/stdin << EOF "strata/build-essential.morph" name: build-essential kind: stratum @@ -322,6 +336,32 @@ another to hold a chunk. git commit -m Initial. git tag -a "test-tag" -m "Tagging test-tag" + # A new branch is created here as the presence of an empty stratum will + # break any morph commands which load all definitions in the repository. + git checkout -b empty-stratum + + install -m644 -D /dev/stdin << EOF "systems/empty-stratum-system.morph" + name: empty-stratum-system + kind: system + arch: $arch + strata: + - name: build-essential + morph: strata/build-essential.morph + - name: core + morph: strata/core.morph + - name: empty + morph: strata/empty.morph + EOF + + install -m644 -D /dev/stdin << EOF "strata/empty.morph" + name: empty + kind: stratum + EOF + + git add . + git commit -m 'Add an empty stratum' + git checkout master + # Start a git daemon to serve our git repositories port_file="$DATADIR/git-daemon-port" pid_file="$DATADIR/git-daemon-pid" @@ -584,16 +624,6 @@ Reporting status of checked out repositories: cd "$DATADIR/workspace/$MATCH_2/$(slashify_colons "$MATCH_1")" git commit -a -m test-commit -Running shell command in each checked out repository: - - IMPLEMENTS WHEN running shell command in each repo in (\S+) - cd "$DATADIR/workspace/$MATCH_1" - run_morph foreach -- pwd > "$DATADIR/morph.stdout" - - IMPLEMENTS THEN morph ran command in (\S+) in (\S+) - grep -Fx "$MATCH_1" "$DATADIR/morph.stdout" - grep -Fx "$DATADIR/workspace/$MATCH_2/$MATCH_1" "$DATADIR/morph.stdout" - Generating a manifest. IMPLEMENTS GIVEN a system artifact @@ -702,7 +732,7 @@ Implementation sections for building ==================================== IMPLEMENTS WHEN the user (attempts to )?((dist)?build)s? the system (\S+) in branch (\S+) - cd "$DATADIR/workspace/$MATCH_5" + cd "$DATADIR/workspace/$MATCH_5/test/morphs" set "$MATCH_2" "$MATCH_4" if [ "$MATCH_1" != "attempts to " ]; then run_morph "$@" else attempt_morph "$@"; fi @@ -711,7 +741,7 @@ Implementation sections for cross-bootstraping ============================================== IMPLEMENTS THEN the user cross-bootstraps the system (\S+) in branch (\S+) of repo (\S+) to the arch (\S+) - cd "$DATADIR/workspace/$MATCH_2" + cd "$DATADIR/workspace/$MATCH_2/test/morphs" set -- cross-bootstrap "$MATCH_4" "$MATCH_3" "$MATCH_2" "$MATCH_1" run_morph "$@" @@ -722,32 +752,44 @@ Defaults are set in the cluster morphology, so we can deploy without setting any extra parameters, but we also need to be able to override them, so they can be added to the end of the implements section. - IMPLEMENTS WHEN the user (attempts to deploy|deploys) the (system|cluster) (\S+) in branch (\S+)( with options (.*))? - cd "$DATADIR/workspace/$MATCH_4" - set -- deploy "$MATCH_3" - if [ "$MATCH_5" != '' ]; then + IMPLEMENTS WHEN (from directory (\S+) )?the user (attempts to deploy|deploys) the (system|cluster) (using the absolute path to )?(\S+) in branch (\S+)( with options (.*))? + if [ "$MATCH_1" != "" ] + then + cd "$DATADIR/$MATCH_2" + else + cd "$DATADIR/workspace/$MATCH_7/test/morphs" + fi + + if [ "$MATCH_5" != "" ] + then + set -- deploy "$(pwd)/$MATCH_6" + else + set -- deploy "$MATCH_6" + fi + + if [ "$MATCH_8" != '' ]; then # eval used so word splitting in the text is preserved - eval set -- '"$@"' $MATCH_6 + eval set -- '"$@"' $MATCH_9 fi - if [ $MATCH_1 == "deploys" ]; then run_morph "$@" + if [ "$MATCH_3" = "deploys" ]; then run_morph "$@" else attempt_morph "$@"; fi IMPLEMENTS WHEN the user (attempts to deploy|deploys) (.*) from cluster (\S+) in branch (\S+) - cd "$DATADIR/workspace/$MATCH_4" + cd "$DATADIR/workspace/$MATCH_4/test/morphs" set -- deploy "$MATCH_3" systems=$(echo "$MATCH_2" | sed -e 's/, /\n/g' -e 's/ and /\n/g') set -- "$@" $systems - if [ $MATCH_1 == "deploys" ]; then run_morph "$@" + if [ "$MATCH_1" = "deploys" ]; then run_morph "$@" else attempt_morph "$@"; fi IMPLEMENTS WHEN the user (attempts to upgrade|upgrades) the (system|cluster) (\S+) in branch (\S+)( with options (.*))? - cd "$DATADIR/workspace/$MATCH_4" + cd "$DATADIR/workspace/$MATCH_4/test/morphs" set -- upgrade "$MATCH_3" if [ "$MATCH_5" != '' ]; then # eval used so word splitting in the text is preserved eval set -- '"$@"' $MATCH_6 fi - if [ $MATCH_1 == "upgrades" ]; then run_morph "$@" + if [ "$MATCH_1" = "upgrades" ]; then run_morph "$@" else attempt_morph "$@"; fi Implementations sections for reading error messages @@ -891,13 +933,21 @@ Implementations for building systems ------------------------------------ IMPLEMENTS THEN morph ((dist)?build) the system (\S+) of the (branch|tag) (\S+) - cd "$DATADIR/workspace/$MATCH_5" + cd "$DATADIR/workspace/$MATCH_5/test/morphs" run_morph "$MATCH_1" "$MATCH_3" IMPLEMENTS WHEN the user builds (\S+) of the (\S+) (branch|tag) - cd "$DATADIR/workspace/$MATCH_2" + cd "$DATADIR/workspace/$MATCH_2/test/morphs" run_morph build "$MATCH_1" + IMPLEMENTS WHEN from the directory (\S+) the user attempts to morph build the system (\S+) + cd "$DATADIR/$MATCH_1" + attempt_morph build "$MATCH_2" + + IMPLEMENTS WHEN from the directory (\S+) the user attempts to morph build the system using the absolute path to (\S+) + cd "$DATADIR/$MATCH_1" + attempt_morph build "$(pwd)/$MATCH_2" + Implementations for tarball inspection -------------------------------------- diff --git a/yarns/morph.shell-lib b/yarns/morph.shell-lib index 9c13e449..e7011091 100644 --- a/yarns/morph.shell-lib +++ b/yarns/morph.shell-lib @@ -3,7 +3,7 @@ # The shell functions in this library are meant to make writing IMPLEMENTS # sections for yarn scenario tests easier. -# Copyright (C) 2013-2014 Codethink Limited +# Copyright (C) 2013-2015 Codethink Limited # # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by @@ -15,8 +15,7 @@ # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License along -# with this program; if not, write to the Free Software Foundation, Inc., -# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. +# with this program. If not, see <http://www.gnu.org/licenses/>. # Add $SRCDIR to PYTHONPATH. @@ -39,9 +38,13 @@ run_morph() set +e "$SRCDIR"/morph --verbose \ --cachedir-min-space=0 --tempdir-min-space=0 \ - --no-default-config --config "$DATADIR/morph.conf" "$@" \ - 2> "$DATADIR/result-$1" > "$DATADIR/out-$1" + --no-default-config --config "$DATADIR/morph.conf" \ + --log="$DATADIR/log-$1" \ + "$@" 2> "$DATADIR/result-$1" > "$DATADIR/out-$1" local exit_code="$?" + for o in log result out; do + ln -sf "$o-$1" "$DATADIR/$o-latest" + done cat "$DATADIR/out-$1" cat "$DATADIR/result-$1" >&2 return "$exit_code" |