# Copyright (C) 2012 Codethink Limited # # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by # the Free Software Foundation; version 2 of the License. # # This program is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License along # with this program; if not, write to the Free Software Foundation, Inc., # 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. import cliapp import copy import glob import json import logging import os import shutil import socket import tempfile import time import urlparse import uuid import morphlib class BranchAndMergePlugin(cliapp.Plugin): def __init__(self): # Start recording changes. self.init_changelog() def enable(self): # User-facing commands self.app.add_subcommand('init', self.init, arg_synopsis='[DIR]') self.app.add_subcommand('branch', self.branch, arg_synopsis='REPO NEW [OLD]') self.app.add_subcommand('checkout', self.checkout, arg_synopsis='REPO BRANCH') self.app.add_subcommand('merge', self.merge, arg_synopsis='BRANCH') self.app.add_subcommand('edit', self.edit, arg_synopsis='SYSTEM STRATUM [CHUNK]') self.app.add_subcommand('petrify', self.petrify) self.app.add_subcommand('unpetrify', self.unpetrify) self.app.add_subcommand('build', self.build, arg_synopsis='SYSTEM') self.app.add_subcommand('status', self.status) # Advanced commands self.app.add_subcommand('foreach', self.foreach, arg_synopsis='-- COMMAND') # Plumbing commands (FIXME: should be hidden from --help by default) self.app.add_subcommand('workspace', self.workspace, arg_synopsis='') self.app.add_subcommand('show-system-branch', self.show_system_branch, arg_synopsis='') self.app.add_subcommand('show-branch-root', self.show_branch_root, arg_synopsis='') def disable(self): pass def init_changelog(self): self.changelog = {} def log_change(self, repo, text): if not repo in self.changelog: self.changelog[repo] = [] self.changelog[repo].append(text) def print_changelog(self, title, early_keys=[]): if self.changelog and self.app.settings['verbose']: msg = '\n%s:\n\n' % title keys = [x for x in early_keys if x in self.changelog] keys.extend([x for x in self.changelog if x not in early_keys]) for key in keys: messages = self.changelog[key] msg += ' %s:\n' % key msg += '\n'.join([' %s' % x for x in messages]) msg += '\n\n' self.app.output.write(msg) @staticmethod def deduce_workspace(): dirname = os.getcwd() while dirname != '/': dot_morph = os.path.join(dirname, '.morph') if os.path.isdir(dot_morph): return dirname dirname = os.path.dirname(dirname) raise cliapp.AppException("Can't find the workspace directory") def deduce_system_branch(self): # 1. Deduce the workspace. If this fails, we're not inside a workspace. workspace = self.deduce_workspace() # 2. We're in a workspace. Check if we're inside a system branch. # If we are, return its name. dirname = os.getcwd() while dirname != workspace and dirname != '/': if os.path.isdir(os.path.join(dirname, '.morph-system-branch')): branch_name = self.get_branch_config(dirname, 'branch.name') return branch_name, dirname dirname = os.path.dirname(dirname) # 3. We're in a workspace but not inside a branch. Try to find a # branch directory in the directories below the current working # directory. Avoid ambiguity by only recursing deeper if there # is only one subdirectory. for dirname in self.walk_special_directories( os.getcwd(), special_subdir='.morph-system-branch', max_subdirs=1): branch_name = self.get_branch_config(dirname, 'branch.name') return branch_name, dirname raise cliapp.AppException("Can't find the system branch directory") def find_repository(self, branch_dir, repo): for dirname in self.walk_special_directories(branch_dir, special_subdir='.git'): try: original_repo = self.get_repo_config( dirname, 'morph.repository') except cliapp.AppException: # The user may have manually put a git repo in the branch continue if repo == original_repo: return dirname return None def find_system_branch(self, workspace, branch_name): for dirname in self.walk_special_directories( workspace, special_subdir='.morph-system-branch'): branch = self.get_branch_config(dirname, 'branch.name') if branch_name == branch: return dirname return None def set_branch_config(self, branch_dir, option, value): filename = os.path.join(branch_dir, '.morph-system-branch', 'config') self.app.runcmd(['git', 'config', '-f', filename, option, value]) def get_branch_config(self, branch_dir, option): filename = os.path.join(branch_dir, '.morph-system-branch', 'config') value = self.app.runcmd(['git', 'config', '-f', filename, option]) return value.strip() def set_repo_config(self, repo_dir, option, value): self.app.runcmd(['git', 'config', option, value], cwd=repo_dir) def get_repo_config(self, repo_dir, option): value = self.app.runcmd(['git', 'config', option], cwd=repo_dir) return value.strip() def get_head(self, repo_path): '''Return the ref that the working tree is on for a repo''' ref = self.app.runcmd(['git', 'rev-parse', '--abbrev-ref', 'HEAD'], cwd=repo_path).strip() if ref == 'HEAD': ref = 'detached HEAD' return ref def get_uncommitted_changes(self, repo_dir, env={}): status = self.app.runcmd(['git', 'status', '--porcelain'], cwd=repo_dir, env=env) changes = [] for change in status.strip().splitlines(): xy, paths = change.strip().split(' ', 1) if xy != '??': changes.append(paths.split()[0]) return changes def get_unmerged_changes(self, repo_dir, env={}): '''Identifies files which have unresolved merge conflicts''' # The second column of the git command output is set either if the # file has changes in the working tree or if it has conflicts. status = self.app.runcmd(['git', 'status', '--porcelain'], cwd=repo_dir, env=env) changes = [] for change in status.strip().splitlines(): xy, paths = change[0:2], change[2:].strip() if xy[1] != ' ' and xy != '??': changes.append(paths.split()[0]) return changes def resolve_ref(self, repodir, ref): try: return self.app.runcmd(['git', 'rev-parse', '--verify', ref], cwd=repodir)[0:40] except: return None def resolve_reponame(self, reponame): '''Return the full pull URL of a reponame.''' resolver = morphlib.repoaliasresolver.RepoAliasResolver( self.app.settings['repo-alias']) return resolver.pull_url(reponame) def get_cached_repo(self, repo_name): '''Return CachedRepo object from the local repository cache Repo is cached and updated if necessary. The cache itself has a mechanism in place to avoid multiple updates per Morph invocation. ''' self.app.status(msg='Updating git repository %s in cache' % repo_name) if not self.app.settings['no-git-update']: repo = self.lrc.cache_repo(repo_name) repo.update() else: repo = self.lrc.get_repo(repo_name) return repo def clone_to_directory(self, dirname, reponame, ref): '''Clone a repository below a directory. As a side effect, clone it into the local repo cache. ''' # Setup. resolver = morphlib.repoaliasresolver.RepoAliasResolver( self.app.settings['repo-alias']) repo = self.get_cached_repo(reponame) # Make sure the parent directories needed for the repo dir exist. parent_dir = os.path.dirname(dirname) if not os.path.exists(parent_dir): os.makedirs(parent_dir) # Clone it from cache to target directory. target_path = os.path.abspath(dirname) repo.clone_checkout(ref, target_path) # Remember the repo name we cloned from in order to be able # to identify the repo again later using the same name, even # if the user happens to rename the directory. self.set_repo_config(dirname, 'morph.repository', reponame) # Create a UUID for the clone. We will use this for naming # temporary refs, e.g. for building. self.set_repo_config(dirname, 'morph.uuid', uuid.uuid4().hex) # URL configuration morphlib.git.set_remote(self.app.runcmd, dirname, 'origin', repo.url) self.set_repo_config( dirname, 'url.%s.pushInsteadOf' % resolver.push_url(reponame), resolver.pull_url(reponame)) morphlib.git.update_submodules(self.app, target_path) self.app.runcmd(['git', 'remote', 'update'], cwd=dirname) def load_morphology(self, repo_dir, name, ref=None): '''Loads a morphology from a repo in a system branch If 'ref' is specified, the version is taken from there instead of the working tree. Note that you shouldn't use this to fetch files on branches other than the current system branch, because the remote in the system branch repo may be completely out of date. Use the local repository cache instead for this. ''' if ref is None: filename = os.path.join(repo_dir, '%s.morph' % name) with open(filename) as f: text = f.read() else: filename = '%s.morph at ref %s in %s' % (name, ref, repo_dir) if not morphlib.git.is_valid_sha1(ref): ref = morphlib.git.rev_parse(self.app.runcmd, repo_dir, ref) try: text = self.app.runcmd(['git', 'cat-file', 'blob', '%s:%s.morph' % (ref, name)], cwd=repo_dir) except cliapp.AppException as e: msg = '%s.morph was not found in %s' % (name, repo_dir) if ref is not None: msg += ' at ref %s' % ref raise cliapp.AppException(msg) try: morphology = morphlib.morph2.Morphology(text) except ValueError as e: raise morphlib.Error("Error parsing %s: %s" % (filename, str(e))) self._validate_morphology(morphology, '%s.morph' % name) return morphology def _validate_morphology(self, morphology, basename): # FIXME: This really should be in MorphologyFactory. Later. def require(field): if field not in morphology: raise morphlib.Error( 'Required field "%s" is missing from morphology %s' % (field, basename)) required = { 'system': [ 'name', 'system-kind', 'arch', 'strata', ], 'stratum': [ 'name', 'chunks', ], 'chunk': [ 'name', ] } also_known = { 'system': [ 'kind', 'description', 'disk-size', '_disk-size', ], 'stratum': [ 'kind', 'description', 'build-depends', ], 'chunk': [ 'kind', 'description', 'build-system', 'configure-commands', 'build-commands', 'test-commands', 'install-commands', 'max-jobs', 'chunks', ] } require('kind') kind = morphology['kind'] if kind not in required: raise morphlib.Error( 'Unknown morphology kind "%s" in %s' % (kind, basename)) for field in required[kind]: require(field) known = required[kind] + also_known[kind] for field in morphology.keys(): if field not in known: msg = 'Unknown field "%s" in %s' % (field, basename) logging.warning(msg) self.app.status(msg=msg) def reset_work_tree_safe(self, repo_dir): # This function avoids throwing any exceptions, so it is safe to call # inside an 'except' block without altering the backtrace. command = 'git', 'reset', '--hard' status, output, error = self.app.runcmd_unchecked(command, cwd=repo_dir) if status != 0: logging.warning ("Warning: error while trying to clean up %s: %s" % (repo_dir, error)) @staticmethod def save_morphology(repo_dir, name, morphology): if not name.endswith('.morph'): name = '%s.morph' % name if os.path.isabs(name): filename = name else: filename = os.path.join(repo_dir, name) filename = os.path.join(repo_dir, '%s' % name) with morphlib.savefile.SaveFile(filename, 'w') as f: morphology.write_to_file(f) if name != morphology['name']: logging.warning('%s: morphology "name" should match filename' % filename) @staticmethod def get_edit_info(morphology_name, morphology, name, collection='strata'): try: return morphology.lookup_child_by_name(name) except KeyError: if collection is 'strata': raise cliapp.AppException( 'Stratum "%s" not found in system "%s"' % (name, morphology_name)) else: raise cliapp.AppException( 'Chunk "%s" not found in stratum "%s"' % (name, morphology_name)) @staticmethod def convert_uri_to_path(uri): parts = urlparse.urlparse(uri) # If the URI path is relative, assume it is an aliased repo (e.g. # baserock:morphs). Otherwise assume it is a full URI where we need # to strip off the scheme and .git suffix. if not os.path.isabs(parts.path): return uri else: path = parts.netloc if parts.path.endswith('.git'): path = os.path.join(path, parts.path[1:-len('.git')]) else: path = os.path.join(path, parts.path[1:]) return path @staticmethod def remove_branch_dir_safe(workspace, branch): # This function avoids throwing any exceptions, so it is safe to call # inside an 'except' block without altering the backtrace. def handle_error(function, path, excinfo): logging.warning ("Warning: error while trying to clean up %s: %s" % (path, excinfo)) branch_dir = os.path.join(workspace, branch) shutil.rmtree(branch_dir, onerror=handle_error) # Remove parent directories that are empty too, avoiding exceptions parent = os.path.dirname(branch_dir) while parent != os.path.abspath(workspace): if len(os.listdir(parent)) > 0 or os.path.islink(parent): break os.rmdir(parent) parent = os.path.dirname(parent) @staticmethod def iterate_branch_repos(branch_path, root_repo_path): '''Produces a sorted list of component repos in a branch checkout''' dirs = [d for d in BranchAndMergePlugin.walk_special_directories( branch_path, special_subdir='.git') if not os.path.samefile(d, root_repo_path)] dirs.sort() for d in [root_repo_path] + dirs: yield d @staticmethod def walk_special_directories(root_dir, special_subdir=None, max_subdirs=0): assert(special_subdir is not None) assert(max_subdirs >= 0) visited = set() for dirname, subdirs, files in os.walk(root_dir, followlinks=True): # Avoid infinite recursion due to symlinks. if dirname in visited: subdirs[:] = [] continue visited.add(dirname) # Check if the current directory has the special subdirectory. if special_subdir in subdirs: yield dirname # Do not recurse into hidden directories. subdirs[:] = [x for x in subdirs if not x.startswith('.')] # Do not recurse if there is more than the maximum number of # subdirectories allowed. if max_subdirs > 0 and len(subdirs) > max_subdirs: break def init(self, args): '''Initialize a workspace directory.''' if not args: args = ['.'] elif len(args) > 1: raise cliapp.AppException('init must get at most one argument') dirname = args[0] # verify the workspace is empty (and thus, can be used) or # create it if it doesn't exist yet if os.path.exists(dirname): if os.listdir(dirname) != []: raise cliapp.AppException('can only initialize empty ' 'directory as a workspace: %s' % dirname) else: try: os.makedirs(dirname) except: raise cliapp.AppException('failed to create workspace: %s' % dirname) os.mkdir(os.path.join(dirname, '.morph')) self.app.status(msg='Initialized morph workspace', chatty=True) def branch(self, args): '''Create a new system branch.''' if len(args) not in [2, 3]: raise cliapp.AppException('morph branch needs name of branch ' 'as parameter') repo = args[0] new_branch = args[1] commit = 'master' if len(args) == 2 else args[2] # Create the system branch directory. workspace = self.deduce_workspace() branch_dir = os.path.join(workspace, new_branch) os.makedirs(branch_dir) self.lrc, self.rrc = morphlib.util.new_repo_caches(self.app) try: # Create a .morph-system-branch directory to clearly identify # this directory as a morph system branch. os.mkdir(os.path.join(branch_dir, '.morph-system-branch')) # Remember the system branch name and the repository we branched # off from initially. self.set_branch_config(branch_dir, 'branch.name', new_branch) self.set_branch_config(branch_dir, 'branch.root', repo) # Generate a UUID for the branch. We will use this for naming # temporary refs, e.g. building. self.set_branch_config(branch_dir, 'branch.uuid', uuid.uuid4().hex) # Clone into system branch directory. repo_dir = os.path.join(branch_dir, self.convert_uri_to_path(repo)) self.clone_to_directory(repo_dir, repo, commit) # Check if branch already exists locally or in a remote if self.resolve_ref(repo_dir, new_branch) is not None: raise cliapp.AppException('branch %s already exists in ' 'repository %s' % (new_branch, repo)) # Create a new branch in the local morphs repository. self.app.runcmd(['git', 'checkout', '-b', new_branch, commit], cwd=repo_dir) except: self.remove_branch_dir_safe(workspace, new_branch) raise def checkout(self, args): '''Check out an existing system branch.''' if len(args) != 2: raise cliapp.AppException('morph checkout needs a repo and the ' 'name of a branch as parameters') repo = args[0] system_branch = args[1] # Create the system branch directory. workspace = self.deduce_workspace() branch_dir = os.path.join(workspace, system_branch) os.makedirs(branch_dir) self.lrc, self.rrc = morphlib.util.new_repo_caches(self.app) try: # Create a .morph-system-branch directory to clearly identify # this directory as a morph system branch. os.mkdir(os.path.join(branch_dir, '.morph-system-branch')) # Remember the system branch name and the repository we # branched off from. self.set_branch_config(branch_dir, 'branch.name', system_branch) self.set_branch_config(branch_dir, 'branch.root', repo) # Generate a UUID for the branch. self.set_branch_config(branch_dir, 'branch.uuid', uuid.uuid4().hex) # Clone into system branch directory. repo_dir = os.path.join(branch_dir, self.convert_uri_to_path(repo)) self.clone_to_directory(repo_dir, repo, system_branch) except: self.remove_branch_dir_safe(workspace, system_branch) raise def checkout_repository(self, branch_dir, repo, ref, parent_ref=None): '''Make a chunk or stratum repository available for a system branch We ensure the 'system_branch' ref within 'repo' is checked out, creating it from 'parent_ref' if required. The function aims for permissiveness, so users can try to fix any weirdness they have caused in the repos with another call to 'morph edit'. ''' parent_ref = parent_ref or ref repo_dir = self.find_repository(branch_dir, repo) if repo_dir is None: repo_url = self.resolve_reponame(repo) repo_dir = os.path.join(branch_dir, self.convert_uri_to_path(repo)) self.clone_to_directory(repo_dir, repo, parent_ref) if self.resolve_ref(repo_dir, ref) is None: self.log_change(repo, 'branch "%s" created from "%s"' % (ref, parent_ref)) command = ['git', 'checkout', '-b', ref] else: # git copes even if the system_branch ref is already checked out command = ['git', 'checkout', ref] status, output, error = self.app.runcmd_unchecked( command, cwd=repo_dir) if status != 0: raise cliapp.AppException('Command failed: %s in repo %s\n%s' % (' '.join(command), repo, error)) return repo_dir def edit_stratum(self, system_branch, branch_dir, branch_root, branch_root_dir, stratum): if stratum['repo'] == branch_root: stratum_repo_dir = branch_root_dir if stratum['ref'] != system_branch: # We need to bring the morphology forwards from its ref to the # current HEAD repo = self.lrc.get_repo(branch_root) stratum_morphology = repo.load_morphology( stratum['ref'], stratum['morph']) self.save_morphology(branch_root_dir, stratum['morph'], stratum_morphology) self.log_change( stratum['repo'], '"%s" copied from "%s" to "%s"' % (stratum['morph'], stratum['ref'], system_branch)) else: # Make sure the stratum repository is available stratum_repo_dir = self.checkout_repository( branch_dir, stratum['repo'], system_branch, parent_ref=stratum['ref']) stratum['ref'] = system_branch return stratum_repo_dir def edit_chunk(self, system_branch, branch_dir, stratum_repo_dir, chunk): if 'unpetrify-ref' in chunk: chunk['ref'] = chunk['unpetrify-ref'] del chunk['unpetrify-ref'] # Make the chunk repository and the ref available locally. chunk_repo_dir = self.checkout_repository( branch_dir, chunk['repo'], system_branch, parent_ref=chunk['ref']) # Check if we need to update anything at all. if chunk['ref'] != system_branch: # Update the reference to the chunk in the stratum morphology. chunk['ref'] = system_branch def edit(self, args): '''Edit a component in a system branch.''' if len(args) not in (2, 3): raise cliapp.AppException( 'morph edit must either get a system and a stratum ' 'or a system, a stratum and a chunk as arguments') workspace = self.deduce_workspace() system_branch, branch_dir = self.deduce_system_branch() # Find out which repository we branched off from. branch_root = self.get_branch_config(branch_dir, 'branch.root') branch_root_dir = self.find_repository(branch_dir, branch_root) system_name = args[0] stratum_name = args[1] chunk_name = args[2] if len(args) > 2 else None self.lrc, self.rrc = morphlib.util.new_repo_caches(self.app) # Load the system morphology and find out which repo and ref # we need to edit the stratum. system_morphology = self.load_morphology(branch_root_dir, system_name) stratum = self.get_edit_info(system_name, system_morphology, stratum_name, collection='strata') stratum_repo_dir = self.edit_stratum( system_branch, branch_dir, branch_root, branch_root_dir, stratum) self.save_morphology(branch_root_dir, system_name, system_morphology) self.log_change(branch_root, '"%s" now includes "%s" from "%s"' % (system_name, stratum_name, system_branch)) if chunk_name: # Load the stratum morphology and find out which repo and ref # we need to edit the chunk. stratum_morphology = self.load_morphology(stratum_repo_dir, stratum_name) chunk = self.get_edit_info(stratum_name, stratum_morphology, chunk_name, collection='chunks') self.edit_chunk(system_branch, branch_dir, stratum_repo_dir, chunk) self.save_morphology(stratum_repo_dir, stratum_name, stratum_morphology) self.log_change(stratum['repo'], '"%s" now includes "%s" from "%s"' % (stratum_name, chunk_name, system_branch)) self.print_changelog('The following changes were made but have not ' 'been committed') def petrify(self, args): '''Convert all chunk refs in a system branch to be fixed SHA1s This isolates the branch from changes made by other developers in the chunk repositories. ''' # Stratum refs are not petrified, because they must all be edited to # set the new chunk refs, which requires branching them all for the # current branch - so they will not be updated outside of the user's # control in any case. Chunks that have already been edited on the # current branch are also not petrified. if len(args) != 0: raise cliapp.AppException('morph petrify takes no arguments') workspace = self.deduce_workspace() branch, branch_path = self.deduce_system_branch() root_repo = self.get_branch_config(branch_path, 'branch.root') root_repo_dir = self.find_repository(branch_path, root_repo) self.lrc, self.rrc = morphlib.util.new_repo_caches(self.app) # We must first get the full set of strata. One same stratum may be # in multiple systems and each system may use a different ref. strata = {} for f in sorted(glob.iglob(os.path.join(root_repo_dir, '*.morph'))): name = os.path.basename(f)[:-len('.morph')] morphology = self.load_morphology(root_repo_dir, name) if morphology['kind'] != 'system': continue for stratum_info in morphology['strata']: key = (stratum_info['repo'], stratum_info['morph']) if key in strata: original_ref = strata[key] if stratum_info['ref'] == branch: strata[key] = branch elif stratum_info['ref'] != original_ref: if original_ref != branch: self.app.output.write( 'WARNING: not merging any differences from ' 'ref %s into %s of stratum %s\n' % (stratum_info['ref'], original_ref, stratum_info['morph'])) stratum_info['ref'] = branch else: strata[key] = stratum_info['ref'] stratum_info['ref'] = branch self.save_morphology(root_repo_dir, name, morphology) for (repo, morph), ref in strata.iteritems(): repo_dir = self.edit_stratum( branch, branch_path, root_repo, root_repo_dir, { 'repo': repo, 'ref': ref, 'morph': morph}) stratum = self.load_morphology(repo_dir, morph) for chunk_info in stratum['chunks']: if (chunk_info['ref'] != branch and 'unpetrify-ref' not in chunk_info): commit_sha1, tree_sha1 = self.app.resolve_ref( self.lrc, self.rrc, chunk_info['repo'], chunk_info['ref'], update=not self.app.settings['no-git-update']) chunk_info['unpetrify-ref'] = chunk_info['ref'] chunk_info['ref'] = commit_sha1 self.save_morphology(repo_dir, morph, stratum) self.print_changelog('The following changes were made but have not ' 'been committed') def unpetrify(self, args): '''Reverse the process of petrification''' # This function makes no attempt to 'unedit' strata that were branched # solely so they could be petrified. if len(args) != 0: raise cliapp.AppException('morph unpetrify takes no arguments') self.lrc, self.rrc = morphlib.util.new_repo_caches(self.app) workspace = self.deduce_workspace() branch, branch_path = self.deduce_system_branch() root_repo = self.get_branch_config(branch_path, 'branch.root') root_repo_dir = self.find_repository(branch_path, root_repo) for f in glob.glob(os.path.join(root_repo_dir, '*.morph')): name = os.path.basename(f)[:-len('.morph')] morphology = self.load_morphology(root_repo_dir, name) if morphology['kind'] != 'system': continue for stratum_info in morphology['strata']: repo_dir = self.edit_stratum( branch, branch_path, root_repo, root_repo_dir, stratum_info) stratum = self.load_morphology(repo_dir, stratum_info['morph']) for chunk_info in stratum['chunks']: if 'unpetrify-ref' in chunk_info: chunk_info['ref'] = chunk_info['unpetrify-ref'] del chunk_info['unpetrify-ref'] self.save_morphology(repo_dir, stratum_info['morph'], stratum) self.save_morphology(root_repo_dir, name, morphology) self.print_changelog('The following changes were made but have not ' 'been committed') # When 'merge' is unset, git doesn't try to resolve conflicts itself in # those files. MERGE_ATTRIBUTE = '*.morph\t-merge\n' def disable_morph_merging(self, repo_dir): attributes_file = os.path.join(repo_dir, ".git", "info", "attributes") with open(attributes_file, 'a') as f: f.write(self.MERGE_ATTRIBUTE) def enable_morph_merging(self, repo_dir): attributes_file = os.path.join(repo_dir, ".git", "info", "attributes") with open(attributes_file, 'r') as f: attributes = f.read() if attributes == self.MERGE_ATTRIBUTE: os.unlink(attributes_file) elif attributes.endswith(self.MERGE_ATTRIBUTE): with morphlib.savefile.SaveFile(attributes_file, 'w') as f: f.write(attributes[:-len(self.MERGE_ATTRIBUTE)]) def get_merge_files(self, repo_dir, from_sha1, to_ref, name): '''Returns merge base, remote and local versions of a morphology We already ran 'git fetch', so the remote branch is available within the target repository. ''' base_sha1 = self.app.runcmd(['git', 'merge-base', from_sha1, to_ref], cwd=repo_dir).strip() base_morph = self.load_morphology(repo_dir, name, ref=base_sha1) from_morph = self.load_morphology(repo_dir, name, ref=from_sha1) to_morph = self.load_morphology(repo_dir, name, ref=to_ref) return base_morph, from_morph, to_morph def check_component(self, parent_kind, parent_path, from_info, to_info): assert (parent_kind in ['system', 'stratum']) kind = 'chunk' if parent_kind == 'stratum' else 'stratum' name = to_info.get('alias', to_info.get('name', to_info.get('morph'))) path = parent_path + '.' + name if kind == 'chunk': # Only chunks can be petrified from_unpetrify_ref = from_info.get('unpetrify-ref', None) to_unpetrify_ref = to_info.get('unpetrify-ref', None) if from_unpetrify_ref is not None and to_unpetrify_ref is None: self.app.output.write( 'WARNING: chunk "%s" is now petrified\n' % path) elif from_unpetrify_ref is None and to_unpetrify_ref is not None: self.app.output.write( 'WARNING: chunk "%s" is no longer petrified\n' % path) elif from_unpetrify_ref != to_unpetrify_ref: raise cliapp.AppException( 'merge conflict: chunk "%s" is petrified to a different ' 'ref in each branch' % path) def diff_morphologies(self, path, from_morph, to_morph): '''Component-level diff between two versions of a morphology''' def component_key(info): # This function needs only to be stable and reproducible key = info['repo'] + '|' + info['morph'] if 'name' in info: key += '|' + info['name'] return key if from_morph['name'] != to_morph['name']: # We should enforce name == filename in load_morphology() raise cliapp.AppException( 'merge conflict: "name" of morphology %s (name should always ' 'match filename)' % path) if from_morph['kind'] != to_morph['kind']: raise cliapp.AppException( 'merge conflict: "kind" of morphology %s changed from %s to %s' % (path, to_morph['kind'], from_morph['kind'])) kind = to_morph['kind'] # copy() makes a shallow copy, so editing the list elements will # change the actual morphologies. if kind == 'system': from_components = copy.copy(from_morph['strata']) to_components = copy.copy(to_morph['strata']) elif kind == 'stratum': from_components = copy.copy(from_morph['chunks']) to_components = copy.copy(to_morph['chunks']) from_components.sort(key=component_key) to_components.sort(key=component_key) # These are not set() purely because a set requires a hashable type intersection = [] # TO n FROM from_diff = [] # FROM \ TO to_diff = [] # TO \ FROM while len(from_components) > 0 and len(to_components) > 0: from_info = from_components.pop(0) to_info = to_components.pop(0) match = cmp(component_key(from_info), component_key(to_info)) if match < 0: from_diff.append(from_info) elif match > 0: to_diff.append(to_info) elif match == 0: intersection.append((from_info, to_info)) if len(from_components) != 0: from_diff.append(from_components.pop(0)) if len(to_components) != 0: to_diff.append(to_components.pop(0)) return intersection, from_diff, to_diff def merge_repo(self, merged_repos, from_branch_dir, from_repo, from_ref, to_branch_dir, to_repo, to_ref): '''Merge changes for a system branch in a specific repository We disable merging for morphologies and do this manually later on. ''' if to_repo in merged_repos: return merged_repos[to_repo] from_repo_dir = self.find_repository(from_branch_dir, from_repo) to_repo_dir = self.checkout_repository(to_branch_dir, to_repo, to_ref) if self.get_uncommitted_changes(from_repo_dir) != []: raise cliapp.AppException('repository %s has uncommitted ' 'changes' % from_repo) if self.get_uncommitted_changes(to_repo_dir) != []: raise cliapp.AppException('repository %s has uncommitted ' 'changes' % to_repo) # Fetch the local FROM branch; its sha1 will be stored in FETCH_HEAD. # ':' in pathnames confuses git, so we have to pass it a URL. from_repo_url = urlparse.urljoin('file://', from_repo_dir) self.app.runcmd(['git', 'fetch', from_repo_url, from_ref], cwd=to_repo_dir) # Merge everything but the morphologies; error output is ignored (it's # not very good) and instead we report conflicts manually later on. self.disable_morph_merging(to_repo_dir) with open(os.path.join(to_repo_dir, '.git', 'FETCH_HEAD')) as f: from_sha1 = f.read(40) status, output, error = self.app.runcmd_unchecked( ['git', 'merge', '--no-commit', '--no-ff', from_sha1], cwd=to_repo_dir) self.enable_morph_merging(to_repo_dir) merged_repos[to_repo] = (to_repo_dir, from_sha1) return (to_repo_dir, from_sha1) def merge(self, args): '''Pull and merge changes from a system branch into the current one. The remote branch is pulled from the current workspace into the target repositories (so any local commits are included). ''' if len(args) != 1: raise cliapp.AppException('morph merge requires a system branch ' 'name as its argument') self.lrc, self.rrc = morphlib.util.new_repo_caches(self.app) workspace = self.deduce_workspace() from_branch = args[0] from_branch_dir = self.find_system_branch(workspace, from_branch) to_branch, to_branch_dir = self.deduce_system_branch() if from_branch_dir is None: raise cliapp.AppException('branch %s must be checked out before ' 'it can be merged' % from_branch) root_repo = self.get_branch_config(from_branch_dir, 'branch.root') other_root_repo = self.get_branch_config(to_branch_dir, 'branch.root') if root_repo != other_root_repo: raise cliapp.AppException('branches do not share a root ' 'repository : %s vs %s' % (root_repo, other_root_repo)) def merge_chunk(parent_path, old_ci, ci): self.merge_repo(merged_repos, from_branch_dir, old_ci['repo'], from_branch, to_branch_dir, ci['repo'], ci['ref']) def merge_stratum(parent_path, old_si, si): path = parent_path + '.' + si['morph'] to_repo_dir, from_sha1 = self.merge_repo(merged_repos, from_branch_dir, old_si['repo'], from_branch, to_branch_dir, si['repo'], si['ref']) base_morph, from_morph, to_morph = self.get_merge_files( to_repo_dir, from_sha1, si['ref'], si['morph']) intersection, from_diff, to_diff = self.diff_morphologies( path, from_morph, to_morph) for from_ci, to_ci in intersection: self.check_component('stratum', path, from_ci, to_ci) changed = False edited_chunks = [ci for ci in from_morph['chunks'] if ci['ref'] == from_branch] for ci in edited_chunks: for old_ci in to_morph['chunks']: if old_ci['repo'] == ci['repo']: break else: raise cliapp.AppException( 'chunk %s was added within this branch and ' 'subsequently edited. This is not yet supported: ' 'refusing to merge.' % ci['name']) changed = True ci['ref'] = old_ci['ref'] merge_chunk(path, old_ci, ci) if changed: self.save_morphology(to_repo_dir, si['morph'], to_morph) self.app.runcmd(['git', 'add', si['morph'] + '.morph'], cwd=to_repo_dir) def merge_system(name): base_morph, from_morph, to_morph = self.get_merge_files( to_root_dir, from_sha1, to_branch, name) if to_morph['kind'] != 'system': return intersection, from_diff, to_diff = self.diff_morphologies( name, from_morph, to_morph) for from_si, to_si in intersection: self.check_component('system', name, from_si, to_si) changed = False edited_strata = [si for si in from_morph['strata'] if si['ref'] == from_branch] for si in edited_strata: for old_si in to_morph['strata']: # We make no attempt at rename / move detection if (old_si['morph'] == si['morph'] and old_si['repo'] == si['repo']): break else: raise cliapp.AppException( 'stratum %s was added within this branch and ' 'subsequently edited. This is not yet supported: ' 'refusing to merge.' % si['morph']) changed = True si['ref'] = old_si['ref'] merge_stratum(name, old_si, si) if changed: self.save_morphology(to_root_dir, name, to_morph) self.app.runcmd(['git', 'add', f], cwd=to_root_dir) merged_repos = {} try: to_root_dir, from_sha1 = self.merge_repo(merged_repos, from_branch_dir, root_repo, from_branch, to_branch_dir, root_repo, to_branch) for f in glob.glob(os.path.join(to_root_dir, '*.morph')): name = os.path.basename(f)[:-len('.morph')] merge_system(name) success = True for repo_name, repo_info in merged_repos.iteritems(): repo_dir = repo_info[0] conflicts = self.get_unmerged_changes(repo_dir) if len(conflicts) > 0: self.app.output.write("Merge conflicts in %s:\n\t%s\n" % (repo_name, '\n\t'.join(conflicts))) success = False elif morphlib.git.index_has_changes(self.app.runcmd, repo_dir): # Repo may not be dirty if the changes only touched refs, # because they may now match the previous state. msg = "Merge system branch '%s'" % from_branch self.app.runcmd(['git', 'commit', '--all', '-m%s' % msg], cwd=repo_dir) if not success: raise cliapp.AppException( "merge errors were encountered. Please manually merge the " "target ref into %s in the remote system branch in each " "case, and then repeat the 'morph merge' operation." % from_branch) self.app.status(msg="Merge successful") except: for repo_dir, sha1 in merged_repos.itervalues(): self.reset_work_tree_safe(repo_dir) raise def build(self, args): '''Build a system from the current system branch''' if len(args) != 1: raise cliapp.AppException('morph build expects exactly one ' 'parameter: the system to build') system_name = args[0] # Deduce workspace and system branch and branch root repository. workspace = self.deduce_workspace() branch, branch_dir = self.deduce_system_branch() branch_root = self.get_branch_config(branch_dir, 'branch.root') branch_uuid = self.get_branch_config(branch_dir, 'branch.uuid') # Generate a UUID for the build. build_uuid = uuid.uuid4().hex build_command = morphlib.buildcommand.BuildCommand(self.app) build_command = self.app.hookmgr.call('new-build-command', build_command) push = self.app.settings['push-build-branches'] self.app.status(msg='Starting build %(uuid)s', uuid=build_uuid) self.app.status(msg='Collecting morphologies involved in ' 'building %(system)s from %(branch)s', system=system_name, branch=branch) # Get repositories of morphologies involved in building this system # from the current system branch. build_repos = self.get_system_build_repos( branch, branch_dir, branch_root, system_name) # Generate temporary build ref names for all these repositories. self.generate_build_ref_names(build_repos, branch_uuid) # Create the build refs for all these repositories and commit # all uncommitted changes to them, updating all references # to system branch refs to point to the build refs instead. self.update_build_refs(build_repos, branch, build_uuid, push) if push: self.push_build_refs(build_repos) build_branch_root = branch_root else: dirname = build_repos[branch_root]['dirname'] build_branch_root = urlparse.urljoin('file://', dirname) # Run the build. build_command.build([build_branch_root, build_repos[branch_root]['build-ref'], system_name]) if push: self.delete_remote_build_refs(build_repos) self.app.status(msg='Finished build %(uuid)s', uuid=build_uuid) def get_system_build_repos(self, system_branch, branch_dir, branch_root, system_name): '''Map upstream repository URLs to their checkouts in the system branch Also provides the list of morphologies stored in each repository, grouped by kind. ''' build_repos = {} def prepare_repo_info(repo, dirname): build_repos[repo] = { 'dirname': dirname, 'systems': [], 'strata': [], 'chunks': [] } def add_morphology_info(info, category): repo = info['repo'] if repo in build_repos: repo_dir = build_repos[repo]['dirname'] else: repo_dir = self.find_repository(branch_dir, repo) if repo_dir: if not repo in build_repos: prepare_repo_info(repo, repo_dir) build_repos[repo][category].append(info['morph']) return repo_dir # Add repository and morphology of the system. branch_root_dir = self.find_repository(branch_dir, branch_root) prepare_repo_info(branch_root, branch_root_dir) build_repos[branch_root]['systems'].append(system_name) # Traverse and add repositories and morphologies involved in # building this system from the system branch. system_morphology = self.load_morphology(branch_root_dir, system_name) for info in system_morphology['strata']: if info['ref'] == system_branch: repo_dir = add_morphology_info(info, 'strata') if repo_dir: stratum_morphology = self.load_morphology( repo_dir, info['morph']) for info in stratum_morphology['chunks']: if info['ref'] == system_branch: add_morphology_info(info, 'chunks') return build_repos def inject_build_refs(self, morphology, build_repos, will_push): # Starting from a system or stratum morphology, update all ref # pointers of strata or chunks involved in a system build (represented # by build_repos) to point to temporary build refs of the repos # involved in the system build. def inject_build_ref(info): if info['repo'] in build_repos and ( info['morph'] in build_repos[info['repo']]['strata'] or info['morph'] in build_repos[info['repo']]['chunks']): info['ref'] = build_repos[info['repo']]['build-ref'] if not will_push: dirname = build_repos[info['repo']]['dirname'] info['repo'] = urlparse.urljoin('file://', dirname) if morphology['kind'] == 'system': for info in morphology['strata']: inject_build_ref(info) elif morphology['kind'] == 'stratum': for info in morphology['chunks']: inject_build_ref(info) def generate_build_ref_names(self, build_repos, branch_uuid): for repo, info in build_repos.iteritems(): repo_dir = info['dirname'] repo_uuid = self.get_repo_config(repo_dir, 'morph.uuid') build_ref = os.path.join(self.app.settings['build-ref-prefix'], branch_uuid, repo_uuid) info['build-ref'] = build_ref def update_build_refs(self, build_repos, system_branch, build_uuid, will_push): '''Update build branches for each repository with any local changes ''' # Define the committer. committer_name = 'Morph (on behalf of %s)' % \ (morphlib.git.get_user_name(self.app.runcmd)) committer_email = '%s@%s' % \ (os.environ.get('LOGNAME'), socket.gethostname()) for repo, info in build_repos.iteritems(): repo_dir = info['dirname'] build_ref = info['build-ref'] self.app.status(msg='%(repo)s: Creating build branch', repo=repo) # Obtain parent SHA1 for the temporary ref tree to be committed. # This will either be the current commit of the temporary ref or # HEAD in case the temporary ref does not exist yet. parent_sha1 = self.resolve_ref(repo_dir, build_ref) if not parent_sha1: parent_sha1 = self.resolve_ref(repo_dir, system_branch) # Prepare an environment with our internal index file. # This index file allows us to commit changes to a tree without # git noticing any change in working tree or its own index. env = dict(os.environ) env['GIT_INDEX_FILE'] = os.path.join( repo_dir, '.git', 'morph-index') env['GIT_COMMITTER_NAME'] = committer_name env['GIT_COMMITTER_EMAIL'] = committer_email # Read tree from parent or current HEAD into the morph index. self.app.runcmd(['git', 'read-tree', parent_sha1], cwd=repo_dir, env=env) self.app.status(msg='%(repo)s: Adding uncommitted changes to ' 'build branch', repo=repo) # Add all local, uncommitted changes to our internal index. changed_files = self.get_uncommitted_changes(repo_dir, env) self.app.runcmd(['git', 'add'] + changed_files, cwd=repo_dir, env=env) self.app.status(msg='%(repo)s: Update morphologies to use ' 'build branch instead of "%(branch)s"', repo=repo, branch=system_branch) # Update all references to the system branches of strata # and chunks to point to the temporary refs, which is needed # for building. filenames = info['systems'] + info['strata'] for filename in filenames: # Inject temporary refs in the right places in each morphology. morphology = self.load_morphology(repo_dir, filename) self.inject_build_refs(morphology, build_repos, will_push) handle, tmpfile = tempfile.mkstemp(suffix='.morph') self.save_morphology(repo_dir, tmpfile, morphology) morphology_sha1 = self.app.runcmd( ['git', 'hash-object', '-t', 'blob', '-w', tmpfile], cwd=repo_dir, env=env) self.app.runcmd( ['git', 'update-index', '--cacheinfo', '100644', morphology_sha1, '%s.morph' % filename], cwd=repo_dir, env=env) # Remove the temporary morphology file. os.remove(tmpfile) # Create a commit message including the build UUID. This allows us # to collect all commits of a build across repositories and thereby # see the changes made to the entire system between any two builds. message = 'Morph build %s\n\nSystem branch: %s\n' % \ (build_uuid, system_branch) # Write and commit the tree and update the temporary build ref. tree = self.app.runcmd( ['git', 'write-tree'], cwd=repo_dir, env=env).strip() commit = self.app.runcmd( ['git', 'commit-tree', tree, '-p', parent_sha1], feed_stdin=message, cwd=repo_dir, env=env).strip() self.app.runcmd( ['git', 'update-ref', '-m', message, 'refs/heads/%s' % build_ref, commit], cwd=repo_dir, env=env) def push_build_refs(self, build_repos): for repo, info in build_repos.iteritems(): self.app.status(msg='%(repo)s: Pushing build branch', repo=repo) self.app.runcmd(['git', 'push', 'origin', '%s:%s' % (info['build-ref'], info['build-ref'])], cwd=info['dirname']) def delete_remote_build_refs(self, build_repos): for repo, info in build_repos.iteritems(): self.app.status(msg='%(repo)s: Deleting remote build branch', repo=repo) self.app.runcmd(['git', 'push', 'origin', ':%s' % info['build-ref']], cwd=info['dirname']) def status(self, args): '''Show information about the current system branch or workspace''' if len(args) != 0: raise cliapp.AppException('morph status takes no arguments') workspace = self.deduce_workspace() try: branch, branch_path = self.deduce_system_branch() except cliapp.AppException: branch = None if branch is None: self.app.output.write("System branches in current workspace:\n") branch_dirs = sorted(self.walk_special_directories( workspace, special_subdir='.morph-system-branch')) for dirname in branch_dirs: branch = self.get_branch_config(dirname, 'branch.name') self.app.output.write(" %s\n" % branch) return root_repo = self.get_branch_config(branch_path, 'branch.root') root_repo_path = self.find_repository(branch_path, root_repo) self.app.output.write("On branch %s, root %s\n" % (branch, root_repo)) has_uncommitted_changes = False for d in self.iterate_branch_repos(branch_path, root_repo_path): try: repo = self.get_repo_config(d, 'morph.repository') except cliapp.AppException: self.app.output.write( ' %s: not part of system branch\n' % d) continue head = self.get_head(d) if head != branch: self.app.output.write( ' %s: unexpected ref checked out "%s"\n' % (repo, head)) if len(self.get_uncommitted_changes(d)) > 0: has_uncommitted_changes = True self.app.output.write(' %s: uncommitted changes\n' % repo) if not has_uncommitted_changes: self.app.output.write("\nNo repos have outstanding changes.\n") def foreach(self, args): '''Run a command in each repository checked out in a system branch Use -- before specifying the command to separate its arguments from Morph's own arguments. ''' # For simplicity, this simply iterates repositories in the directory # rather than walking through the morphologies as 'morph merge' does. if len(args) == 0: raise cliapp.AppException('morph foreach expects a command to run') workspace = self.deduce_workspace() branch, branch_path = self.deduce_system_branch() root_repo = self.get_branch_config(branch_path, 'branch.root') root_repo_path = self.find_repository(branch_path, root_repo) for d in self.iterate_branch_repos(branch_path, root_repo_path): try: repo = self.get_repo_config(d, 'morph.repository') except cliapp.AppException: continue if d != root_repo_path: self.app.output.write('\n') self.app.output.write('%s\n' % repo) status, output, error = self.app.runcmd_unchecked(args, cwd=d) self.app.output.write(output) if status != 0: self.app.output.write(error) raise cliapp.AppException( 'Command failed at repo %s: %s' % (repo, ' '.join(args))) def workspace(self, args): '''Find the toplevel directory of the current workspace''' self.app.output.write('%s\n' % self.deduce_workspace()) def show_system_branch(self, args): '''Print name of current system branch''' branch, dirname = self.deduce_system_branch() self.app.output.write('%s\n' % branch) def show_branch_root(self, args): '''Print name of the repository holding the system morphologies''' workspace = self.deduce_workspace() system_branch, branch_dir = self.deduce_system_branch() branch_root = self.get_branch_config(branch_dir, 'branch.root') self.app.output.write('%s\n' % branch_root)