# -*- coding: utf-8 -*- # # Copyright © 2014, 2015 Codethink Limited # # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by # the Free Software Foundation; version 2 of the License. # # This program is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License along # with this program; if not, write to the Free Software Foundation, Inc., # 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. import ansicolor import cliapp import logging import os import pipes import sys import requests import json import morphlib import baserockimport class BaserockImportApplication(cliapp.Application): def add_settings(self): self.settings.string(['lorries-dir'], "location for Lorry files", metavar="PATH", default=os.path.abspath('./lorries')) self.settings.string(['definitions-dir'], "location for morphology files", metavar="PATH", default=os.path.abspath('./definitions')) self.settings.string(['checkouts-dir'], "location for Git checkouts", metavar="PATH", default=os.path.abspath('./checkouts')) self.settings.string(['lorry-working-dir'], "Lorry working directory", metavar="PATH", default=os.path.abspath('./lorry-working-dir')) self.settings.boolean(['force-stratum-generation', 'force-stratum'], "always create a stratum, overwriting any " "existing stratum morphology, and ignoring any " "components where errors occurred during import", default=False) self.settings.boolean(['update-existing'], "update all the checked-out Git trees and " "generated definitions", default=False) self.settings.boolean(['use-local-sources'], "use file:/// URLs in the stratum 'repo' " "fields, instead of upstream: URLs", default=False) self.settings.boolean(['use-master-if-no-tag'], "if the correct tag for a version can't be " "found, use 'master' instead of raising an " "error", default=False) def _stream_has_colours(self, stream): # http://blog.mathieu-leplatre.info/colored-output-in-console-with-python.html if not hasattr(stream, "isatty"): return False if not stream.isatty(): return False # auto color only on TTYs try: import curses curses.setupterm() return curses.tigetnum("colors") > 2 except: # guess false in case of error return False def setup(self): self.add_subcommand('npm', self.import_npm, arg_synopsis='PACKAGE [VERSION]') self.add_subcommand('omnibus', self.import_omnibus, arg_synopsis='REPO PROJECT_NAME SOFTWARE_NAME') self.add_subcommand('rubygems', self.import_rubygems, arg_synopsis='GEM_NAME [GEM_VERSION]') self.add_subcommand('python', self.import_python, arg_synopsis='PACKAGE_NAME [VERSION]') self.add_subcommand('cpan', self.import_cpan, arg_synopsis='MODULE_NAME [VERSION]') self.stdout_has_colours = self._stream_has_colours(sys.stdout) def setup_logging_formatter_for_file(self): root_logger = logging.getLogger() root_logger.name = 'main' # You need recent cliapp for this to work, with commit "Split logging # setup into further overrideable methods". return logging.Formatter("%(name)s: %(levelname)s: %(message)s") def process_args(self, args): if len(args) == 0: # Cliapp default is to just say "ERROR: must give subcommand" if # no args are passed, I prefer this. args = ['help'] super(BaserockImportApplication, self).process_args(args) def status(self, msg, *args, **kwargs): text = msg % args if kwargs.get('error') == True: logging.error(text) if self.stdout_has_colours: sys.stdout.write(ansicolor.red(text)) else: sys.stdout.write(text) else: logging.info(text) sys.stdout.write(text) sys.stdout.write('\n') def import_omnibus(self, args): '''Import a software component from an Omnibus project. Omnibus is a tool for generating application bundles for various platforms. See for more information. ''' if len(args) != 3: raise cliapp.AppException( 'Please give the location of the Omnibus definitions repo, ' 'and the name of the project and the top-level software ' 'component.') def running_inside_bundler(): return 'BUNDLE_GEMFILE' in os.environ def command_to_run_python_in_directory(directory, args): # Bundler requires that we run it from the Omnibus project # directory. That messes up any relative paths the user may have # passed on the commandline, so we do a bit of a hack to change # back to the original directory inside the `bundle exec` process. return "cd %s; exec python %s" % ( pipes.quote(directory), ' '.join(map(pipes.quote, args))) def reexecute_self_with_bundler(path): script = sys.argv[0] logging.info('Reexecuting %s within Bundler, so that extensions ' 'use the correct dependencies for Omnibus and the ' 'Omnibus project definitions.', script) command = command_to_run_python_in_directory(os.getcwd(), sys.argv) logging.debug('Running: `bundle exec %s` in dir %s', command, path) os.chdir(path) os.execvp('bundle', [script, 'exec', command]) # Omnibus definitions are spread across multiple repos, and there is # no stability guarantee for the definition format. The official advice # is to use Bundler to execute Omnibus, so let's do that. if not running_inside_bundler(): reexecute_self_with_bundler(args[0]) definitions_dir = args[0] project_name = args[1] loop = baserockimport.mainloop.ImportLoop( app=self, goal_kind='omnibus', goal_name=args[2], goal_version='master') loop.enable_importer('omnibus', extra_args=[definitions_dir, project_name]) loop.enable_importer('rubygems') loop.run() def import_npm(self, args): '''Import one or more Node.js npm packages.''' if len(args) not in [1, 2]: raise cliapp.AppException( 'Please pass the name and version of an npm package on the ' 'commandline.') goal_name = args[0] goal_version = args[1] if len(args) == 2 else 'master' loop = baserockimport.mainloop.ImportLoop( app=self, goal_kind='npm', goal_name=goal_name, goal_version=goal_version) loop.enable_importer('npm', strata=['strata/nodejs.morph']) loop.run() def import_rubygems(self, args): '''Import one or more RubyGems.''' if len(args) not in [1, 2]: raise cliapp.AppException( 'Please pass the name and version of a RubyGem on the ' 'commandline.') goal_name = args[0] goal_version = args[1] if len(args) == 2 else 'master' loop = baserockimport.mainloop.ImportLoop( app=self, goal_kind='rubygems', goal_name=args[0], goal_version=goal_version) loop.enable_importer('rubygems', strata=['strata/ruby.morph']) loop.run() def import_python(self, args): '''Import one or more python packages.''' def comp(x, y): return x.replace('-', '_').lower() == y.replace('-', '_').lower() if len(args) < 1 or len(args) > 2: raise cliapp.AppException( 'Please pass the name of the python package on the commandline.') package_name = args[0] package_version = args[1] if len(args) == 2 else 'master' loop = baserockimport.mainloop.ImportLoop(app=self, goal_kind='python', goal_name=package_name, goal_version=package_version, generate_chunk_morphs=False, ignore_version_field=True) loop.enable_importer('python', strata=['strata/core.morph'], package_comp_callback=comp) loop.run() def import_cpan(self, args): '''Import one or more perl modules We have to do a little work before we're ready to run the first extension, the user will provide a module name as input but we import entire distributions not individual modules. So we first query metacpan to find the distribution that provides the given module, we also write a miniumum amount of metadata to a 'ROOT.meta' file which is passed onto the root package via IMPORT_METAPATH, this metadata is used to map distributions to the modules they provide. ''' UNEXPECTED_RESPONSE_ERRMSG = ("Couldn't obtain distribution for " "`%s' version `%s': server returned unexpected query response") NO_DIST_WITH_MODULE_FOR_VERSION_ERRMSG = ( "Couldn't find a distribution containing module `%s' " "with version `%s'") NO_DIST_WITH_MODULE_ERRMSG = ( "Couldn't find distribution for module `%s'") def get_module_metadata(module_name, module_version): return (get_metadata_for_module_with_version(module_name, module_version) if module_version is not None else get_metadata_for_module(module_name)) def get_metadata_for_module(module_name): ''' Gets metadata for the latest release of a module from metacpan ''' try: r = requests.get('http://api.metacpan.org/module/%s' % module_name) r.raise_for_status() except Exception as e: errmsg = ("%s: %s" % (NO_DIST_WITH_MODULE_ERRMSG % module_name, e)) raise cliapp.AppException(errmsg) json = r.json() if 'distribution' not in json: raise cliapp.AppException(NO_DIST_WITH_MODULE_ERRMSG % module_name) return {'distribution': json['distribution'], 'version': None} def get_metadata_for_module_with_version(module_name, module_version): ''' Gets metadata for a specific version of a module from metacpan ''' q = {'query': { 'filtered': {'query': {'match_all': {}}, 'filter': {'and': [ {'term': {'file.module.name': module_name}}, {'term': {'file.module.version': module_version}} ]} }}, 'fields': ['distribution', 'version'] } distribution = None version = None try: query_url = 'http://api.metacpan.org/v0/file/_search' r = requests.post(query_url, json=q) r.raise_for_status() except Exception as e: errmsg = ("Couldn't query metacpan with %s: %s" % (query_url, e)) raise cliapp.AppException(errmsg) try: hits = r.json()['hits']['total'] if hits == 0: raise cliapp.AppException( NO_DIST_WITH_MODULE_FOR_VERSION_ERRMSG % (module_name, module_version)) fields = r.json()['hits']['hits'][0]['fields'] distribution = fields['distribution'] version = fields['version'] except KeyError: raise cliapp.AppException(UNEXPECTED_RESPONSE_ERRMSG % (module_name, module_version)) return {'distribution': distribution, 'version': version} def write_root_metadata(metadata, module, module_version): ''' Constructs the initial metadata file to be passed to the first import we run via IMPORT_METAPATH. ROOT.meta will map the module we require to the distribution that provides it. ''' distribution = metadata['distribution'] depends_filename = 'strata/%s/ROOT.meta' % distribution defsdir = self.settings['definitions-dir'] depends_path = os.path.join(defsdir, depends_filename) if not os.path.exists(defsdir): os.makedirs(defsdir) morphlib.gitdir.init(defsdir) p, _ = os.path.split(depends_path) if not os.path.exists(p): try: os.makedirs(p) except OSError as e: if e.errno != errno.EEXIST: raise e metadata = {'cpan': {'dist-meta': {distribution: {'modules': {module: {'minimum_version': module_version} }, 'pathname': None}}}} with open(depends_path, 'w') as f: json.dump(metadata, f) return depends_path if len(args) not in (1, 2): raise cliapp.AppException('usage: %s cpan MODULE_NAME [VERSION]' % sys.argv[0]) module_name = args[0] module_version = args[1] if len(args) == 2 else None metadata = get_module_metadata(module_name, module_version) metadata_path = write_root_metadata(metadata, module_name, module_version) os.environ['IMPORT_METAPATH'] = metadata_path dist_name = metadata['distribution'] dist_version = metadata['version'] or 'master' self.status("Distribution `%s' provides module `%s', " "importing `%s-%s'...\n", dist_name, module_name, dist_name, dist_version, bold=True) loop = baserockimport.mainloop.ImportLoop(app=self, goal_kind='cpan', goal_name=dist_name, goal_version=dist_version, generate_chunk_morphs=False, ignore_version_field=True) loop.enable_importer('cpan', strata=['strata/core.morph']) loop.run()