diff options
Diffstat (limited to 'baserockimport')
-rw-r--r-- | baserockimport/app.py | 7 | ||||
-rw-r--r-- | baserockimport/data/python.yaml | 2 | ||||
-rw-r--r-- | baserockimport/exts/importer_python_common.py | 20 | ||||
-rwxr-xr-x | baserockimport/exts/npm.to_lorry | 12 | ||||
-rwxr-xr-x | baserockimport/exts/python.find_deps | 45 | ||||
-rwxr-xr-x | baserockimport/exts/python.to_lorry | 122 | ||||
-rwxr-xr-x | baserockimport/exts/python_lorry_tests.py | 11 | ||||
-rwxr-xr-x | baserockimport/exts/rubygems.to_lorry | 5 | ||||
-rw-r--r-- | baserockimport/lorryset.py | 5 | ||||
-rw-r--r-- | baserockimport/mainloop.py | 36 |
10 files changed, 161 insertions, 104 deletions
diff --git a/baserockimport/app.py b/baserockimport/app.py index cd3203e..0b190e5 100644 --- a/baserockimport/app.py +++ b/baserockimport/app.py @@ -206,6 +206,10 @@ class BaserockImportApplication(cliapp.Application): def import_python(self, args): '''Import one or more python packages.''' + + def comp(x, y): + return x.replace('-', '_').lower() == y.replace('-', '_').lower() + if len(args) < 1 or len(args) > 2: raise cliapp.AppException( 'Please pass the name of the python package on the commandline.') @@ -218,5 +222,6 @@ class BaserockImportApplication(cliapp.Application): goal_kind='python', goal_name=package_name, goal_version=package_version) - loop.enable_importer('python', strata=['strata/core.morph']) + loop.enable_importer('python', strata=['strata/core.morph'], + package_comp_callback=comp) loop.run() diff --git a/baserockimport/data/python.yaml b/baserockimport/data/python.yaml new file mode 100644 index 0000000..6836714 --- /dev/null +++ b/baserockimport/data/python.yaml @@ -0,0 +1,2 @@ +--- +lorry-prefix: python diff --git a/baserockimport/exts/importer_python_common.py b/baserockimport/exts/importer_python_common.py index b2e7c51..eedc81b 100644 --- a/baserockimport/exts/importer_python_common.py +++ b/baserockimport/exts/importer_python_common.py @@ -23,6 +23,15 @@ from importer_base import ImportExtension PYPI_URL = 'http://pypi.python.org/pypi' +def get_releases(client, package_name): + try: + # True here just means show hidden releases + releases = client.package_releases(package_name, True) + except Exception as e: + error("Couldn't fetch release data:", e) + + return releases + def warn(*args, **kwargs): print('%s:' % sys.argv[0], *args, file=sys.stderr, **kwargs) @@ -60,10 +69,11 @@ def name_or_closest(client, package_name): # # so look for both the hyphenated version that is passed to this function # and the underscored version. + package_name = package_name.replace('_', '-') underscored_package_name = package_name.replace('-', '_') for name in [package_name, underscored_package_name]: - results = client.package_releases(name) + results = get_releases(client, name) if len(results) > 0: logging.debug('Found package %s' % name) @@ -88,12 +98,4 @@ def name_or_closest(client, package_name): return results[0]['name'] if len(results) > 0 else None -# We subclass the ImportExtension to setup the logger, -# so that we can send logs to the import tool's log -class PythonExtension(ImportExtension): - def __init__(self): - super(PythonExtension, self).__init__() - def process_args(self, _): - import __main__ - __main__.main() diff --git a/baserockimport/exts/npm.to_lorry b/baserockimport/exts/npm.to_lorry index ba0f442..65d8712 100755 --- a/baserockimport/exts/npm.to_lorry +++ b/baserockimport/exts/npm.to_lorry @@ -20,11 +20,13 @@ npm = require("npm"); base = require("./importer_base"); npm.load(function(er, npm) { - if (process.argv.length === 3) - packageName = process.argv[2]; - else - throw ("Error! Too many command line arguments! Usage: " + - "./npm.to_lorry PACKAGENAME"); + + if (! (process.argv.length == 3 || process.argv.length == 4)) { + throw ("Usage: ./npm.to_lorry PACKAGENAME [VERSION]"); + } + + packageName = process.argv[2]; + if (er) throw er; npm.commands.view([packageName], "silent", getRepo); diff --git a/baserockimport/exts/python.find_deps b/baserockimport/exts/python.find_deps index b9791ef..a73c457 100755 --- a/baserockimport/exts/python.find_deps +++ b/baserockimport/exts/python.find_deps @@ -189,7 +189,7 @@ def resolve_versions(specsets): logging.debug("Treating %s as %s" % (proj_name, new_proj_name)) proj_name = new_proj_name - releases = client.package_releases(proj_name) + releases = get_releases(client, proj_name) logging.debug('Found %d releases of %s: %s' % (len(releases), proj_name, releases)) @@ -327,31 +327,36 @@ def find_runtime_deps(source, name, version=None, use_requirements_file=False): return runtime_deps -def main(): - if len(sys.argv) not in [3, 4]: - print('usage: %s PACKAGE_SOURCE_DIR NAME [VERSION]' % sys.argv[0]) - sys.exit(1) +class PythonFindDepsExtension(ImportExtension): - logging.debug('%s: sys.argv[1:]: %s' % (sys.argv[0], sys.argv[1:])) - source, name = sys.argv[1:3] - version = sys.argv[3] if len(sys.argv) == 4 else None + def __init__(self): + super(PythonFindDepsExtension, self).__init__() - client = xmlrpclib.ServerProxy(PYPI_URL) - new_name = name_or_closest(client, name) + def run(self): + if len(sys.argv) not in [3, 4]: + print('usage: %s PACKAGE_SOURCE_DIR NAME [VERSION]' % sys.argv[0]) + sys.exit(1) - if new_name == None: - error("Couldn't find any project with name '%s'" % name) + logging.debug('%s: sys.argv[1:]: %s' % (sys.argv[0], sys.argv[1:])) + source, name = sys.argv[1:3] + version = sys.argv[3] if len(sys.argv) == 4 else None - logging.debug('Treating %s as %s' % (name, new_name)) - name = new_name + client = xmlrpclib.ServerProxy(PYPI_URL) + new_name = name_or_closest(client, name) + + if new_name == None: + error("Couldn't find any project with name '%s'" % name) + + logging.debug('Treating %s as %s' % (name, new_name)) + name = new_name - deps = {} - deps['build-dependencies'] = find_build_deps(source, name, version) - deps['runtime-dependencies'] = find_runtime_deps(source, name, version) + deps = {} + deps['build-dependencies'] = find_build_deps(source, name, version) + deps['runtime-dependencies'] = find_runtime_deps(source, name, version) - root = {'python': deps} + root = {'python': deps} - print(json.dumps(root)) + print(json.dumps(root)) if __name__ == '__main__': - PythonExtension().run() + PythonFindDepsExtension().run() diff --git a/baserockimport/exts/python.to_lorry b/baserockimport/exts/python.to_lorry index accc9dc..8ee20eb 100755 --- a/baserockimport/exts/python.to_lorry +++ b/baserockimport/exts/python.to_lorry @@ -28,7 +28,7 @@ import shutil import tempfile import xmlrpclib import logging -import select +import yaml import pkg_resources @@ -47,6 +47,8 @@ def fetch_package_metadata(package_name): def find_repo_type(url): + debug_vcss = False + # Don't bother with detection if we can't get a 200 OK logging.debug("Getting '%s' ..." % url) @@ -79,7 +81,8 @@ def find_repo_type(url): if line == '': break - logging.debug(line.rstrip('\n')) + if debug_vcss: + logging.debug(line.rstrip('\n')) p.wait() # even with eof on both streams, we still wait @@ -109,11 +112,13 @@ def get_compression(url): return None # Assumption: url passed to this function must have a 'standard' tar extension -def make_tarball_lorry(package_name, url): - # TODO: this prefix probably shouldn't be hardcoded here either - name = 'python-packages/%s' % package_name.lower() +def make_tarball_lorry(lorry_prefix, package_name, url): + name = '%s/%s' % (lorry_prefix, package_name) - lorry = {'type': 'tarball', 'url': url} + # TODO: shouldn't have 'x-products-python' field hardcoded here either + lorry = {'type': 'tarball', + 'url': url, + 'x-products-python': [package_name]} compression = get_compression(url) if compression: lorry['compression'] = compression @@ -130,36 +135,35 @@ def filter_urls(urls): return filter(allowed_extension, urls) -def get_releases(client, requirement): +def get_releases(client, package_name): try: - releases = client.package_releases(requirement.project_name) + releases = client.package_releases(package_name) except Exception as e: error("Couldn't fetch release data:", e) return releases -def generate_tarball_lorry(client, requirement): - releases = get_releases(client, requirement) +def generate_tarball_lorry(lorry_prefix, client, package_name, version=None): + releases = get_releases(client, package_name) if len(releases) == 0: - error("Couldn't find any releases for package %s" - % requirement.project_name) + error("Couldn't find any releases for package %s" % package_name) - releases = [v for v in releases if specs_satisfied(v, requirement.specs)] + logging.debug('Found releases: %s', str(releases)) - if len(releases) == 0: - error("Couldn't find any releases of %s" - " that satisfy version constraints: %s" - % (requirement.project_name, requirement.specs)) + # Use latest release if no version specified + version = version or releases[0] - release_version = releases[0] + if version not in releases: + error("Couldn't find any releases of %s with version: %s" + % (package_name, version)) - logging.debug('Fetching urls for package %s with version %s' - % (requirement.project_name, release_version)) + logging.debug('Fetching urls for package %s with version %s', + package_name, version) try: # Get a list of dicts, the dicts contain the urls. - urls = client.release_urls(requirement.project_name, release_version) + urls = client.release_urls(package_name, version) except Exception as e: error("Couldn't fetch release urls:", e) @@ -173,48 +177,66 @@ def generate_tarball_lorry(client, requirement): warn("\t%s" % url['url']) error("Cannot proceed") else: - error("Couldn't find any download urls for package %s" - % requirement.project_name) + error("Couldn't find any download urls for package %s" % package_name) url = urls[0]['url'] - return make_tarball_lorry(requirement.project_name, url) + return make_tarball_lorry(lorry_prefix, package_name, url) + +def str_repo_lorry(lorry_prefix, package_name, repo_type, url): + name = '%s/%s' % (lorry_prefix, package_name) + + # TODO: this products field 'x-products-python' + # probably shouldn't be hardcoded here + + lorry = {'type': repo_type, 'url': url, 'x-products-python': [package_name]} + + if repo_type == 'svn': + lorry['layout'] = 'standard' + + return json.dumps({name: lorry}, indent=4, sort_keys=True) + +class PythonLorryExtension(ImportExtension): + + def __init__(self): + super(PythonLorryExtension, self).__init__() -def str_repo_lorry(package_name, repo_type, url): - # TODO: this prefix probably shouldn't be hardcoded here - name = 'python-packages/%s' % package_name.lower() + def run(self): + if len(sys.argv) not in [2, 3]: + print('usage: %s NAME [VERSION]' % sys.argv[0], file=sys.stderr) + sys.exit(1) - return json.dumps({name: {'type': repo_type, 'url': url}}, - indent=4, sort_keys=True) + client = xmlrpclib.ServerProxy(PYPI_URL) -def main(): - if len(sys.argv) != 2: - # TODO explain the format of python requirements - # warn the user that they probably want to quote their arg - # > < will be interpreted as redirection by the shell - print('usage: %s requirement' % sys.argv[0], file=sys.stderr) - sys.exit(1) + package_name = sys.argv[1] + version = sys.argv[2] if len(sys.argv) == 3 else None - client = xmlrpclib.ServerProxy(PYPI_URL) + logging.debug('Looking for package: %s with version %s', + package_name, version) - req = pkg_resources.parse_requirements(sys.argv[1]).next() + with open(self.local_data_path('python.yaml')) as f: + lorry_prefix = yaml.load(f)['lorry-prefix'] - new_proj_name = name_or_closest(client, req.project_name) + new_proj_name = name_or_closest(client, package_name) - if new_proj_name == None: - error("Couldn't find any project with name '%s'" % req.project_name) + if new_proj_name == None: + error("Couldn't find any project with name '%s'" % package_name) - logging.debug('Treating %s as %s' % (req.project_name, new_proj_name)) - req.project_name = new_proj_name + logging.debug('Treating %s as %s' % (package_name, new_proj_name)) + package_name = new_proj_name - metadata = fetch_package_metadata(req.project_name) - info = metadata['info'] + metadata = fetch_package_metadata(package_name) + info = metadata['info'] - repo_type = (find_repo_type(info['home_page']) - if 'home_page' in info else None) + repo_type = (find_repo_type(info['home_page']) + if 'home_page' in info else None) - print(str_repo_lorry(req.project_name, repo_type, info['home_page']) - if repo_type else generate_tarball_lorry(client, req)) + if repo_type: + print(str_repo_lorry(lorry_prefix, package_name, + repo_type, info['home_page'])) + else: + print(generate_tarball_lorry(lorry_prefix, client, + package_name, version)) if __name__ == '__main__': - PythonExtension().run() + PythonLorryExtension().run() diff --git a/baserockimport/exts/python_lorry_tests.py b/baserockimport/exts/python_lorry_tests.py index 12ef564..be4bcac 100755 --- a/baserockimport/exts/python_lorry_tests.py +++ b/baserockimport/exts/python_lorry_tests.py @@ -21,6 +21,8 @@ import json import unittest +LORRY_PREFIX = 'python' + class Tests(unittest.TestCase): def test_make_tarball_lorry(self): @@ -35,18 +37,19 @@ class Tests(unittest.TestCase): return 'http://foobar/baz.%s' % extension def get_tarball_lorry_url(name, lorry_json): - return json.loads(lorry_json)['python-packages/' + return json.loads(lorry_json)[LORRY_PREFIX + '/' + name + '-tarball']['url'] def get_tarball_lorry_compression(name, lorry_json): - return json.loads(lorry_json)['python-packages/' + return json.loads(lorry_json)[LORRY_PREFIX + '/' + name + '-tarball']['compression'] fake_package_name = 'name' urls = [(make_url(ext), ext) for ext in valid_extensions] for (url, ext) in urls: - lorry_json = python_lorry.make_tarball_lorry('name', url) + lorry_json = python_lorry.make_tarball_lorry(LORRY_PREFIX, + 'name', url) print lorry_json tarball_url = get_tarball_lorry_url(fake_package_name, lorry_json) @@ -61,7 +64,7 @@ class Tests(unittest.TestCase): self.assertEqual(tarball_compression, valid_extensions[ext]) url = 'http://foobar/baz.tar' - lorry_json = python_lorry.make_tarball_lorry('name', url) + lorry_json = python_lorry.make_tarball_lorry(LORRY_PREFIX, 'name', url) self.assertEqual(get_tarball_lorry_url(fake_package_name, lorry_json), url) self.assertTrue('compression' not in lorry_json) diff --git a/baserockimport/exts/rubygems.to_lorry b/baserockimport/exts/rubygems.to_lorry index d5f1efa..0413204 100755 --- a/baserockimport/exts/rubygems.to_lorry +++ b/baserockimport/exts/rubygems.to_lorry @@ -76,9 +76,10 @@ class RubyGemLorryGenerator(ImportExtension): "Loaded %i known source URIs from local metadata.", len(self.known_source_uris)) def process_args(self, args): - if len(args) != 1: + if len(args) not in [1, 2]: raise ImportException( - 'Please call me with the name of a RubyGem as an argument.') + 'Please call me with the name of a RubyGem as an argument' + ' and optionally its version number (format: NAME [VERSION])') gem_name = args[0] diff --git a/baserockimport/lorryset.py b/baserockimport/lorryset.py index 8cc73af..f252b9f 100644 --- a/baserockimport/lorryset.py +++ b/baserockimport/lorryset.py @@ -107,7 +107,7 @@ class LorrySet(object): '''Return the lorry entry for the named project.''' return {name: self.data[name]} - def find_lorry_for_package(self, kind, package_name): + def find_lorry_for_package(self, kind, package_name, comp): '''Find the lorry entry for a given foreign package, or return None. This makes use of an extension to the .lorry format made by the @@ -116,11 +116,12 @@ class LorrySet(object): named $KIND. ''' + key = 'x-products-%s' % kind for name, lorry in self.data.iteritems(): products = lorry.get(key, []) for entry in products: - if entry == package_name: + if comp(entry, package_name): return {name: lorry} return None diff --git a/baserockimport/mainloop.py b/baserockimport/mainloop.py index 057ab98..2ffbd82 100644 --- a/baserockimport/mainloop.py +++ b/baserockimport/mainloop.py @@ -202,11 +202,15 @@ class ImportLoop(object): # 1. Make the source code available. - lorry = self._find_or_create_lorry_file(kind, name) + lorry = self._find_or_create_lorry_file(kind, name, version) source_repo, url = self._fetch_or_update_source(lorry) checked_out_version, ref = self._checkout_source_version_for_package( source_repo, package) + + logging.debug('Checked out version: %s\tRef: %s', + checked_out_version, ref) + package.set_version_in_use(checked_out_version) repo_path = os.path.relpath(source_repo.dirname) @@ -288,15 +292,22 @@ class ImportLoop(object): dep_package.set_is_build_dep(True) processed.add_edge(dep_package, current_item) - def _find_or_create_lorry_file(self, kind, name): + def _find_or_create_lorry_file(self, kind, name, version): # Note that the lorry file may already exist for 'name', but lorry # files are named for project name rather than package name. In this # case we will generate the lorry, and try to add it to the set, at # which point LorrySet will notice the existing one and merge the two. - lorry = self.lorry_set.find_lorry_for_package(kind, name) + comp = None + + if 'package_comp_callback' in self.importers[kind]['kwargs']: + comp = self.importers[kind]['kwargs']['package_comp_callback'] + else: + comp = lambda x, y: x == y + + lorry = self.lorry_set.find_lorry_for_package(kind, name, comp) if lorry is None: - lorry = self._generate_lorry_for_package(kind, name) + lorry = self._generate_lorry_for_package(kind, name, version) if len(lorry) != 1: raise Exception( @@ -323,14 +334,18 @@ class ImportLoop(object): return lorry - def _generate_lorry_for_package(self, kind, name): + def _generate_lorry_for_package(self, kind, name, version): tool = '%s.to_lorry' % kind if kind not in self.importers: raise Exception('Importer for %s was not enabled.' % kind) extra_args = self.importers[kind]['extra_args'] self.app.status( '%s: calling %s to generate lorry', name, tool) - lorry_text = run_extension(tool, extra_args + [name]) + + args = extra_args + [name] + if version != 'master': + args.append(version) + lorry_text = run_extension(tool, args) try: lorry = json.loads(lorry_text) except ValueError: @@ -391,11 +406,10 @@ class ImportLoop(object): name = package.name version = package.version - possible_names = [ - version, - 'v%s' % version, - '%s-%s' % (name, version) - ] + possible_names = [] + + for v in [version, version.rstrip('.0'), '%s.0' % version]: + possible_names += [v, 'v%s' % v, '%s-%s' % (name, v)] for tag_name in possible_names: if source_repo.ref_exists(tag_name): |