From c42ae5bf25c4e938794844590fb5e1be2177844b Mon Sep 17 00:00:00 2001 From: Sam Thursfield Date: Mon, 22 Sep 2014 15:34:29 +0100 Subject: import: Put all lorries for rubygems into a single .lorry file This is what we do for Python packages already. --- import/main.py | 20 +++++++++++++++++++- import/rubygems.to_chunk | 2 +- import/rubygems.to_lorry | 4 +++- import/rubygems.yaml | 2 ++ 4 files changed, 25 insertions(+), 3 deletions(-) diff --git a/import/main.py b/import/main.py index 8d3194af..173cc119 100644 --- a/import/main.py +++ b/import/main.py @@ -162,8 +162,19 @@ class LorrySet(object): else: self.data[project_name] = info + self._add_lorry_entry_to_lorry_file(filename, lorry_entry) + + def _add_lorry_entry_to_lorry_file(self, filename, entry): + if os.path.exists(filename): + with open(filename) as f: + contents = json.load(f) + else: + contents = {} + + contents.update(entry) + with morphlib.savefile.SaveFile(filename, 'w') as f: - json.dump(lorry_entry, f, indent=4) + json.dump(contents, f, indent=4, sort_keys=True) class MorphologySet(morphlib.morphset.MorphologySet): @@ -508,6 +519,13 @@ class BaserockImportApplication(cliapp.Application): lorry_filename = lorry.keys()[0] + if '/' in lorry_filename: + # We try to be a bit clever and guess that if there's a prefix + # in the name, e.g. 'ruby-gems/chef' then it should go in a + # mega-lorry file, such as ruby-gems.lorry. + parts = lorry_filename.split('/', 1) + lorry_filename = parts[0] + if lorry_filename == '': raise cliapp.AppException( 'Invalid lorry data for %s: %s' % (name, lorry)) diff --git a/import/rubygems.to_chunk b/import/rubygems.to_chunk index 39dba14f..fbf9de96 100755 --- a/import/rubygems.to_chunk +++ b/import/rubygems.to_chunk @@ -184,7 +184,7 @@ class RubyGemChunkMorphologyGenerator # control the build and deployment environment, and we obviously can't # provide the private key of the Gem's maintainer. configure_commands << - "sed -e '/cert_chain\\w+=/d' -e '/signing_key\\w+=/d' -i " + + "sed -e '/cert_chain\\s*=/d' -e '/signing_key\\s*=/d' -i " + "#{spec.name}.gemspec" end diff --git a/import/rubygems.to_lorry b/import/rubygems.to_lorry index cd83e33b..898f0514 100755 --- a/import/rubygems.to_lorry +++ b/import/rubygems.to_lorry @@ -69,6 +69,7 @@ class RubyGemLorryGenerator(ImportExtension): with open('rubygems.yaml', 'r') as f: local_data = yaml.load(f.read()) + self.lorry_prefix = local_data['lorry-prefix'] self.known_source_uris = local_data['known-source-uris'] logging.debug( @@ -139,12 +140,13 @@ class RubyGemLorryGenerator(ImportExtension): logging.info('Got URL <%s> for %s', gem_source_url, gem_name) project_name = self.project_name_from_repo(gem_source_url) + lorry_name = self.lorry_prefix + project_name # One repo may produce multiple Gems. It's up to the caller to merge # multiple .lorry files that get generated for the same repo. lorry = { - project_name: { + lorry_name: { 'type': 'git', 'url': gem_source_url, 'x-products-rubygems': [gem_name] diff --git a/import/rubygems.yaml b/import/rubygems.yaml index d31a625a..c9370444 100644 --- a/import/rubygems.yaml +++ b/import/rubygems.yaml @@ -1,5 +1,7 @@ --- +lorry-prefix: ruby-gems/ + # The :development dependency set is way too broad for our needs: for most Gems, # it includes test tools and development aids that aren't necessary for just # building the Gem. It's hard to even get a stratum if we include all these -- cgit v1.2.1