import: Move hardcoded URIs into external data file

2 files changed, 52 insertions, 38 deletions

diff --git a/import/rubygem.to_lorry b/import/rubygem.to_lorry
index 60e9d925..bb381da3 100755
--- a/import/rubygem.to_lorry
+++ b/import/rubygem.to_lorry
@@ -20,6 +20,7 @@
 
 import requests
 import requests_cache
+import yaml
 
 import logging
 import json
@@ -28,39 +29,6 @@ import sys
 import urlparse
 
 
-known_source_uris = {
-    'ast': 'https://github.com/openSUSE/ast',
-    'brass': 'https://github.com/rubyworks/brass',
-    'coveralls': 'https://github.com/lemurheavy/coveralls-ruby',
-    'diff-lcs': 'https://github.com/halostatue/diff-lcs',
-    'erubis': 'https://github.com/kwatch/erubis',
-    'fog-brightbox': 'https://github.com/brightbox/fog-brightbox',
-    'highline': 'https://github.com/JEG2/highline',
-    'hoe': 'https://github.com/seattlerb/hoe',
-    'indexer': 'https://github.com/rubyworks/indexer',
-    'json': 'https://github.com/flori/json',
-    'method_source': 'https://github.com/banister/method_source',
-    'mixlib-authentication': 'https://github.com/opscode/mixlib-authentication',
-    'mixlib-cli': 'https://github.com/opscode/mixlib-cli',
-    'mixlib-log': 'https://github.com/opscode/mixlib-log',
-    'mixlib-shellout': 'http://github.com/opscode/mixlib-shellout',
-    'ohai': 'http://github.com/opscode/ohai',
-    'rack-cache': 'https://github.com/rtomayko/rack-cache',
-    'actionmailer': 'https://github.com/rails/rails',
-    'actionpack': 'https://github.com/rails/rails',
-    'actionview': 'https://github.com/rails/rails',
-    'activemodel': 'https://github.com/rails/rails',
-    'activerecord': 'https://github.com/rails/rails',
-    'activesupport': 'https://github.com/rails/rails',
-    'rails': 'https://github.com/rails/rails',
-    'railties': 'https://github.com/rails/rails',
-    'pg': 'https://github.com/ged/ruby-pg',
-    'sigar': 'https://github.com/hyperic/sigar',
-    'sprockets': 'https://github.com/sstephenson/sprockets',
-    'tins': 'https://github.com/flori/tins',
-}
-
-
 class RubyGemsWebServiceClient(object):
     def __init__(self):
         # Save hammering the rubygems.org API: 'requests' API calls are
@@ -87,12 +55,21 @@ class RubyGemsWebServiceClient(object):
 
 
 class RubyGemLorryGenerator(object):
+    def __init__(self):
+        with open('rubygems.yaml', 'r') as f:
+            local_data = yaml.load(f.read())
+
+        self.known_source_uris = local_data['known-source-uris']
+
+        logging.debug(
+            "Loaded %i known source URIs from local metadata.", len(self.known_source_uris))
+
     def find_upstream_repo_for_gem(self, gem_name, gem_info):
         source_code_uri = gem_info['source_code_uri']
 
-        if gem_name in known_source_uris:
-            logging.debug('Found %s in known_source_uris', gem_name)
-            known_uri = known_source_uris[gem_name]
+        if gem_name in self.known_source_uris:
+            logging.debug('Found %s in known-source-uris', gem_name)
+            known_uri = self.known_source_uris[gem_name]
             if source_code_uri is not None and known_uri != source_code_uri:
                 sys.stderr.write(
                     '%s: Hardcoded source URI %s doesn\'t match spec URI %s\n' %
@@ -119,8 +96,10 @@ class RubyGemLorryGenerator(object):
         # https://github.com/search?q=$gemname -> often the first result is
         #   the correct one, but you can never know.
 
-        raise Exception('Did not manage to automatically find the upstream '
-                        'source URL for Gem %s.' % gem_name)
+        raise Exception(
+            "Did not manage to find the upstream source URL for Gem '%s'. "
+            "Please manually create a .lorry file, or add the Gem to "
+            "known-source-uris in rubygems.yaml." % gem_name)
 
     def project_name_from_repo(self, repo_url):
         if repo_url.endswith('/tree/master'):
diff --git a/import/rubygems.yaml b/import/rubygems.yaml
new file mode 100644
index 00000000..8fdf57b3
--- /dev/null
+++ b/import/rubygems.yaml
@@ -0,0 +1,35 @@
+---
+
+# The following Gems don't provide a source_code_uri in their Gem metadata.
+# Ideally ... they would do.
+known-source-uris:
+  ast: https://github.com/openSUSE/ast
+  brass: https://github.com/rubyworks/brass
+  coveralls: https://github.com/lemurheavy/coveralls-ruby
+  diff-lcs: https://github.com/halostatue/diff-lcs
+  erubis: https://github.com/kwatch/erubis
+  fog-brightbox: https://github.com/brightbox/fog-brightbox
+  highline: https://github.com/JEG2/highline
+  hoe: https://github.com/seattlerb/hoe
+  indexer: https://github.com/rubyworks/indexer
+  json: https://github.com/flori/json
+  method_source: https://github.com/banister/method_source
+  mixlib-authentication: https://github.com/opscode/mixlib-authentication
+  mixlib-cli: https://github.com/opscode/mixlib-cli
+  mixlib-log: https://github.com/opscode/mixlib-log
+  mixlib-shellout: http://github.com/opscode/mixlib-shellout
+  ohai: http://github.com/opscode/ohai
+  rack-cache: https://github.com/rtomayko/rack-cache
+  actionmailer: https://github.com/rails/rails
+  actionpack: https://github.com/rails/rails
+  actionview: https://github.com/rails/rails
+  activejob: https://github.com/rails/rails
+  activemodel: https://github.com/rails/rails
+  activerecord: https://github.com/rails/rails
+  activesupport: https://github.com/rails/rails
+  rails: https://github.com/rails/rails
+  railties: https://github.com/rails/rails
+  pg: https://github.com/ged/ruby-pg
+  sigar: https://github.com/hyperic/sigar
+  sprockets: https://github.com/sstephenson/sprockets
+  tins: https://github.com/flori/tins