Add import/ tools

This is a generic tool which allows using metadata from foreign
packaging systems to create morphologies.

So far it supports RubyGems, but it should be extendable to other
packaging systems.

It should be considered 'beta' quality right now.

1 files changed, 163 insertions, 0 deletions

diff --git a/import/rubygems.to_lorry b/import/rubygems.to_lorry
new file mode 100755
index 00000000..cd83e33b
--- /dev/null
+++ b/import/rubygems.to_lorry
@@ -0,0 +1,163 @@
+#!/usr/bin/python
+#
+# Create a Baserock .lorry file for a given RubyGem
+#
+# Copyright (C) 2014  Codethink Limited
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation; version 2 of the License.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License along
+# with this program; if not, write to the Free Software Foundation, Inc.,
+# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
+
+
+import requests
+import requests_cache
+import yaml
+
+import logging
+import json
+import os
+import sys
+import urlparse
+
+from importer_base import ImportException, ImportExtension
+
+
+class GenerateLorryException(ImportException):
+    pass
+
+
+class RubyGemsWebServiceClient(object):
+    def __init__(self):
+        # Save hammering the rubygems.org API: 'requests' API calls are
+        # transparently cached in an SQLite database, instead.
+        requests_cache.install_cache('rubygems_api_cache')
+
+    def _request(self, url):
+        r = requests.get(url)
+        if r.ok:
+            return json.loads(r.text)
+        else:
+            raise GenerateLorryException(
+                'Request to %s failed: %s' % (r.url, r.reason))
+
+    def get_gem_info(self, gem_name):
+        info = self._request(
+            'http://rubygems.org/api/v1/gems/%s.json' % gem_name)
+
+        if info['name'] != gem_name:
+            # Sanity check
+            raise GenerateLorryException(
+                 'Received info for Gem "%s", requested "%s"' % info['name'],
+                  gem_name)
+
+        return info
+
+
+class RubyGemLorryGenerator(ImportExtension):
+    def __init__(self):
+        super(RubyGemLorryGenerator, self).__init__()
+
+        with open('rubygems.yaml', 'r') as f:
+            local_data = yaml.load(f.read())
+
+        self.known_source_uris = local_data['known-source-uris']
+
+        logging.debug(
+            "Loaded %i known source URIs from local metadata.", len(self.known_source_uris))
+
+    def process_args(self, args):
+        if len(args) != 1:
+            raise ImportException(
+                'Please call me with the name of a RubyGem as an argument.\n')
+
+        gem_name = args[0]
+
+        lorry = self.generate_lorry_for_gem(gem_name)
+        self.write_lorry(sys.stdout, lorry)
+
+    def find_upstream_repo_for_gem(self, gem_name, gem_info):
+        source_code_uri = gem_info['source_code_uri']
+
+        if gem_name in self.known_source_uris:
+            logging.debug('Found %s in known-source-uris', gem_name)
+            known_uri = self.known_source_uris[gem_name]
+            if source_code_uri is not None and known_uri != source_code_uri:
+                sys.stderr.write(
+                    '%s: Hardcoded source URI %s doesn\'t match spec URI %s\n' %
+                    (gem_name, known_uri, source_code_uri))
+            return known_uri
+
+        if source_code_uri is not None and len(source_code_uri) > 0:
+            logging.debug('Got source_code_uri %s', source_code_uri)
+            if source_code_uri.endswith('/tree'):
+                source_code_uri = source_code_uri[:-len('/tree')]
+
+            return source_code_uri
+
+        homepage_uri = gem_info['homepage_uri']
+        if homepage_uri is not None and len(homepage_uri) > 0:
+            logging.debug('Got homepage_uri %s', source_code_uri)
+            netloc = urlparse.urlsplit(homepage_uri)[1]
+            if netloc == 'github.com':
+                return homepage_uri
+
+        # Further possible leads on locating source code.
+        # http://ruby-toolbox.com/projects/$gemname -> sometimes contains an
+        #   upstream link, even if the gem info does not.
+        # https://github.com/search?q=$gemname -> often the first result is
+        #   the correct one, but you can never know.
+
+        raise GenerateLorryException(
+            "Did not manage to find the upstream source URL for Gem '%s'. "
+            "Please manually create a .lorry file, or add the Gem to "
+            "known-source-uris in rubygems.yaml." % gem_name)
+
+    def project_name_from_repo(self, repo_url):
+        if repo_url.endswith('/tree/master'):
+            repo_url = repo_url[:-len('/tree/master')]
+        if repo_url.endswith('/'):
+            repo_url = repo_url[:-1]
+        if repo_url.endswith('.git'):
+            repo_url = repo_url[:-len('.git')]
+        return os.path.basename(repo_url)
+
+    def generate_lorry_for_gem(self, gem_name):
+        rubygems_client = RubyGemsWebServiceClient()
+
+        gem_info = rubygems_client.get_gem_info(gem_name)
+
+        gem_source_url = self.find_upstream_repo_for_gem(gem_name, gem_info)
+        logging.info('Got URL <%s> for %s', gem_source_url, gem_name)
+
+        project_name = self.project_name_from_repo(gem_source_url)
+
+        # One repo may produce multiple Gems. It's up to the caller to merge
+        # multiple .lorry files that get generated for the same repo.
+
+        lorry = {
+            project_name: {
+                'type': 'git',
+                'url': gem_source_url,
+                'x-products-rubygems': [gem_name]
+            }
+        }
+
+        return lorry
+
+    def write_lorry(self, stream, lorry):
+        json.dump(lorry, stream, indent=4)
+        # Needed so the morphlib.extensions code will pick up the last line.
+        stream.write('\n')
+
+
+if __name__ == '__main__':
+    RubyGemLorryGenerator().run()