_downloadablefilesource.py: Support version trackingjuerg/tar-tracking

2 files changed, 61 insertions, 12 deletions

diff --git a/buildstream/plugins/sources/_downloadablefilesource.py b/buildstream/plugins/sources/_downloadablefilesource.py
index dba2a7356..c7a622906 100644
--- a/buildstream/plugins/sources/_downloadablefilesource.py
+++ b/buildstream/plugins/sources/_downloadablefilesource.py
@@ -4,30 +4,46 @@ import os
 import urllib.request
 import urllib.error
 import contextlib
+import re
 import shutil
 
+from distutils.version import LooseVersion
+
 from buildstream import Source, SourceError, Consistency
 from buildstream import utils
 
 
 class DownloadableFileSource(Source):
 
-    COMMON_CONFIG_KEYS = Source.COMMON_CONFIG_KEYS + ['url', 'ref', 'etag']
+    COMMON_CONFIG_KEYS = Source.COMMON_CONFIG_KEYS + ['url', 'ref', 'etag', 'version']
 
     def configure(self, node):
         self.original_url = self.node_get_member(node, str, 'url')
+        self.version = self.node_get_member(node, str, 'version', '') or None
         self.ref = self.node_get_member(node, str, 'ref', '') or None
         self.etag = self.node_get_member(node, str, 'etag', '') or None
         self.url = self.translate_url(self.original_url)
 
+        if '{version}' in os.path.basename(self.url):
+            self.url_format = self.url
+            if self.version:
+                self.alias_url = self.original_url.format(version=self.version)
+                self.url = self.url_format.format(version=self.version)
+            else:
+                self.alias_url = None
+                self.url = None
+        else:
+            self.alias_url = self.original_url
+            self.url_format = None
+
     def preflight(self):
         return
 
     def get_unique_key(self):
-        return [self.original_url, self.ref]
+        return [self.alias_url, self.ref]
 
     def get_consistency(self):
-        if self.ref is None:
+        if self.alias_url is None or self.ref is None:
             return Consistency.INCONSISTENT
 
         if os.path.isfile(self._get_mirror_file()):
@@ -37,31 +53,64 @@ class DownloadableFileSource(Source):
             return Consistency.RESOLVED
 
     def get_ref(self):
-        return (self.ref, self.etag)
+        return (self.ref, self.etag, self.version)
 
     def set_ref(self, ref, node):
-        self.ref, self.etag = ref
+        self.ref, self.etag, self.version = ref
 
-        node['ref'] = self.ref
+        if self.version:
+            node['version'] = self.version
         if self.etag:
             node['etag'] = self.etag
+        node['ref'] = self.ref
 
     def track(self):
         # there is no 'track' field in the source to determine what/whether
         # or not to update refs, because tracking a ref is always a conscious
         # decision by the user.
-        with self.timed_activity("Tracking {}".format(self.url),
+        with self.timed_activity("Tracking {}".format(self.url_format or self.url),
                                  silent_nested=True):
+            if self.url_format:
+                try:
+                    filename_format = os.path.basename(self.url_format)
+                    index = filename_format.find('{version}')
+                    escaped_prefix = re.escape('href="' + filename_format[:index])
+                    escaped_suffix = re.escape(filename_format[index + len('{version}'):] + '"')
+                    pattern = re.compile(escaped_prefix + r'([0-9.]+)' + escaped_suffix)
+                    request = urllib.request.Request(self.url_format[:-len(filename_format)])
+                    with contextlib.closing(urllib.request.urlopen(request)) as response:
+                        info = response.info()
+                        charset = info.get_content_charset()
+                        listing = response.read().decode(charset or 'utf-8')
+
+                        new_version = None
+                        for match in re.findall(pattern, listing):
+                            if new_version is None or LooseVersion(match) > LooseVersion(new_version):
+                                new_version = match
+
+                except (urllib.error.URLError, urllib.error.ContentTooShortError, OSError) as e:
+                    raise SourceError("{}: Error tracking {}: {}"
+                                      .format(self, self.url_format, e)) from e
+
+                if new_version is None:
+                    raise SourceError("{}: Error tracking {}: Pattern not found"
+                                      .format(self, self.url_format))
+
+                self.alias_url = self.original_url.format(version=new_version)
+                self.url = self.url_format.format(version=new_version)
+            else:
+                new_version = None
+
             new_ref, new_etag = self._ensure_mirror()
 
-            if self.ref and self.ref != new_ref:
+            if self.ref and self.ref != new_ref and self.version == new_version:
                 detail = "When tracking, new ref differs from current ref:\n" \
                     + "  Tracked URL: {}\n".format(self.url) \
                     + "  Current ref: {}\n".format(self.ref) \
                     + "  New ref: {}\n".format(new_ref)
                 self.warn("Potential man-in-the-middle attack!", detail=detail)
 
-            return (new_ref, new_etag)
+            return (new_ref, new_etag, new_version)
 
     def fetch(self):
 
@@ -125,7 +174,7 @@ class DownloadableFileSource(Source):
 
     def _get_mirror_dir(self):
         return os.path.join(self.get_mirror_directory(),
-                            utils.url_directory_name(self.original_url))
+                            utils.url_directory_name(self.alias_url))
 
     def _get_mirror_file(self, sha=None):
         return os.path.join(self._get_mirror_dir(), sha or self.ref)
diff --git a/buildstream/plugins/sources/tar.py b/buildstream/plugins/sources/tar.py
index 284554fe1..3f020ae36 100644
--- a/buildstream/plugins/sources/tar.py
+++ b/buildstream/plugins/sources/tar.py
@@ -71,7 +71,7 @@ class TarSource(DownloadableFileSource):
 
     def preflight(self):
         self.host_lzip = None
-        if self.url.endswith('.lz'):
+        if self.original_url.endswith('.lz'):
             self.host_lzip = utils.get_host_tool('lzip')
 
     def get_unique_key(self):
@@ -92,7 +92,7 @@ class TarSource(DownloadableFileSource):
 
     @contextmanager
     def _get_tar(self):
-        if self.url.endswith('.lz'):
+        if self.original_url.endswith('.lz'):
             with self._run_lzip() as lzip_dec:
                 with tarfile.open(fileobj=lzip_dec, mode='r:') as tar:
                     yield tar