Make DownloadableFileSource clearly define public and private fieldsjonathan/expose-downloadablefilesource

TarSource and RemoteSource now parse url from the config themselves,
because they need it, but the url is private.

4 files changed, 16 insertions, 6 deletions

diff --git a/buildstream/plugins/sources/deb.py b/buildstream/plugins/sources/deb.py
index 1b7dafd31..b62a976da 100644
--- a/buildstream/plugins/sources/deb.py
+++ b/buildstream/plugins/sources/deb.py
@@ -68,7 +68,7 @@ class DebSource(TarSource):
 
     @contextmanager
     def _get_tar(self):
-        with open(self._get_mirror_file(), 'rb') as deb_file:
+        with open(self.get_mirror_file(), 'rb') as deb_file:
             arpy_archive = arpy.Archive(fileobj=deb_file)
             arpy_archive.read_all_headers()
             data_tar_arpy = [v for k, v in arpy_archive.archived_files.items() if b"data.tar" in k][0]
diff --git a/buildstream/plugins/sources/remote.py b/buildstream/plugins/sources/remote.py
index c296d3158..fbfff9056 100644
--- a/buildstream/plugins/sources/remote.py
+++ b/buildstream/plugins/sources/remote.py
@@ -61,7 +61,13 @@ class RemoteSource(DownloadableFileSource):
     def configure(self, node):
         super().configure(node)
 
-        self.filename = self.node_get_member(node, str, 'filename', os.path.basename(self.url))
+        self.filename = self.node_get_member(node, str, 'filename', None)
+        if not self.filename:
+            # url in DownloadableFileSource is private, so we read it again
+            original_url = self.node_get_member(node, str, 'url')
+            url = self.translate_url(original_url)
+            self.filename = os.path.basename(url)
+
         self.executable = self.node_get_member(node, bool, 'executable', False)
 
         if os.sep in self.filename:
@@ -78,7 +84,7 @@ class RemoteSource(DownloadableFileSource):
         dest = os.path.join(directory, self.filename)
         with self.timed_activity("Staging remote file to {}".format(dest)):
 
-            utils.safe_copy(self._get_mirror_file(), dest)
+            utils.safe_copy(self.get_mirror_file(), dest)
 
             # To prevent user's umask introducing variability here, explicitly set
             # file modes.
diff --git a/buildstream/plugins/sources/tar.py b/buildstream/plugins/sources/tar.py
index 75219dc89..667b28379 100644
--- a/buildstream/plugins/sources/tar.py
+++ b/buildstream/plugins/sources/tar.py
@@ -71,6 +71,10 @@ class TarSource(DownloadableFileSource):
     def configure(self, node):
         super().configure(node)
 
+        # url in DownloadableFileSource is private, so we read it again
+        original_url = self.node_get_member(node, str, 'url')
+        self.url = self.translate_url(original_url)
+
         self.base_dir = self.node_get_member(node, str, 'base-dir', '*') or None
 
         self.node_validate(node, DownloadableFileSource.COMMON_CONFIG_KEYS + ['base-dir'])
@@ -87,7 +91,7 @@ class TarSource(DownloadableFileSource):
     def _run_lzip(self):
         assert self.host_lzip
         with TemporaryFile() as lzip_stdout:
-            with open(self._get_mirror_file(), 'r') as lzip_file:
+            with open(self.get_mirror_file(), 'r') as lzip_file:
                 self.call([self.host_lzip, '-d'],
                           stdin=lzip_file,
                           stdout=lzip_stdout)
@@ -102,7 +106,7 @@ class TarSource(DownloadableFileSource):
                 with tarfile.open(fileobj=lzip_dec, mode='r:') as tar:
                     yield tar
         else:
-            with tarfile.open(self._get_mirror_file()) as tar:
+            with tarfile.open(self.get_mirror_file()) as tar:
                 yield tar
 
     def stage(self, directory):
diff --git a/buildstream/plugins/sources/zip.py b/buildstream/plugins/sources/zip.py
index dafd6f500..5773e0dba 100644
--- a/buildstream/plugins/sources/zip.py
+++ b/buildstream/plugins/sources/zip.py
@@ -83,7 +83,7 @@ class ZipSource(DownloadableFileSource):
         noexec_rights = exec_rights & ~(stat.S_IXUSR | stat.S_IXGRP | stat.S_IXOTH)
 
         try:
-            with zipfile.ZipFile(self._get_mirror_file()) as archive:
+            with zipfile.ZipFile(self.get_mirror_file()) as archive:
                 base_dir = None
                 if self.base_dir:
                     base_dir = self._find_base_dir(archive, self.base_dir)