Add support for .netrc in remote/tar/zip plugins

Fixes #723.

8 files changed, 429 insertions, 1 deletions

diff --git a/buildstream/plugins/sources/_downloadablefilesource.py b/buildstream/plugins/sources/_downloadablefilesource.py
index 7d1fc07bf..f5c5b3d08 100644
--- a/buildstream/plugins/sources/_downloadablefilesource.py
+++ b/buildstream/plugins/sources/_downloadablefilesource.py
@@ -5,16 +5,77 @@ import urllib.request
 import urllib.error
 import contextlib
 import shutil
+import netrc
 
 from buildstream import Source, SourceError, Consistency
 from buildstream import utils
 
 
+class _NetrcFTPOpener(urllib.request.FTPHandler):
+
+    def __init__(self, netrc_config):
+        self.netrc = netrc_config
+
+    def _split(self, netloc):
+        userpass, hostport = urllib.parse.splituser(netloc)
+        host, port = urllib.parse.splitport(hostport)
+        if userpass:
+            user, passwd = urllib.parse.splitpasswd(userpass)
+        else:
+            user = None
+            passwd = None
+        return host, port, user, passwd
+
+    def _unsplit(self, host, port, user, passwd):
+        if port:
+            host = '{}:{}'.format(host, port)
+        if user:
+            if passwd:
+                user = '{}:{}'.format(user, passwd)
+            host = '{}@{}'.format(user, host)
+
+        return host
+
+    def ftp_open(self, req):
+        host, port, user, passwd = self._split(req.host)
+
+        if user is None and self.netrc:
+            entry = self.netrc.authenticators(host)
+            if entry:
+                user, _, passwd = entry
+
+        req.host = self._unsplit(host, port, user, passwd)
+
+        return super().ftp_open(req)
+
+
+class _NetrcPasswordManager:
+
+    def __init__(self, netrc_config):
+        self.netrc = netrc_config
+
+    def add_password(self, realm, uri, user, passwd):
+        pass
+
+    def find_user_password(self, realm, authuri):
+        if not self.netrc:
+            return None, None
+        parts = urllib.parse.urlsplit(authuri)
+        entry = self.netrc.authenticators(parts.hostname)
+        if not entry:
+            return None, None
+        else:
+            login, _, password = entry
+            return login, password
+
+
 class DownloadableFileSource(Source):
     # pylint: disable=attribute-defined-outside-init
 
     COMMON_CONFIG_KEYS = Source.COMMON_CONFIG_KEYS + ['url', 'ref', 'etag']
 
+    __urlopener = None
+
     def configure(self, node):
         self.original_url = self.node_get_member(node, str, 'url')
         self.ref = self.node_get_member(node, str, 'ref', None)
@@ -118,7 +179,8 @@ class DownloadableFileSource(Source):
                     if etag and self.get_consistency() == Consistency.CACHED:
                         request.add_header('If-None-Match', etag)
 
-                with contextlib.closing(urllib.request.urlopen(request)) as response:
+                opener = self.__get_urlopener()
+                with contextlib.closing(opener.open(request)) as response:
                     info = response.info()
 
                     etag = info['ETag'] if 'ETag' in info else None
@@ -164,3 +226,19 @@ class DownloadableFileSource(Source):
 
     def _get_mirror_file(self, sha=None):
         return os.path.join(self._get_mirror_dir(), sha or self.ref)
+
+    def __get_urlopener(self):
+        if not DownloadableFileSource.__urlopener:
+            try:
+                netrc_config = netrc.netrc()
+            except FileNotFoundError:
+                DownloadableFileSource.__urlopener = urllib.request.build_opener()
+            except netrc.NetrcParseError as e:
+                self.warn('{}: While reading .netrc: {}'.format(self, e))
+                return urllib.request.build_opener()
+            else:
+                netrc_pw_mgr = _NetrcPasswordManager(netrc_config)
+                http_auth = urllib.request.HTTPBasicAuthHandler(netrc_pw_mgr)
+                ftp_handler = _NetrcFTPOpener(netrc_config)
+                DownloadableFileSource.__urlopener = urllib.request.build_opener(http_auth, ftp_handler)
+        return DownloadableFileSource.__urlopener
diff --git a/dev-requirements.txt b/dev-requirements.txt
index 1b175c257..380e734c8 100644
--- a/dev-requirements.txt
+++ b/dev-requirements.txt
@@ -9,3 +9,4 @@ pytest-pep8
 pytest-pylint
 pytest-xdist
 pytest-timeout
+pyftpdlib
diff --git a/tests/sources/remote.py b/tests/sources/remote.py
index d3968395f..a4a9d5965 100644
--- a/tests/sources/remote.py
+++ b/tests/sources/remote.py
@@ -5,6 +5,7 @@ import pytest
 from buildstream._exceptions import ErrorDomain
 from buildstream import _yaml
 from tests.testutils import cli
+from tests.testutils.file_server import create_file_server
 
 DATA_DIR = os.path.join(
     os.path.dirname(os.path.realpath(__file__)),
@@ -22,6 +23,16 @@ def generate_project(project_dir, tmpdir):
     }, project_file)
 
 
+def generate_project_file_server(server, project_dir):
+    project_file = os.path.join(project_dir, "project.conf")
+    _yaml.dump({
+        'name': 'foo',
+        'aliases': {
+            'tmpdir': server.base_url()
+        }
+    }, project_file)
+
+
 # Test that without ref, consistency is set appropriately.
 @pytest.mark.datafiles(os.path.join(DATA_DIR, 'no-ref'))
 def test_no_ref(cli, tmpdir, datafiles):
@@ -164,3 +175,35 @@ def test_executable(cli, tmpdir, datafiles):
     assert (mode & stat.S_IEXEC)
     # Assert executable by anyone
     assert(mode & (stat.S_IEXEC | stat.S_IXGRP | stat.S_IXOTH))
+
+
+@pytest.mark.parametrize('server_type', ('FTP', 'HTTP'))
+@pytest.mark.datafiles(os.path.join(DATA_DIR, 'single-file'))
+def test_use_netrc(cli, datafiles, server_type, tmpdir):
+    fake_home = os.path.join(str(tmpdir), 'fake_home')
+    os.makedirs(fake_home, exist_ok=True)
+    project = str(datafiles)
+    checkoutdir = os.path.join(str(tmpdir), 'checkout')
+
+    os.environ['HOME'] = fake_home
+    with open(os.path.join(fake_home, '.netrc'), 'wb') as f:
+        os.fchmod(f.fileno(), 0o700)
+        f.write(b'machine 127.0.0.1\n')
+        f.write(b'login testuser\n')
+        f.write(b'password 12345\n')
+
+    with create_file_server(server_type) as server:
+        server.add_user('testuser', '12345', project)
+        generate_project_file_server(server, project)
+
+        server.start()
+
+        result = cli.run(project=project, args=['fetch', 'target.bst'])
+        result.assert_success()
+        result = cli.run(project=project, args=['build', 'target.bst'])
+        result.assert_success()
+        result = cli.run(project=project, args=['checkout', 'target.bst', checkoutdir])
+        result.assert_success()
+
+        checkout_file = os.path.join(checkoutdir, 'file')
+        assert(os.path.exists(checkout_file))
diff --git a/tests/sources/tar.py b/tests/sources/tar.py
index 1fd79f10b..1a1f54f87 100644
--- a/tests/sources/tar.py
+++ b/tests/sources/tar.py
@@ -3,11 +3,13 @@ import pytest
 import tarfile
 import tempfile
 import subprocess
+import urllib.parse
 from shutil import copyfile, rmtree
 
 from buildstream._exceptions import ErrorDomain
 from buildstream import _yaml
 from tests.testutils import cli
+from tests.testutils.file_server import create_file_server
 from tests.testutils.site import HAVE_LZIP
 from . import list_dir_contents
 
@@ -49,6 +51,16 @@ def generate_project(project_dir, tmpdir):
     }, project_file)
 
 
+def generate_project_file_server(base_url, project_dir):
+    project_file = os.path.join(project_dir, "project.conf")
+    _yaml.dump({
+        'name': 'foo',
+        'aliases': {
+            'tmpdir': base_url
+        }
+    }, project_file)
+
+
 # Test that without ref, consistency is set appropriately.
 @pytest.mark.datafiles(os.path.join(DATA_DIR, 'no-ref'))
 def test_no_ref(cli, tmpdir, datafiles):
@@ -302,3 +314,77 @@ def test_read_only_dir(cli, tmpdir, datafiles):
             else:
                 os.remove(path)
         rmtree(str(tmpdir), onerror=make_dir_writable)
+
+
+@pytest.mark.parametrize('server_type', ('FTP', 'HTTP'))
+@pytest.mark.datafiles(os.path.join(DATA_DIR, 'fetch'))
+def test_use_netrc(cli, datafiles, server_type, tmpdir):
+    file_server_files = os.path.join(str(tmpdir), 'file_server')
+    fake_home = os.path.join(str(tmpdir), 'fake_home')
+    os.makedirs(file_server_files, exist_ok=True)
+    os.makedirs(fake_home, exist_ok=True)
+    project = str(datafiles)
+    checkoutdir = os.path.join(str(tmpdir), 'checkout')
+
+    os.environ['HOME'] = fake_home
+    with open(os.path.join(fake_home, '.netrc'), 'wb') as f:
+        os.fchmod(f.fileno(), 0o700)
+        f.write(b'machine 127.0.0.1\n')
+        f.write(b'login testuser\n')
+        f.write(b'password 12345\n')
+
+    with create_file_server(server_type) as server:
+        server.add_user('testuser', '12345', file_server_files)
+        generate_project_file_server(server.base_url(), project)
+
+        src_tar = os.path.join(file_server_files, 'a.tar.gz')
+        _assemble_tar(os.path.join(str(datafiles), 'content'), 'a', src_tar)
+
+        server.start()
+
+        result = cli.run(project=project, args=['track', 'target.bst'])
+        result.assert_success()
+        result = cli.run(project=project, args=['fetch', 'target.bst'])
+        result.assert_success()
+        result = cli.run(project=project, args=['build', 'target.bst'])
+        result.assert_success()
+        result = cli.run(project=project, args=['checkout', 'target.bst', checkoutdir])
+        result.assert_success()
+
+        original_dir = os.path.join(str(datafiles), 'content', 'a')
+        original_contents = list_dir_contents(original_dir)
+        checkout_contents = list_dir_contents(checkoutdir)
+        assert(checkout_contents == original_contents)
+
+
+@pytest.mark.parametrize('server_type', ('FTP', 'HTTP'))
+@pytest.mark.datafiles(os.path.join(DATA_DIR, 'fetch'))
+def test_netrc_already_specified_user(cli, datafiles, server_type, tmpdir):
+    file_server_files = os.path.join(str(tmpdir), 'file_server')
+    fake_home = os.path.join(str(tmpdir), 'fake_home')
+    os.makedirs(file_server_files, exist_ok=True)
+    os.makedirs(fake_home, exist_ok=True)
+    project = str(datafiles)
+    checkoutdir = os.path.join(str(tmpdir), 'checkout')
+
+    os.environ['HOME'] = fake_home
+    with open(os.path.join(fake_home, '.netrc'), 'wb') as f:
+        os.fchmod(f.fileno(), 0o700)
+        f.write(b'machine 127.0.0.1\n')
+        f.write(b'login testuser\n')
+        f.write(b'password 12345\n')
+
+    with create_file_server(server_type) as server:
+        server.add_user('otheruser', '12345', file_server_files)
+        parts = urllib.parse.urlsplit(server.base_url())
+        base_url = urllib.parse.urlunsplit([parts[0]] + ['otheruser@{}'.format(parts[1])] + list(parts[2:]))
+        generate_project_file_server(base_url, project)
+
+        src_tar = os.path.join(file_server_files, 'a.tar.gz')
+        _assemble_tar(os.path.join(str(datafiles), 'content'), 'a', src_tar)
+
+        server.start()
+
+        result = cli.run(project=project, args=['track', 'target.bst'])
+        result.assert_main_error(ErrorDomain.STREAM, None)
+        result.assert_task_error(ErrorDomain.SOURCE, None)
diff --git a/tests/sources/zip.py b/tests/sources/zip.py
index 73767ee79..6ad6d4077 100644
--- a/tests/sources/zip.py
+++ b/tests/sources/zip.py
@@ -5,6 +5,7 @@ import zipfile
 from buildstream._exceptions import ErrorDomain
 from buildstream import _yaml
 from tests.testutils import cli
+from tests.testutils.file_server import create_file_server
 from . import list_dir_contents
 
 DATA_DIR = os.path.join(
@@ -35,6 +36,16 @@ def generate_project(project_dir, tmpdir):
     }, project_file)
 
 
+def generate_project_file_server(server, project_dir):
+    project_file = os.path.join(project_dir, "project.conf")
+    _yaml.dump({
+        'name': 'foo',
+        'aliases': {
+            'tmpdir': server.base_url()
+        }
+    }, project_file)
+
+
 # Test that without ref, consistency is set appropriately.
 @pytest.mark.datafiles(os.path.join(DATA_DIR, 'no-ref'))
 def test_no_ref(cli, tmpdir, datafiles):
@@ -176,3 +187,44 @@ def test_stage_explicit_basedir(cli, tmpdir, datafiles):
     original_contents = list_dir_contents(original_dir)
     checkout_contents = list_dir_contents(checkoutdir)
     assert(checkout_contents == original_contents)
+
+
+@pytest.mark.parametrize('server_type', ('FTP', 'HTTP'))
+@pytest.mark.datafiles(os.path.join(DATA_DIR, 'fetch'))
+def test_use_netrc(cli, datafiles, server_type, tmpdir):
+    file_server_files = os.path.join(str(tmpdir), 'file_server')
+    fake_home = os.path.join(str(tmpdir), 'fake_home')
+    os.makedirs(file_server_files, exist_ok=True)
+    os.makedirs(fake_home, exist_ok=True)
+    project = str(datafiles)
+    checkoutdir = os.path.join(str(tmpdir), 'checkout')
+
+    os.environ['HOME'] = fake_home
+    with open(os.path.join(fake_home, '.netrc'), 'wb') as f:
+        os.fchmod(f.fileno(), 0o700)
+        f.write(b'machine 127.0.0.1\n')
+        f.write(b'login testuser\n')
+        f.write(b'password 12345\n')
+
+    with create_file_server(server_type) as server:
+        server.add_user('testuser', '12345', file_server_files)
+        generate_project_file_server(server, project)
+
+        src_zip = os.path.join(file_server_files, 'a.zip')
+        _assemble_zip(os.path.join(str(datafiles), 'content'), src_zip)
+
+        server.start()
+
+        result = cli.run(project=project, args=['track', 'target.bst'])
+        result.assert_success()
+        result = cli.run(project=project, args=['fetch', 'target.bst'])
+        result.assert_success()
+        result = cli.run(project=project, args=['build', 'target.bst'])
+        result.assert_success()
+        result = cli.run(project=project, args=['checkout', 'target.bst', checkoutdir])
+        result.assert_success()
+
+        original_dir = os.path.join(str(datafiles), 'content', 'a')
+        original_contents = list_dir_contents(original_dir)
+        checkout_contents = list_dir_contents(checkoutdir)
+        assert(checkout_contents == original_contents)
diff --git a/tests/testutils/file_server.py b/tests/testutils/file_server.py
new file mode 100644
index 000000000..05f896013
--- /dev/null
+++ b/tests/testutils/file_server.py
@@ -0,0 +1,19 @@
+from contextlib import contextmanager
+
+from .ftp_server import SimpleFtpServer
+from .http_server import SimpleHttpServer
+
+
+@contextmanager
+def create_file_server(file_server_type):
+    if file_server_type == 'FTP':
+        server = SimpleFtpServer()
+    elif file_server_type == 'HTTP':
+        server = SimpleHttpServer()
+    else:
+        assert False
+
+    try:
+        yield server
+    finally:
+        server.stop()
diff --git a/tests/testutils/ftp_server.py b/tests/testutils/ftp_server.py
new file mode 100644
index 000000000..52c05f8ba
--- /dev/null
+++ b/tests/testutils/ftp_server.py
@@ -0,0 +1,32 @@
+import multiprocessing
+
+from pyftpdlib.authorizers import DummyAuthorizer
+from pyftpdlib.handlers import FTPHandler
+from pyftpdlib.servers import FTPServer
+
+
+class SimpleFtpServer(multiprocessing.Process):
+    def __init__(self):
+        super().__init__()
+        self.authorizer = DummyAuthorizer()
+        handler = FTPHandler
+        handler.authorizer = self.authorizer
+        self.server = FTPServer(('127.0.0.1', 0), handler)
+
+    def run(self):
+        self.server.serve_forever()
+
+    def stop(self):
+        self.server.close_all()
+        self.server.close()
+        self.terminate()
+        self.join()
+
+    def allow_anonymous(self, cwd):
+        self.authorizer.add_anonymous(cwd)
+
+    def add_user(self, user, password, cwd):
+        self.authorizer.add_user(user, password, cwd, perm='elradfmwMT')
+
+    def base_url(self):
+        return 'ftp://127.0.0.1:{}'.format(self.server.address[1])
diff --git a/tests/testutils/http_server.py b/tests/testutils/http_server.py
new file mode 100644
index 000000000..129003836
--- /dev/null
+++ b/tests/testutils/http_server.py
@@ -0,0 +1,117 @@
+import multiprocessing
+import os
+import posixpath
+import html
+import threading
+import base64
+from http.server import SimpleHTTPRequestHandler, HTTPServer, HTTPStatus
+
+
+class Unauthorized(Exception):
+    pass
+
+
+class RequestHandler(SimpleHTTPRequestHandler):
+
+    def get_root_dir(self):
+        authorization = self.headers.get('authorization')
+        if not authorization:
+            if not self.server.anonymous_dir:
+                raise Unauthorized('unauthorized')
+            return self.server.anonymous_dir
+        else:
+            authorization = authorization.split()
+            if len(authorization) != 2 or authorization[0].lower() != 'basic':
+                raise Unauthorized('unauthorized')
+            try:
+                decoded = base64.decodebytes(authorization[1].encode('ascii'))
+                user, password = decoded.decode('ascii').split(':')
+                expected_password, directory = self.server.users[user]
+                if password == expected_password:
+                    return directory
+            except:
+                raise Unauthorized('unauthorized')
+            return None
+
+    def unauthorized(self):
+        shortmsg, longmsg = self.responses[HTTPStatus.UNAUTHORIZED]
+        self.send_response(HTTPStatus.UNAUTHORIZED, shortmsg)
+        self.send_header('Connection', 'close')
+
+        content = (self.error_message_format % {
+            'code': HTTPStatus.UNAUTHORIZED,
+            'message': html.escape(longmsg, quote=False),
+            'explain': html.escape(longmsg, quote=False)
+        })
+        body = content.encode('UTF-8', 'replace')
+        self.send_header('Content-Type', self.error_content_type)
+        self.send_header('Content-Length', str(len(body)))
+        self.send_header('WWW-Authenticate', 'Basic realm="{}"'.format(self.server.realm))
+        self.end_headers()
+        self.end_headers()
+
+        if self.command != 'HEAD' and body:
+            self.wfile.write(body)
+
+    def do_GET(self):
+        try:
+            super().do_GET()
+        except Unauthorized:
+            self.unauthorized()
+
+    def do_HEAD(self):
+        try:
+            super().do_HEAD()
+        except Unauthorized:
+            self.unauthorized()
+
+    def translate_path(self, path):
+        path = path.split('?', 1)[0]
+        path = path.split('#', 1)[0]
+        path = posixpath.normpath(path)
+        assert(posixpath.isabs(path))
+        path = posixpath.relpath(path, '/')
+        return os.path.join(self.get_root_dir(), path)
+
+
+class AuthHTTPServer(HTTPServer):
+    def __init__(self, *args, **kwargs):
+        self.users = {}
+        self.anonymous_dir = None
+        self.realm = 'Realm'
+        super().__init__(*args, **kwargs)
+
+
+class SimpleHttpServer(multiprocessing.Process):
+    def __init__(self):
+        self.__stop = multiprocessing.Queue()
+        super().__init__()
+        self.server = AuthHTTPServer(('127.0.0.1', 0), RequestHandler)
+        self.started = False
+
+    def start(self):
+        self.started = True
+        super().start()
+
+    def run(self):
+        t = threading.Thread(target=self.server.serve_forever)
+        t.start()
+        self.__stop.get()
+        self.server.shutdown()
+        t.join()
+
+    def stop(self):
+        if not self.started:
+            return
+        self.__stop.put(None)
+        self.terminate()
+        self.join()
+
+    def allow_anonymous(self, cwd):
+        self.server.anonymous_dir = cwd
+
+    def add_user(self, user, password, cwd):
+        self.server.users[user] = (password, cwd)
+
+    def base_url(self):
+        return 'http://127.0.0.1:{}'.format(self.server.server_port)