Fix descend can not follow symlinks

4 files changed, 194 insertions, 7 deletions

diff --git a/src/buildstream/storage/_casbaseddirectory.py b/src/buildstream/storage/_casbaseddirectory.py
index 7bd9ceea0..424b7ef63 100644
--- a/src/buildstream/storage/_casbaseddirectory.py
+++ b/src/buildstream/storage/_casbaseddirectory.py
@@ -159,7 +159,15 @@ class CasBasedDirectory(Directory):
 
         self.__invalidate_digest()
 
-    def descend(self, *paths, create=False):
+    def find_root(self):
+        """ Finds the root of this directory tree by following 'parent' until there is
+        no parent. """
+        if self.parent:
+            return self.parent.find_root()
+        else:
+            return self
+
+    def descend(self, *paths, create=False, follow_symlinks=False):
         """Descend one or more levels of directory hierarchy and return a new
         Directory object for that directory.
 
@@ -177,6 +185,7 @@ class CasBasedDirectory(Directory):
         """
 
         current_dir = self
+        paths = list(paths)
 
         for path in paths:
             # Skip empty path segments
@@ -184,20 +193,37 @@ class CasBasedDirectory(Directory):
                 continue
 
             entry = current_dir.index.get(path)
+
             if entry:
                 if entry.type == _FileType.DIRECTORY:
                     current_dir = entry.get_directory(current_dir)
+                elif follow_symlinks and entry.type == _FileType.SYMLINK:
+                    linklocation = entry.target
+                    newpaths = linklocation.split(os.path.sep)
+                    if os.path.isabs(linklocation):
+                        current_dir = current_dir.find_root().descend(*newpaths, follow_symlinks=True)
+                    else:
+                        current_dir = current_dir.descend(*newpaths, follow_symlinks=True)
                 else:
                     error = "Cannot descend into {}, which is a '{}' in the directory {}"
                     raise VirtualDirectoryError(error.format(path,
                                                              current_dir.index[path].type,
-                                                             current_dir))
+                                                             current_dir),
+                                                reason="not-a-directory")
             else:
-                if create:
+                if path == '.':
+                    continue
+                elif path == '..':
+                    if current_dir.parent is not None:
+                        current_dir = current_dir.parent
+                    # In POSIX /.. == / so just stay at the root dir
+                    continue
+                elif create:
                     current_dir = current_dir._add_directory(path)
                 else:
                     error = "'{}' not found in {}"
-                    raise VirtualDirectoryError(error.format(path, str(current_dir)))
+                    raise VirtualDirectoryError(error.format(path, str(current_dir)),
+                                                reason="directory-not-found")
 
         return current_dir
 
@@ -618,6 +644,23 @@ class CasBasedDirectory(Directory):
 
         return self.__digest
 
+    def _exists(self, *path, follow_symlinks=False):
+        try:
+            subdir = self.descend(*path[:-1], follow_symlinks=follow_symlinks)
+            target = subdir.index.get(path[-1])
+            if target is not None:
+                if target.type == _FileType.REGULAR_FILE:
+                    return True
+                elif follow_symlinks and target.type == _FileType.SYMLINK:
+                    linklocation = target.target
+                    newpath = linklocation.split(os.path.sep)
+                    if os.path.isabs(linklocation):
+                        return subdir.find_root()._exists(*newpath, follow_symlinks=True)
+                    return subdir._exists(*newpath, follow_symlinks=True)
+            return False
+        except VirtualDirectoryError:
+            return False
+
     def __invalidate_digest(self):
         if self.__digest:
             self.__digest = None
diff --git a/src/buildstream/storage/_filebaseddirectory.py b/src/buildstream/storage/_filebaseddirectory.py
index 8c55819c9..a083f6507 100644
--- a/src/buildstream/storage/_filebaseddirectory.py
+++ b/src/buildstream/storage/_filebaseddirectory.py
@@ -37,6 +37,7 @@ from .. import utils
 from ..utils import link_files, copy_files, list_relative_paths, _get_link_mtime, BST_ARBITRARY_TIMESTAMP
 from ..utils import _set_deterministic_user, _set_deterministic_mtime
 from ..utils import FileListResult
+from .._exceptions import ImplError
 
 # FileBasedDirectory intentionally doesn't call its superclass constuctor,
 # which is meant to be unimplemented.
@@ -47,9 +48,12 @@ class FileBasedDirectory(Directory):
     def __init__(self, external_directory=None):
         self.external_directory = external_directory
 
-    def descend(self, *paths, create=False):
+    def descend(self, *paths, create=False, follow_symlinks=False):
         """ See superclass Directory for arguments """
 
+        if follow_symlinks:
+            ImplError("FileBasedDirectory.Decend dose not implement follow_symlinks=True")
+
         current_dir = self
 
         for path in paths:
diff --git a/src/buildstream/storage/directory.py b/src/buildstream/storage/directory.py
index c9906b058..d32ac0063 100644
--- a/src/buildstream/storage/directory.py
+++ b/src/buildstream/storage/directory.py
@@ -52,7 +52,7 @@ class Directory():
     def __init__(self, external_directory=None):
         raise NotImplementedError()
 
-    def descend(self, *paths, create=False):
+    def descend(self, *paths, create=False, follow_symlinks=False):
         """Descend one or more levels of directory hierarchy and return a new
         Directory object for that directory.
 
diff --git a/tests/internals/storage_vdir_import.py b/tests/internals/storage_vdir_import.py
index 9d42c6e8d..7c6cbe4fb 100644
--- a/tests/internals/storage_vdir_import.py
+++ b/tests/internals/storage_vdir_import.py
@@ -1,3 +1,18 @@
+#
+#  Copyright (C) 2019 Bloomberg LP
+#
+#  This program is free software; you can redistribute it and/or
+#  modify it under the terms of the GNU Lesser General Public
+#  License as published by the Free Software Foundation; either
+#  version 2 of the License, or (at your option) any later version.
+#
+#  This library is distributed in the hope that it will be useful,
+#  but WITHOUT ANY WARRANTY; without even the implied warranty of
+#  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.	 See the GNU
+#  Lesser General Public License for more details.
+#
+#  You should have received a copy of the GNU Lesser General Public
+#  License along with this library. If not, see <http://www.gnu.org/licenses/>.
 from hashlib import sha256
 import os
 import random
@@ -7,6 +22,7 @@ import pytest
 from buildstream.storage._casbaseddirectory import CasBasedDirectory
 from buildstream.storage._filebaseddirectory import FileBasedDirectory
 from buildstream._cas import CASCache
+from buildstream.storage.directory import VirtualDirectoryError
 
 
 # These are comparitive tests that check that FileBasedDirectory and
@@ -42,9 +58,13 @@ NUM_RANDOM_TESTS = 10
 def generate_import_roots(rootno, directory):
     rootname = "root{}".format(rootno)
     rootdir = os.path.join(directory, "content", rootname)
+    generate_import_root(rootdir, root_filesets[rootno - 1])
+
+
+def generate_import_root(rootdir, filelist):
     if os.path.exists(rootdir):
         return
-    for (path, typesymbol, content) in root_filesets[rootno - 1]:
+    for (path, typesymbol, content) in filelist:
         if typesymbol == 'F':
             (dirnames, filename) = os.path.split(path)
             os.makedirs(os.path.join(rootdir, dirnames), exist_ok=True)
@@ -251,3 +271,123 @@ def test_random_directory_listing(tmpdir, root):
 @pytest.mark.parametrize("root", [1, 2, 3, 4, 5])
 def test_fixed_directory_listing(tmpdir, root):
     _listing_test(str(tmpdir), root, generate_import_roots)
+
+
+# Check that the vdir is decending and readable
+def test_descend(tmpdir):
+    cas_dir = os.path.join(str(tmpdir), 'cas')
+    cas_cache = CASCache(cas_dir)
+    d = CasBasedDirectory(cas_cache)
+
+    Content_to_check = 'You got me'
+    test_dir = os.path.join(str(tmpdir), 'importfrom')
+    filesys_discription = [
+        ('a', 'D', ''),
+        ('a/l', 'D', ''),
+        ('a/l/g', 'F', Content_to_check)
+    ]
+    generate_import_root(test_dir, filesys_discription)
+
+    d.import_files(test_dir)
+    digest = d.descend('a', 'l').index['g'].get_digest()
+
+    assert Content_to_check == open(cas_cache.objpath(digest)).read()
+
+
+# Check symlink logic for edgecases
+# Make sure the correct erros are raised when trying
+# to decend in to files or links to files
+def test_bad_symlinks(tmpdir):
+    cas_dir = os.path.join(str(tmpdir), 'cas')
+    cas_cache = CASCache(cas_dir)
+    d = CasBasedDirectory(cas_cache)
+
+    test_dir = os.path.join(str(tmpdir), 'importfrom')
+    filesys_discription = [
+        ('a', 'D', ''),
+        ('a/l', 'S', '../target'),
+        ('target', 'F', 'You got me')
+    ]
+    generate_import_root(test_dir, filesys_discription)
+    d.import_files(test_dir)
+    exp_reason = "not-a-directory"
+
+    with pytest.raises(VirtualDirectoryError) as error:
+        d.descend('a', 'l', follow_symlinks=True)
+        assert error.reason == exp_reason
+
+    with pytest.raises(VirtualDirectoryError) as error:
+        d.descend('a', 'l')
+        assert error.reason == exp_reason
+
+    with pytest.raises(VirtualDirectoryError) as error:
+        d.descend('a', 'f')
+        assert error.reason == exp_reason
+
+
+# Check symlink logic for edgecases
+# Check decend accross relitive link
+def test_relitive_symlink(tmpdir):
+    cas_dir = os.path.join(str(tmpdir), 'cas')
+    cas_cache = CASCache(cas_dir)
+    d = CasBasedDirectory(cas_cache)
+
+    Content_to_check = 'You got me'
+    test_dir = os.path.join(str(tmpdir), 'importfrom')
+    filesys_discription = [
+        ('a', 'D', ''),
+        ('a/l', 'S', '../target'),
+        ('target', 'D', ''),
+        ('target/file', 'F', Content_to_check)
+    ]
+    generate_import_root(test_dir, filesys_discription)
+    d.import_files(test_dir)
+
+    digest = d.descend('a', 'l', follow_symlinks=True).index['file'].get_digest()
+    assert Content_to_check == open(cas_cache.objpath(digest)).read()
+
+
+# Check symlink logic for edgecases
+# Check deccend accross abs link
+def test_abs_symlink(tmpdir):
+    cas_dir = os.path.join(str(tmpdir), 'cas')
+    cas_cache = CASCache(cas_dir)
+    d = CasBasedDirectory(cas_cache)
+
+    Content_to_check = 'two step file'
+    test_dir = os.path.join(str(tmpdir), 'importfrom')
+    filesys_discription = [
+        ('a', 'D', ''),
+        ('a/l', 'S', '/target'),
+        ('target', 'D', ''),
+        ('target/file', 'F', Content_to_check)
+    ]
+    generate_import_root(test_dir, filesys_discription)
+    d.import_files(test_dir)
+
+    digest = d.descend('a', 'l', follow_symlinks=True).index['file'].get_digest()
+
+    assert Content_to_check == open(cas_cache.objpath(digest)).read()
+
+
+# Check symlink logic for edgecases
+# Check symlink can not escape root
+def test_bad_sym_escape(tmpdir):
+    cas_dir = os.path.join(str(tmpdir), 'cas')
+    cas_cache = CASCache(cas_dir)
+    d = CasBasedDirectory(cas_cache)
+
+    test_dir = os.path.join(str(tmpdir), 'importfrom')
+    filesys_discription = [
+        ('jail', 'D', ''),
+        ('jail/a', 'D', ''),
+        ('jail/a/l', 'S', '../../target'),
+        ('target', 'D', ''),
+        ('target/file', 'F', 'two step file')
+    ]
+    generate_import_root(test_dir, filesys_discription)
+    d.import_files(os.path.join(test_dir, 'jail'))
+
+    with pytest.raises(VirtualDirectoryError) as error:
+        d.descend('a', 'l', follow_symlinks=True)
+        assert error.reason == "directory-not-found"