Merge branch 'juerg/symlinks2' into 'master'

Do not resolve or mangle symlinks during staging

See merge request BuildStream/buildstream!1140

36 files changed, 86 insertions, 333 deletions

diff --git a/NEWS b/NEWS
index 9876210c3..752434cfe 100644
--- a/NEWS
+++ b/NEWS
@@ -135,6 +135,10 @@ buildstream 1.3.1
     the cached build tree input it will remain unaltered, however the availbility of the
     build tree content may differ.
 
+  o BREAKING CHANGE: Symlinks are no longer resolved during staging and absolute
+    symlinks are now preserved instead of being converted to relative symlinks.
+
+
 =================
 buildstream 1.1.5
 =================
diff --git a/buildstream/_versions.py b/buildstream/_versions.py
index f24a74714..a9eb86d69 100644
--- a/buildstream/_versions.py
+++ b/buildstream/_versions.py
@@ -33,4 +33,4 @@ BST_FORMAT_VERSION = 21
 # or if buildstream was changed in a way which can cause
 # the same cache key to produce something that is no longer
 # the same.
-BST_CORE_ARTIFACT_VERSION = 7
+BST_CORE_ARTIFACT_VERSION = 8
diff --git a/buildstream/storage/_casbaseddirectory.py b/buildstream/storage/_casbaseddirectory.py
index b6a39b83d..b8a247134 100644
--- a/buildstream/storage/_casbaseddirectory.py
+++ b/buildstream/storage/_casbaseddirectory.py
@@ -79,151 +79,6 @@ class UnexpectedFileException(ResolutionException):
         super().__init__(message)
 
 
-class _Resolver():
-    """A class for resolving symlinks inside CAS-based directories. As
-    well as providing a namespace for some functions, this also
-    contains two flags which are constant throughout one resolution
-    operation and the 'seen_objects' list used to detect infinite
-    symlink loops.
-
-    """
-
-    def __init__(self, absolute_symlinks_resolve=True, force_create=False):
-        self.absolute_symlinks_resolve = absolute_symlinks_resolve
-        self.force_create = force_create
-        self.seen_objects = []
-
-    def resolve(self, name, directory):
-        """Resolves any name to an object. If the name points to a symlink in
-        the directory, it returns the thing it points to,
-        recursively.
-
-        Returns a CasBasedDirectory, FileNode or None. None indicates
-        either that 'target' does not exist in this directory, or is a
-        symlink chain which points to a nonexistent name (broken
-        symlink).
-
-        Raises:
-
-        - InfiniteSymlinkException if 'name' points to an infinite
-          symlink loop.
-        - AbsoluteSymlinkException if 'name' points to an absolute
-          symlink and absolute_symlinks_resolve is False.
-        - UnexpectedFileException if at any point during resolution we
-          find a file which we expected to be a directory or symlink.
-
-        If force_create is set, this will attempt to create
-        directories to make symlinks and directories resolve.  Files
-        present in symlink target paths will also be removed and
-        replaced with directories.  If force_create is off, this will
-        never alter 'directory'.
-
-        """
-
-        # First check for nonexistent things or 'normal' objects and return them
-        if name not in directory.index:
-            return None
-        index_entry = directory.index[name]
-        if isinstance(index_entry.buildstream_object, Directory):
-            return index_entry.buildstream_object
-        elif isinstance(index_entry.pb_object, remote_execution_pb2.FileNode):
-            return index_entry.pb_object
-
-        # Now we must be dealing with a symlink.
-        assert isinstance(index_entry.pb_object, remote_execution_pb2.SymlinkNode)
-
-        symlink_object = index_entry.pb_object
-        if symlink_object in self.seen_objects:
-            # Infinite symlink loop detected
-            message = ("Infinite symlink loop found during resolution. " +
-                       "First repeated element is {}".format(name))
-            raise InfiniteSymlinkException(message=message)
-
-        self.seen_objects.append(symlink_object)
-
-        components = symlink_object.target.split(CasBasedDirectory._pb2_path_sep)
-        absolute = symlink_object.target.startswith(CasBasedDirectory._pb2_absolute_path_prefix)
-
-        if absolute:
-            if self.absolute_symlinks_resolve:
-                directory = directory.find_root()
-                # Discard the first empty element
-                components.pop(0)
-            else:
-                # Unresolvable absolute symlink
-                message = "{} is an absolute symlink, which was disallowed during resolution".format(name)
-                raise AbsoluteSymlinkException(message=message)
-
-        resolution = directory
-        while components and isinstance(resolution, CasBasedDirectory):
-            c = components.pop(0)
-            directory = resolution
-
-            try:
-                resolution = self._resolve_path_component(c, directory, components)
-            except UnexpectedFileException as original:
-                errormsg = ("Reached a file called {} while trying to resolve a symlink; " +
-                            "cannot proceed. The remaining path components are {}.")
-                raise UnexpectedFileException(errormsg.format(c, components)) from original
-
-        return resolution
-
-    def _resolve_path_component(self, c, directory, components_remaining):
-        if c == ".":
-            resolution = directory
-        elif c == "..":
-            if directory.parent is not None:
-                resolution = directory.parent
-            else:
-                # If directory.parent *is* None, this is an attempt to
-                # access '..' from the root, which is valid under
-                # POSIX; it just returns the root.
-                resolution = directory
-        elif c in directory.index:
-            try:
-                resolution = self._resolve_through_files(c, directory, components_remaining)
-            except UnexpectedFileException as original:
-                errormsg = ("Reached a file called {} while trying to resolve a symlink; " +
-                            "cannot proceed. The remaining path components are {}.")
-                raise UnexpectedFileException(errormsg.format(c, components_remaining)) from original
-        else:
-            # c is not in our index
-            if self.force_create:
-                resolution = directory.descend(c, create=True)
-            else:
-                resolution = None
-        return resolution
-
-    def _resolve_through_files(self, c, directory, require_traversable):
-        """A wrapper to resolve() which deals with files being found
-        in the middle of paths, for example trying to resolve a symlink
-        which points to /usr/lib64/libfoo when 'lib64' is a file.
-
-        require_traversable: If this is True, never return a file
-        node.  Instead, if force_create is set, destroy the file node,
-        then create and return a normal directory in its place. If
-        force_create is off, throws ResolutionException.
-
-        """
-        resolved_thing = self.resolve(c, directory)
-
-        if isinstance(resolved_thing, remote_execution_pb2.FileNode):
-            if require_traversable:
-                # We have components still to resolve, but one of the path components
-                # is a file.
-                if self.force_create:
-                    directory.delete_entry(c)
-                    resolved_thing = directory.descend(c, create=True)
-                else:
-                    # This is a signal that we hit a file, but don't
-                    # have the data to give a proper message, so the
-                    # caller should reraise this with a proper
-                    # description.
-                    raise UnexpectedFileException()
-
-        return resolved_thing
-
-
 # CasBasedDirectory intentionally doesn't call its superclass constuctor,
 # which is meant to be unimplemented.
 # pylint: disable=super-init-not-called
@@ -407,10 +262,6 @@ class CasBasedDirectory(Directory):
             if isinstance(entry, CasBasedDirectory):
                 return entry.descend(subdirectory_spec[1:], create)
             else:
-                # May be a symlink
-                target = self._resolve(subdirectory_spec[0], force_create=create)
-                if isinstance(target, CasBasedDirectory):
-                    return target
                 error = "Cannot descend into {}, which is a '{}' in the directory {}"
                 raise VirtualDirectoryError(error.format(subdirectory_spec[0],
                                                          type(self.index[subdirectory_spec[0]].pb_object).__name__,
@@ -434,10 +285,6 @@ class CasBasedDirectory(Directory):
         else:
             return self
 
-    def _resolve(self, name, absolute_symlinks_resolve=True, force_create=False):
-        resolver = _Resolver(absolute_symlinks_resolve, force_create)
-        return resolver.resolve(name, self)
-
     def _check_replacement(self, name, path_prefix, fileListResult):
         """ Checks whether 'name' exists, and if so, whether we can overwrite it.
         If we can, add the name to 'overwritten_files' and delete the existing entry.
@@ -468,36 +315,13 @@ class CasBasedDirectory(Directory):
                        .format(name, type(existing_entry)))
         return False  # In case asserts are disabled
 
-    def _replace_anything_with_dir(self, name, path_prefix, overwritten_files_list):
-        self.delete_entry(name)
-        subdir = self._add_directory(name)
-        overwritten_files_list.append(os.path.join(path_prefix, name))
-        return subdir
-
     def _import_files_from_directory(self, source_directory, files, path_prefix=""):
         """ Imports files from a traditional directory. """
 
-        def _ensure_followable(name, path_prefix):
-            """ Makes sure 'name' is a directory or symlink to a directory which can be descended into. """
-            if isinstance(self.index[name].buildstream_object, Directory):
-                return self.descend(name)
-            try:
-                target = self._resolve(name, force_create=True)
-            except InfiniteSymlinkException:
-                return self._replace_anything_with_dir(name, path_prefix, result.overwritten)
-            if isinstance(target, CasBasedDirectory):
-                return target
-            elif isinstance(target, remote_execution_pb2.FileNode):
-                return self._replace_anything_with_dir(name, path_prefix, result.overwritten)
-            return target
-
         def _import_directory_recursively(directory_name, source_directory, remaining_path, path_prefix):
             """ _import_directory_recursively and _import_files_from_directory will be called alternately
             as a directory tree is descended. """
-            if directory_name in self.index:
-                subdir = _ensure_followable(directory_name, path_prefix)
-            else:
-                subdir = self._add_directory(directory_name)
+            subdir = self.descend(directory_name, create=True)
             new_path_prefix = os.path.join(path_prefix, directory_name)
             subdir_result = subdir._import_files_from_directory(os.path.join(source_directory, directory_name),
                                                                 [os.path.sep.join(remaining_path)],
@@ -566,15 +390,8 @@ class CasBasedDirectory(Directory):
                 if dirname not in processed_directories:
                     # Now strip off the first directory name and import files recursively.
                     subcomponents = CasBasedDirectory._files_in_subdir(files, dirname)
-                    # We will fail at this point if there is a file or symlink to file called 'dirname'.
-                    if dirname in self.index:
-                        resolved_component = self._resolve(dirname, force_create=True)
-                        if isinstance(resolved_component, remote_execution_pb2.FileNode):
-                            dest_subdir = self._replace_anything_with_dir(dirname, path_prefix, result.overwritten)
-                        else:
-                            dest_subdir = resolved_component
-                    else:
-                        dest_subdir = self.descend(dirname, create=True)
+                    # We will fail at this point if there is a file or symlink called 'dirname'.
+                    dest_subdir = self.descend(dirname, create=True)
                     src_subdir = source_directory.descend(dirname)
                     import_result = dest_subdir._partial_import_cas_into_cas(src_subdir, subcomponents,
                                                                              path_prefix=fullname,
diff --git a/buildstream/utils.py b/buildstream/utils.py
index 12407ba30..a4e84097b 100644
--- a/buildstream/utils.py
+++ b/buildstream/utils.py
@@ -34,7 +34,6 @@ import string
 import subprocess
 import tempfile
 import itertools
-import functools
 from contextlib import contextmanager
 
 import psutil
@@ -767,37 +766,31 @@ def _copy_directories(srcdir, destdir, target):
                                 'directory expected: {}'.format(old_dir))
 
 
-@functools.lru_cache(maxsize=64)
-def _resolve_symlinks(path):
-    return os.path.realpath(path)
-
-
-def _ensure_real_directory(root, destpath):
-    # The realpath in the sandbox may refer to a file outside of the
-    # sandbox when any of the direcory branches are a symlink to an
-    # absolute path.
-    #
-    # This should not happen as we rely on relative_symlink_target() below
-    # when staging the actual symlinks which may lead up to this path.
-    #
-    destpath_resolved = _resolve_symlinks(destpath)
-    if not destpath_resolved.startswith(_resolve_symlinks(root)):
-        raise UtilError('Destination path resolves to a path outside ' +
-                        'of the staging area\n\n' +
-                        '  Destination path: {}\n'.format(destpath) +
-                        '  Real path: {}'.format(destpath_resolved))
-
-    # Ensure the real destination path exists before trying to get the mode
-    # of the real destination path.
-    #
-    # It is acceptable that chunks create symlinks inside artifacts which
-    # refer to non-existing directories, they will be created on demand here
-    # at staging time.
-    #
-    if not os.path.exists(destpath_resolved):
-        os.makedirs(destpath_resolved)
-
-    return destpath_resolved
+# _ensure_real_directory()
+#
+# Ensure `path` is a real directory and there are no symlink components.
+#
+# Symlink components are allowed in `root`.
+#
+def _ensure_real_directory(root, path):
+    destpath = root
+    for name in os.path.split(path):
+        destpath = os.path.join(destpath, name)
+        try:
+            deststat = os.lstat(destpath)
+            if not stat.S_ISDIR(deststat.st_mode):
+                relpath = destpath[len(root):]
+
+                if stat.S_ISLNK(deststat.st_mode):
+                    filetype = 'symlink'
+                elif stat.S_ISREG(deststat.st_mode):
+                    filetype = 'regular file'
+                else:
+                    filetype = 'special file'
+
+                raise UtilError('Destination is a {}, not a directory: {}'.format(filetype, relpath))
+        except FileNotFoundError:
+            os.makedirs(destpath)
 
 
 # _process_list()
@@ -836,6 +829,10 @@ def _process_list(srcdir, destdir, filelist, actionfunc, result,
         srcpath = os.path.join(srcdir, path)
         destpath = os.path.join(destdir, path)
 
+        # Ensure that the parent of the destination path exists without symlink
+        # components.
+        _ensure_real_directory(destdir, os.path.dirname(path))
+
         # Add to the results the list of files written
         if report_written:
             result.files_written.append(path)
@@ -847,11 +844,6 @@ def _process_list(srcdir, destdir, filelist, actionfunc, result,
         # The destination directory may not have been created separately
         permissions.extend(_copy_directories(srcdir, destdir, path))
 
-        # Ensure that broken symlinks to directories have their targets
-        # created before attempting to stage files across broken
-        # symlink boundaries
-        _ensure_real_directory(destdir, os.path.dirname(destpath))
-
         try:
             file_stat = os.lstat(srcpath)
             mode = file_stat.st_mode
@@ -865,13 +857,7 @@ def _process_list(srcdir, destdir, filelist, actionfunc, result,
 
         if stat.S_ISDIR(mode):
             # Ensure directory exists in destination
-            if not os.path.exists(destpath):
-                _ensure_real_directory(destdir, destpath)
-
-            dest_stat = os.lstat(_resolve_symlinks(destpath))
-            if not stat.S_ISDIR(dest_stat.st_mode):
-                raise UtilError('Destination not a directory. source has {}'
-                                ' destination has {}'.format(srcpath, destpath))
+            _ensure_real_directory(destdir, path)
             permissions.append((destpath, os.stat(srcpath).st_mode))
 
         elif stat.S_ISLNK(mode):
@@ -880,7 +866,6 @@ def _process_list(srcdir, destdir, filelist, actionfunc, result,
                 continue
 
             target = os.readlink(srcpath)
-            target = _relative_symlink_target(destdir, destpath, target)
             os.symlink(target, destpath)
 
         elif stat.S_ISREG(mode):
@@ -918,51 +903,6 @@ def _process_list(srcdir, destdir, filelist, actionfunc, result,
         os.chmod(d, perms)
 
 
-# _relative_symlink_target()
-#
-# Fetches a relative path for symlink with an absolute target
-#
-# @root:    The staging area root location
-# @symlink: Location of the symlink in staging area (including the root path)
-# @target:  The symbolic link target, which may be an absolute path
-#
-# If @target is an absolute path, a relative path from the symbolic link
-# location will be returned, otherwise if @target is a relative path, it will
-# be returned unchanged.
-#
-# Using relative symlinks helps to keep the target self contained when staging
-# files onto the target.
-#
-def _relative_symlink_target(root, symlink, target):
-
-    if os.path.isabs(target):
-        # First fix the input a little, the symlink itself must not have a
-        # trailing slash, otherwise we fail to remove the symlink filename
-        # from its directory components in os.path.split()
-        #
-        # The absolute target filename must have its leading separator
-        # removed, otherwise os.path.join() will discard the prefix
-        symlink = symlink.rstrip(os.path.sep)
-        target = target.lstrip(os.path.sep)
-
-        # We want a relative path from the directory in which symlink
-        # is located, not from the symlink itself.
-        symlinkdir, _ = os.path.split(_resolve_symlinks(symlink))
-
-        # Create a full path to the target, including the leading staging
-        # directory
-        fulltarget = os.path.join(_resolve_symlinks(root), target)
-
-        # now get the relative path from the directory where the symlink
-        # is located within the staging root, to the target within the same
-        # staging root
-        newtarget = os.path.relpath(fulltarget, symlinkdir)
-
-        return newtarget
-    else:
-        return target
-
-
 # _set_deterministic_user()
 #
 # Set the uid/gid for every file in a directory tree to the process'
diff --git a/tests/cachekey/project/elements/build1.expected b/tests/cachekey/project/elements/build1.expected
index 901f6acf7..322fc7d31 100644
--- a/tests/cachekey/project/elements/build1.expected
+++ b/tests/cachekey/project/elements/build1.expected
@@ -1 +1 @@
-0eddcb5c12b2bdb4ce83bef7b997f25c89a9396c6321fe471b31dabba8eba928
\ No newline at end of file
+17ba3f62b31a5cb91815cb13b86f6ad3b486d6851ee0cc29ae3300ecdee21b86
\ No newline at end of file
diff --git a/tests/cachekey/project/elements/build2.expected b/tests/cachekey/project/elements/build2.expected
index cfff92ad0..bb549fad4 100644
--- a/tests/cachekey/project/elements/build2.expected
+++ b/tests/cachekey/project/elements/build2.expected
@@ -1 +1 @@
-d54339437d5495f573a01434c61b8f0496cef0f833566b9c9f0a6cb362075edb
\ No newline at end of file
+38903bbc1fadbc2de16b42219cac113eb865d3f028a5bc3f80b9c45cde504f9c
\ No newline at end of file
diff --git a/tests/cachekey/project/elements/compose1.expected b/tests/cachekey/project/elements/compose1.expected
index d8f256190..c3f5fd484 100644
--- a/tests/cachekey/project/elements/compose1.expected
+++ b/tests/cachekey/project/elements/compose1.expected
@@ -1 +1 @@
-d14a3d59da4fcb18c81f0cd6b5e893dc5f25e1d5eecc1a0845fb9b2ef499d878
\ No newline at end of file
+46ce6f56c7c0746bb9ae4046b6f3127c2e54e02a7f0b1855485a9f9705d018e5
\ No newline at end of file
diff --git a/tests/cachekey/project/elements/compose2.expected b/tests/cachekey/project/elements/compose2.expected
index f7de6d183..be63a2587 100644
--- a/tests/cachekey/project/elements/compose2.expected
+++ b/tests/cachekey/project/elements/compose2.expected
@@ -1 +1 @@
-29857451bb3797e258911e485f94c7d59e13f239f3ac904e7524c4d0a187e240
\ No newline at end of file
+1f8b611725e5dd55824c3370bc89dcf205cd9cb82549a254344018682048b3c5
\ No newline at end of file
diff --git a/tests/cachekey/project/elements/compose3.expected b/tests/cachekey/project/elements/compose3.expected
index 0cf041baf..73652d417 100644
--- a/tests/cachekey/project/elements/compose3.expected
+++ b/tests/cachekey/project/elements/compose3.expected
@@ -1 +1 @@
-2dbb668ebc50445dc03cd44d6d1110cc14d50093a37ef4a25499055318835174
\ No newline at end of file
+09e611ec8fe3b4e560389c2889d84337a1d7b16241f8b70b138b89a290b4a795
\ No newline at end of file
diff --git a/tests/cachekey/project/elements/compose4.expected b/tests/cachekey/project/elements/compose4.expected
index db0879543..69b0b1a06 100644
--- a/tests/cachekey/project/elements/compose4.expected
+++ b/tests/cachekey/project/elements/compose4.expected
@@ -1 +1 @@
-7bc4f89c2a4c27999626c9c8a28fd2865695588375660236e3aec49e847705c1
\ No newline at end of file
+9f62d46fbfedc487f6f14e8c20786c5fb2e45791ebc97a97e8cc5bfaeeae05cf
\ No newline at end of file
diff --git a/tests/cachekey/project/elements/compose5.expected b/tests/cachekey/project/elements/compose5.expected
index 8300b0302..fe0c9a5a4 100644
--- a/tests/cachekey/project/elements/compose5.expected
+++ b/tests/cachekey/project/elements/compose5.expected
@@ -1 +1 @@
-12753c08f60587f22f4ef877b636f6e237f958233ea9b512471de213c84ff927
\ No newline at end of file
+56f2a72a3653270d03406437fa9c1c456c5c2405b93392cc38bcde573f8c893a
\ No newline at end of file
diff --git a/tests/cachekey/project/elements/import1.expected b/tests/cachekey/project/elements/import1.expected
index 32f292fd0..ee01875d7 100644
--- a/tests/cachekey/project/elements/import1.expected
+++ b/tests/cachekey/project/elements/import1.expected
@@ -1 +1 @@
-bbe25b17a5190532ab36663eb08d3f855a45e09108a80ef619a1198e9d730dcf
\ No newline at end of file
+3664f199e099282328691d859fbbfa8710369e694e67a694afd3ea42ffd2a7c5
\ No newline at end of file
diff --git a/tests/cachekey/project/elements/import2.expected b/tests/cachekey/project/elements/import2.expected
index f271519e0..354c2fc7c 100644
--- a/tests/cachekey/project/elements/import2.expected
+++ b/tests/cachekey/project/elements/import2.expected
@@ -1 +1 @@
-532277cf8cf2b5d609bbd690dd3cf2eb2e1042329cdd4b25f7a03f77209bf316
\ No newline at end of file
+8d46e86f80c9ef42b5eee0ffcead38685428e577b26f8a4f0271c21e9980d77b
\ No newline at end of file
diff --git a/tests/cachekey/project/elements/import3.expected b/tests/cachekey/project/elements/import3.expected
index 075d6af90..9675c0456 100644
--- a/tests/cachekey/project/elements/import3.expected
+++ b/tests/cachekey/project/elements/import3.expected
@@ -1 +1 @@
-5b5911dad04b5f592cedd29b46aa55b87428c31b2bf6539f9c2b2e51a932adbe
\ No newline at end of file
+c0ea2f2f372f9e09c1a366ba35786c6f9f1326c1a1e5cb13ceec01b21cdad949
\ No newline at end of file
diff --git a/tests/cachekey/project/elements/script1.expected b/tests/cachekey/project/elements/script1.expected
index 58f79dde5..c6d388a2c 100644
--- a/tests/cachekey/project/elements/script1.expected
+++ b/tests/cachekey/project/elements/script1.expected
@@ -1 +1 @@
-818bda0985ef569204c392de0ae2e7ebefd8fcac94f0c33226270936978d52ee
\ No newline at end of file
+9ec0412f42940f86732a99882913df8a0b38ae5a833c5f47ff70746b285fb5a0
\ No newline at end of file
diff --git a/tests/cachekey/project/sources/bzr1.expected b/tests/cachekey/project/sources/bzr1.expected
index dd527da83..52b8409b5 100644
--- a/tests/cachekey/project/sources/bzr1.expected
+++ b/tests/cachekey/project/sources/bzr1.expected
@@ -1 +1 @@
-767d481ad88648c7a926190ac6697c2d24c563c5133abab9e6ab16c6156941d3
\ No newline at end of file
+a2b2bffb50eee26b1e256d1c3da311ee028f084fa3f56b38900f5ade41c80333
\ No newline at end of file
diff --git a/tests/cachekey/project/sources/git1.expected b/tests/cachekey/project/sources/git1.expected
index 9a6d7c991..4ac2db135 100644
--- a/tests/cachekey/project/sources/git1.expected
+++ b/tests/cachekey/project/sources/git1.expected
@@ -1 +1 @@
-4ccd8fdfcca973bd9079960f260c22df8bef5df7f598009f0ee896d3b5905b6e
\ No newline at end of file
+581e4215e37adf0de20e24fd8267f5685e81c63ec7cbbb17eb9a24749519e992
\ No newline at end of file
diff --git a/tests/cachekey/project/sources/git2.expected b/tests/cachekey/project/sources/git2.expected
index 54cc4bff3..827e63ebd 100644
--- a/tests/cachekey/project/sources/git2.expected
+++ b/tests/cachekey/project/sources/git2.expected
@@ -1 +1 @@
-444e3a9bd1f3cd8805370a9e560b5c3720d58e778dc4ae812d9023d3b06d77fc
\ No newline at end of file
+382bd353e1438d232785fbdaf7253eeac3c619bed3c2267a9f3cd46227ced0a9
\ No newline at end of file
diff --git a/tests/cachekey/project/sources/git3.expected b/tests/cachekey/project/sources/git3.expected
index 2c6477e57..1716a4762 100644
--- a/tests/cachekey/project/sources/git3.expected
+++ b/tests/cachekey/project/sources/git3.expected
@@ -1 +1 @@
-f2c3ab6ee644ba9507f3ddd33880fa78f92c80ff872bc1ad676e0a5c40ff3493
\ No newline at end of file
+a312459ab9a8b8b78c509525e6b6d07d4134021f2c747ddd1c687f0c4d4733f8
\ No newline at end of file
diff --git a/tests/cachekey/project/sources/local1.expected b/tests/cachekey/project/sources/local1.expected
index 32f292fd0..ee01875d7 100644
--- a/tests/cachekey/project/sources/local1.expected
+++ b/tests/cachekey/project/sources/local1.expected
@@ -1 +1 @@
-bbe25b17a5190532ab36663eb08d3f855a45e09108a80ef619a1198e9d730dcf
\ No newline at end of file
+3664f199e099282328691d859fbbfa8710369e694e67a694afd3ea42ffd2a7c5
\ No newline at end of file
diff --git a/tests/cachekey/project/sources/local2.expected b/tests/cachekey/project/sources/local2.expected
index f0e548eef..892763956 100644
--- a/tests/cachekey/project/sources/local2.expected
+++ b/tests/cachekey/project/sources/local2.expected
@@ -1 +1 @@
-36e4110f09d9a4ba06fb5c18c8633a7d9a482da28a51e44873624014a2b87438
\ No newline at end of file
+3f1ad9d1476aedaf64d2fe4daac35f0f8eb815f399a8958cd9799724bce9869f
\ No newline at end of file
diff --git a/tests/cachekey/project/sources/ostree1.expected b/tests/cachekey/project/sources/ostree1.expected
index de7d92294..f86971ec4 100644
--- a/tests/cachekey/project/sources/ostree1.expected
+++ b/tests/cachekey/project/sources/ostree1.expected
@@ -1 +1 @@
-dfb4df92f5afb7fd4856db0d028ece5fc465040045f6043e8081a63683297c8a
\ No newline at end of file
+41b27b94ec76b5df79a441c30889da5b82a0b658a498933a05b6a813d8c0e2b7
\ No newline at end of file
diff --git a/tests/cachekey/project/sources/patch1.expected b/tests/cachekey/project/sources/patch1.expected
index 026dbdf55..7e0d1419a 100644
--- a/tests/cachekey/project/sources/patch1.expected
+++ b/tests/cachekey/project/sources/patch1.expected
@@ -1 +1 @@
-a85bc578b9c12f0b380e064cbc835d675ac1d0d3c74f821d20c7ace2b802351f
\ No newline at end of file
+7762d600e85e947615938bd013a35508dd24c4b3c55dae9669f7b5c95a05c23b
\ No newline at end of file
diff --git a/tests/cachekey/project/sources/patch2.expected b/tests/cachekey/project/sources/patch2.expected
index 9f5b12437..ab55cd4ee 100644
--- a/tests/cachekey/project/sources/patch2.expected
+++ b/tests/cachekey/project/sources/patch2.expected
@@ -1 +1 @@
-ec64492225ef13d90236385b6d3e19aa127b8db1e33837cce180e7dd1657d95c
\ No newline at end of file
+81502440f3205b49e3a05032ddd0207f2e98ac61c4c71fa9c2a0a56552433ad5
\ No newline at end of file
diff --git a/tests/cachekey/project/sources/patch3.expected b/tests/cachekey/project/sources/patch3.expected
index ca1243717..40e262ca3 100644
--- a/tests/cachekey/project/sources/patch3.expected
+++ b/tests/cachekey/project/sources/patch3.expected
@@ -1 +1 @@
-d6eb497e7760976b8e9e41f3f80c0f152361188876662eb703c0ddc4984945a6
\ No newline at end of file
+749c80639435617706eb99f10159071e046d0550c2ded240cd5cbb50988e20f5
\ No newline at end of file
diff --git a/tests/cachekey/project/sources/pip1.expected b/tests/cachekey/project/sources/pip1.expected
index 05b7cda23..fc8269383 100644
--- a/tests/cachekey/project/sources/pip1.expected
+++ b/tests/cachekey/project/sources/pip1.expected
@@ -1 +1 @@
-ff8b62cdb1eb65e84082e7af4c745a42e0d0259c25ab0101f42f812e873a53a8
\ No newline at end of file
+a4757beb4b0292cf8d0e3f260f3f730618730fcd5ee2734b97258394d9393580
\ No newline at end of file
diff --git a/tests/cachekey/project/sources/remote1.expected b/tests/cachekey/project/sources/remote1.expected
index a86a0f419..c252de13a 100644
--- a/tests/cachekey/project/sources/remote1.expected
+++ b/tests/cachekey/project/sources/remote1.expected
@@ -1 +1 @@
-009d779779361f1546d808945aaa05765e7d17b6b8b7a383d658ceaeddcff0db
\ No newline at end of file
+f0cdf59cf7e89f3aab80998cbaa1a92229e68e7a4b3180eae4708dae9f854bc6
\ No newline at end of file
diff --git a/tests/cachekey/project/sources/remote2.expected b/tests/cachekey/project/sources/remote2.expected
index 43b5ce0c8..65b9cb62f 100644
--- a/tests/cachekey/project/sources/remote2.expected
+++ b/tests/cachekey/project/sources/remote2.expected
@@ -1 +1 @@
-d4f791a9185a4f2c18c3046c41efb26c0a833b73fc52abb3cddd4fe3ac130196
\ No newline at end of file
+2e392069425a7c59e130af549e869c2948a56b15c81638690f2cc2125312b552
\ No newline at end of file
diff --git a/tests/cachekey/project/sources/tar1.expected b/tests/cachekey/project/sources/tar1.expected
index 8fcec06f6..f14e0d604 100644
--- a/tests/cachekey/project/sources/tar1.expected
+++ b/tests/cachekey/project/sources/tar1.expected
@@ -1 +1 @@
-81542513d239d19a2b1ba826fb1c650cbc106e9457cbadb0ae49c9af07aeb263
\ No newline at end of file
+6b44f68ec188316e563c4a7295301ad0dc70c93e1e6babde1a4f945991ca5dd5
\ No newline at end of file
diff --git a/tests/cachekey/project/sources/tar2.expected b/tests/cachekey/project/sources/tar2.expected
index e0c85d6bd..81410b8d9 100644
--- a/tests/cachekey/project/sources/tar2.expected
+++ b/tests/cachekey/project/sources/tar2.expected
@@ -1 +1 @@
-29f5688daecb262a4d818ec00a1bfc071fa9e54081aa098f9ef36f7e5dbdb065
\ No newline at end of file
+6d2398eb851bf9ac4dcfde382b4665b67096d3eba5e84e32a47123a81633f85e
\ No newline at end of file
diff --git a/tests/cachekey/project/sources/zip1.expected b/tests/cachekey/project/sources/zip1.expected
index 2f6a7d72c..ba2e2da89 100644
--- a/tests/cachekey/project/sources/zip1.expected
+++ b/tests/cachekey/project/sources/zip1.expected
@@ -1 +1 @@
-0b8b08e15b1d2ac12d7c9c4f4f9977b9c1363e017f938f6a871516f443d985b8
\ No newline at end of file
+66f9507afa8769d2b14d973bef1ad2d8bb3c911a6116ffed9f8757313fb9af96
\ No newline at end of file
diff --git a/tests/cachekey/project/sources/zip2.expected b/tests/cachekey/project/sources/zip2.expected
index 59fae2cce..17ae0db35 100644
--- a/tests/cachekey/project/sources/zip2.expected
+++ b/tests/cachekey/project/sources/zip2.expected
@@ -1 +1 @@
-943e869041c5a7319d63dacf4d10b717f43863d7754f5417d6f784a2f287d1be
\ No newline at end of file
+4eae2715c821e820df192446ff8642a5c73f9583785800df1d6a294f375a1e1d
\ No newline at end of file
diff --git a/tests/cachekey/project/target.expected b/tests/cachekey/project/target.expected
index 9be3be5f5..c6214eb23 100644
--- a/tests/cachekey/project/target.expected
+++ b/tests/cachekey/project/target.expected
@@ -1 +1 @@
-1b1f2ab00c24e9c1da970922d2f80a4cee71c5e578a0c5c0b2bca24249d5e822
\ No newline at end of file
+faf895b1292cdcba9724c337c70442aedc350bf053b9df937da2515a8cb69e5a
\ No newline at end of file
diff --git a/tests/integration/compose-symlinks.py b/tests/integration/compose-symlinks.py
index c6027bf2b..027feea7c 100644
--- a/tests/integration/compose-symlinks.py
+++ b/tests/integration/compose-symlinks.py
@@ -18,7 +18,7 @@ DATA_DIR = os.path.join(
 )
 
 
-# Test that staging a file inside a directory symlink works as expected.
+# Test that staging a file inside a directory symlink fails.
 #
 # Regression test for https://gitlab.com/BuildStream/buildstream/issues/270
 @pytest.mark.datafiles(DATA_DIR)
@@ -34,11 +34,6 @@ def test_compose_symlinks(cli, tmpdir, datafiles):
     os.symlink(os.path.join('usr', 'sbin'), symlink_file, target_is_directory=True)
 
     result = cli.run(project=project, args=['build', 'compose-symlinks/compose.bst'])
-    result.assert_success()
 
-    result = cli.run(project=project, args=['artifact', 'checkout', 'compose-symlinks/compose.bst',
-                                            '--directory', checkout])
-    result.assert_success()
-
-    assert set(walk_dir(checkout)) == set(['/sbin', '/usr', '/usr/sbin',
-                                           '/usr/sbin/init', '/usr/sbin/dummy'])
+    assert result.exit_code == -1
+    assert 'Destination is a symlink, not a directory: /sbin' in result.stderr
diff --git a/tests/integration/symlinks.py b/tests/integration/symlinks.py
index 22ff527f8..5db09d3d0 100644
--- a/tests/integration/symlinks.py
+++ b/tests/integration/symlinks.py
@@ -20,7 +20,7 @@ DATA_DIR = os.path.join(
 
 @pytest.mark.datafiles(DATA_DIR)
 @pytest.mark.skipif(not HAVE_SANDBOX, reason='Only available with a functioning sandbox')
-def test_absolute_symlinks_made_relative(cli, tmpdir, datafiles):
+def test_absolute_symlinks(cli, tmpdir, datafiles):
     project = os.path.join(datafiles.dirname, datafiles.basename)
     checkout = os.path.join(cli.directory, 'checkout')
     element_name = 'symlinks/dangling-symlink.bst'
@@ -34,17 +34,14 @@ def test_absolute_symlinks_made_relative(cli, tmpdir, datafiles):
     symlink = os.path.join(checkout, 'opt', 'orgname')
     assert os.path.islink(symlink)
 
-    # The symlink is created to point to /usr/orgs/orgname, but BuildStream
-    # should make all symlink target relative when assembling the artifact.
-    # This is done so that nothing points outside the sandbox and so that
-    # staging artifacts in locations other than / doesn't cause the links to
-    # all break.
-    assert os.readlink(symlink) == '../usr/orgs/orgname'
+    # The symlink is created to point to /usr/orgs/orgname and BuildStream
+    # should not mangle symlinks.
+    assert os.readlink(symlink) == '/usr/orgs/orgname'
 
 
 @pytest.mark.datafiles(DATA_DIR)
 @pytest.mark.skipif(not HAVE_SANDBOX, reason='Only available with a functioning sandbox')
-def test_allow_overlaps_inside_symlink_with_dangling_target(cli, tmpdir, datafiles):
+def test_disallow_overlaps_inside_symlink_with_dangling_target(cli, tmpdir, datafiles):
     project = os.path.join(datafiles.dirname, datafiles.basename)
     checkout = os.path.join(cli.directory, 'checkout')
     element_name = 'symlinks/dangling-symlink-overlap.bst'
@@ -53,10 +50,8 @@ def test_allow_overlaps_inside_symlink_with_dangling_target(cli, tmpdir, datafil
     assert result.exit_code == 0
 
     result = cli.run(project=project, args=['artifact', 'checkout', element_name, '--directory', checkout])
-    assert result.exit_code == 0
-
-    # See the dangling-symlink*.bst elements for details on what we are testing.
-    assert_contains(checkout, ['/usr/orgs/orgname/etc/org.conf'])
+    assert result.exit_code == -1
+    assert 'Destination is a symlink, not a directory: /opt/orgname' in result.stderr
 
 
 @pytest.mark.datafiles(DATA_DIR)
@@ -75,4 +70,4 @@ def test_detect_symlink_overlaps_pointing_outside_sandbox(cli, tmpdir, datafiles
     # tries to actually write there.
     result = cli.run(project=project, args=['artifact', 'checkout', element_name, '--directory', checkout])
     assert result.exit_code == -1
-    assert "Destination path resolves to a path outside of the staging area" in result.stderr
+    assert 'Destination is a symlink, not a directory: /opt/escape-hatch' in result.stderr
diff --git a/tests/internals/storage_vdir_import.py b/tests/internals/storage_vdir_import.py
index 1d61a6e5f..0531b7c1b 100644
--- a/tests/internals/storage_vdir_import.py
+++ b/tests/internals/storage_vdir_import.py
@@ -28,12 +28,12 @@ root_filesets = [
     [('a/b/c/textfile1', 'F', 'This is textfile 1\n')],
     [('a/b/c/textfile1', 'F', 'This is the replacement textfile 1\n')],
     [('a/b/d', 'D', '')],
-    [('a/b/c', 'S', '/a/b/d')],
-    [('a/b/d', 'S', '/a/b/c')],
-    [('a/b/d', 'D', ''), ('a/b/c', 'S', '/a/b/d')],
-    [('a/b/c', 'D', ''), ('a/b/d', 'S', '/a/b/c')],
-    [('a/b', 'F', 'This is textfile 1\n')],
-    [('a/b/c', 'F', 'This is textfile 1\n')],
+    [('a/b/e', 'S', '/a/b/d')],
+    [('a/b/f', 'S', '/a/b/c')],
+    [('a/b/d', 'D', ''), ('a/b/e', 'S', '/a/b/d')],
+    [('a/b/c', 'D', ''), ('a/b/f', 'S', '/a/b/c')],
+    [('a/c', 'F', 'This is textfile 1\n')],
+    [('a/b/e', 'F', 'This is textfile 1\n')],
     [('a/b/c', 'D', '')]
 ]
 
@@ -77,6 +77,8 @@ def generate_random_root(rootno, directory):
         location = random.choice(locations)
         thingname = "node{}".format(i)
         thing = random.choice(['dir', 'link', 'file'])
+        if thing == 'dir':
+            thingname = "dir" + thingname
         target = os.path.join(rootdir, location, thingname)
         if thing == 'dir':
             os.makedirs(target)