From d164ae319fced1e41ce5978a90c2bd64983cb54e Mon Sep 17 00:00:00 2001 From: Tristan van Berkom Date: Sun, 25 Oct 2020 18:10:18 +0900 Subject: src/buildstream/testing/_utils/site.py: Adding have_subsecond_mtime() A utility for tests to check if subsecond precision mtime is supported at a given filesystem path (it will depend on underlying filesystem, so we cannot have a simple HAVE_SUBSECOND_MTIME variable to check, because we don't know where tests will operate at this stage). --- src/buildstream/testing/_utils/site.py | 32 ++++++++++++++++++++++++++++++++ 1 file changed, 32 insertions(+) diff --git a/src/buildstream/testing/_utils/site.py b/src/buildstream/testing/_utils/site.py index 705419f58..727fe015b 100644 --- a/src/buildstream/testing/_utils/site.py +++ b/src/buildstream/testing/_utils/site.py @@ -5,6 +5,7 @@ import os import stat import subprocess import sys +import tempfile from typing import Optional # pylint: disable=unused-import from buildstream import utils, ProgramNotFoundError @@ -73,3 +74,34 @@ try: HAVE_SANDBOX = "buildbox-run" except (ProgramNotFoundError, OSError, subprocess.CalledProcessError): pass + + +# Check if we have subsecond mtime support on the +# filesystem where @directory is located. +# +def have_subsecond_mtime(directory): + + try: + test_file, test_filename = tempfile.mkstemp(dir=directory) + os.close(test_file) + except OSError: + # If we can't create a temp file, lets just say this is False + return False + + try: + os.utime(test_filename, times=None, ns=(int(12345), int(12345))) + except OSError: + # If we can't set the mtime, lets just say this is False + os.unlink(test_filename) + return False + + try: + stat_result = os.stat(test_filename) + except OSError: + # If we can't stat the file, lets just say this is False + os.unlink(test_filename) + return False + + os.unlink(test_filename) + + return stat_result.st_mtime_ns == 12345 -- cgit v1.2.1 From f2579bea17ed4660b1e06f584c62bad3ec3fcaf0 Mon Sep 17 00:00:00 2001 From: Tristan van Berkom Date: Sun, 25 Oct 2020 18:11:54 +0900 Subject: tests/internals/storage_vdir_import.py: Conditionally skip Conditionally skip tests which require subsecond mtime precision. --- tests/internals/storage_vdir_import.py | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/tests/internals/storage_vdir_import.py b/tests/internals/storage_vdir_import.py index ffd727ffe..ff2b14026 100644 --- a/tests/internals/storage_vdir_import.py +++ b/tests/internals/storage_vdir_import.py @@ -24,6 +24,7 @@ from buildstream.storage._filebaseddirectory import FileBasedDirectory from buildstream._cas import CASCache from buildstream.storage.directory import VirtualDirectoryError from buildstream.utils import _set_file_mtime, _parse_timestamp +from buildstream.testing._utils.site import have_subsecond_mtime # These are comparitive tests that check that FileBasedDirectory and @@ -185,6 +186,12 @@ def directory_not_empty(path): def _import_test(tmpdir, original, overlay, generator_function, verify_contents=False): + + # Skip this test if we do not have support for subsecond precision mtimes + # + if not have_subsecond_mtime(str(tmpdir)): + pytest.skip("Filesystem does not support subsecond mtime precision: {}".format(str(tmpdir))) + cas_cache = CASCache(tmpdir, log_directory=os.path.join(tmpdir, "logs")) try: # Create some fake content -- cgit v1.2.1 From f9ddbaaa4ae14d4c733dcc011ce6fcbb39f7607f Mon Sep 17 00:00:00 2001 From: Tristan van Berkom Date: Sun, 25 Oct 2020 18:22:03 +0900 Subject: tests/artifactcache/expiry.py: Conditionally skip some tests Skip some of the artifact expiry tests in the case we don't have subsecond mtime precision. --- tests/artifactcache/expiry.py | 45 +++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 45 insertions(+) diff --git a/tests/artifactcache/expiry.py b/tests/artifactcache/expiry.py index f2be797c3..f56811981 100644 --- a/tests/artifactcache/expiry.py +++ b/tests/artifactcache/expiry.py @@ -28,6 +28,7 @@ import pytest from buildstream._cas import CASCache from buildstream.exceptions import ErrorDomain, LoadErrorReason from buildstream.testing import cli # pylint: disable=unused-import +from buildstream.testing._utils.site import have_subsecond_mtime from tests.testutils import create_element_size, wait_for_cache_granularity @@ -58,6 +59,17 @@ def test_artifact_expires(cli, datafiles): project = str(datafiles) element_path = "elements" + # Skip this test if we do not have support for subsecond precision mtimes + # + # The artifact expiry logic relies on mtime changes, in real life second precision + # should be enough for this to work almost all the time, but test cases happen very + # quickly, resulting in all artifacts having the same mtime. + # + # This test requires subsecond mtime to be reliable. + # + if not have_subsecond_mtime(project): + pytest.skip("Filesystem does not support subsecond mtime precision: {}".format(project)) + cli.configure({"cache": {"quota": 10000000,}}) # Create an element that uses almost the entire cache (an empty @@ -99,6 +111,17 @@ def test_artifact_too_large(cli, datafiles, size): project = str(datafiles) element_path = "elements" + # Skip this test if we do not have support for subsecond precision mtimes + # + # The artifact expiry logic relies on mtime changes, in real life second precision + # should be enough for this to work almost all the time, but test cases happen very + # quickly, resulting in all artifacts having the same mtime. + # + # This test requires subsecond mtime to be reliable. + # + if not have_subsecond_mtime(project): + pytest.skip("Filesystem does not support subsecond mtime precision: {}".format(project)) + cli.configure({"cache": {"quota": 400000}}) # Create an element whose artifact is too large @@ -169,6 +192,17 @@ def test_keep_dependencies(cli, datafiles): project = str(datafiles) element_path = "elements" + # Skip this test if we do not have support for subsecond precision mtimes + # + # The artifact expiry logic relies on mtime changes, in real life second precision + # should be enough for this to work almost all the time, but test cases happen very + # quickly, resulting in all artifacts having the same mtime. + # + # This test requires subsecond mtime to be reliable. + # + if not have_subsecond_mtime(project): + pytest.skip("Filesystem does not support subsecond mtime precision: {}".format(project)) + cli.configure({"cache": {"quota": 10000000}}) # Create a pretty big dependency @@ -210,6 +244,17 @@ def test_never_delete_required(cli, datafiles): project = str(datafiles) element_path = "elements" + # Skip this test if we do not have support for subsecond precision mtimes + # + # The artifact expiry logic relies on mtime changes, in real life second precision + # should be enough for this to work almost all the time, but test cases happen very + # quickly, resulting in all artifacts having the same mtime. + # + # This test requires subsecond mtime to be reliable. + # + if not have_subsecond_mtime(project): + pytest.skip("Filesystem does not support subsecond mtime precision: {}".format(project)) + cli.configure({"cache": {"quota": 10000000}, "scheduler": {"fetchers": 1, "builders": 1}}) # Create a linear build tree -- cgit v1.2.1 From 3a604d784c929a9dd01eb4dc65519ea53de0df9f Mon Sep 17 00:00:00 2001 From: Tristan van Berkom Date: Sun, 25 Oct 2020 19:11:25 +0900 Subject: tests/internals/utils_move_atomic.py: Conditionally skip a test Skip the mtime related test if the underlying filesystem does not support subsecond precision mtime. --- tests/internals/utils_move_atomic.py | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/tests/internals/utils_move_atomic.py b/tests/internals/utils_move_atomic.py index dd417cb66..f78ac67fc 100644 --- a/tests/internals/utils_move_atomic.py +++ b/tests/internals/utils_move_atomic.py @@ -10,6 +10,7 @@ from buildstream.utils import ( _set_file_mtime, _parse_timestamp, ) +from buildstream.testing._utils.site import have_subsecond_mtime @pytest.fixture @@ -98,6 +99,12 @@ def test_move_to_existing_non_empty_dir(src, tmp_path): def test_move_to_empty_dir_set_mtime(src, tmp_path): + + # Skip this test if we do not have support for subsecond precision mtimes + # + if not have_subsecond_mtime(str(tmp_path)): + pytest.skip("Filesystem does not support subsecond mtime precision: {}".format(str(tmp_path))) + dst = tmp_path.joinpath("dst") move_atomic(src, dst) assert dst.joinpath("test").exists() -- cgit v1.2.1 From aee49156c9f97d47df321c25affef7e6d00d16d2 Mon Sep 17 00:00:00 2001 From: Tristan van Berkom Date: Sun, 25 Oct 2020 19:32:49 +0900 Subject: tests/frontend/track.py: Conditionally skip a test Skip a test which relies on mtimes differing within a short timespan, this will fail if it happens fast enough (which it should) on systems which do not support subsecond precision mtimes. --- tests/frontend/track.py | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/tests/frontend/track.py b/tests/frontend/track.py index 682ee0e98..bd8444973 100644 --- a/tests/frontend/track.py +++ b/tests/frontend/track.py @@ -8,6 +8,7 @@ import pytest from buildstream.testing import create_repo, generate_project from buildstream.testing import cli # pylint: disable=unused-import +from buildstream.testing._utils.site import have_subsecond_mtime from buildstream.exceptions import ErrorDomain, LoadErrorReason from buildstream import _yaml from tests.testutils import generate_junction @@ -366,6 +367,11 @@ def test_no_needless_overwrite(cli, tmpdir, datafiles): element_path = os.path.join(project, "elements") target = "track-test-target.bst" + # Skip this test if we do not have support for subsecond precision mtimes + # + if not have_subsecond_mtime(project): + pytest.skip("Filesystem does not support subsecond mtime precision: {}".format(project)) + # Create our repo object of the given source type with # the dev files, and then collect the initial ref. # -- cgit v1.2.1