From 177c42649a3c6633b71a0e599ae0e5275476c964 Mon Sep 17 00:00:00 2001 From: William Salmon Date: Wed, 1 Aug 2018 14:36:46 +0100 Subject: Trying to mitigate a mtime granularity braking the cache tests This patch mitigates the granularity of the mtimes used on the default gitlab runners, Allowing the test suite to pass on these runners. --- tests/artifactcache/expiry.py | 23 ++++++++++++++++++++++- tests/frontend/push.py | 26 +++++++++++++++++++++++++- tests/frontend/workspace.py | 29 ++++++++++++++++++++++++++++- tests/testutils/__init__.py | 26 ++++++++++++++++++++++++++ tests/testutils/runner_integration.py | 32 ++++++++++++++++++++++++++++++++ 5 files changed, 133 insertions(+), 3 deletions(-) create mode 100644 tests/testutils/runner_integration.py diff --git a/tests/artifactcache/expiry.py b/tests/artifactcache/expiry.py index 9c74eb1c4..9b77fbe3e 100644 --- a/tests/artifactcache/expiry.py +++ b/tests/artifactcache/expiry.py @@ -1,3 +1,22 @@ +# +# Copyright (C) 2018 Codethink Limited +# +# This program is free software; you can redistribute it and/or +# modify it under the terms of the GNU Lesser General Public +# License as published by the Free Software Foundation; either +# version 2 of the License, or (at your option) any later version. +# +# This library is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU +# Lesser General Public License for more details. +# +# You should have received a copy of the GNU Lesser General Public +# License along with this library. If not, see . +# +# Authors: Tristan Maat +# + import os import pytest @@ -5,7 +24,7 @@ import pytest from buildstream import _yaml from buildstream._exceptions import ErrorDomain, LoadErrorReason -from tests.testutils import cli, create_element_size +from tests.testutils import cli, create_element_size, wait_for_cache_granularity DATA_DIR = os.path.join( @@ -108,6 +127,8 @@ def test_expiry_order(cli, datafiles, tmpdir): res = cli.run(project=project, args=['build', 'target2.bst']) res.assert_success() + wait_for_cache_granularity() + # Now extract dep.bst res = cli.run(project=project, args=['checkout', 'dep.bst', checkout]) res.assert_success() diff --git a/tests/frontend/push.py b/tests/frontend/push.py index 6ee301e6d..f351e33be 100644 --- a/tests/frontend/push.py +++ b/tests/frontend/push.py @@ -1,9 +1,31 @@ +# +# Copyright (C) 2018 Codethink Limited +# Copyright (C) 2018 Bloomberg Finance LP +# +# This program is free software; you can redistribute it and/or +# modify it under the terms of the GNU Lesser General Public +# License as published by the Free Software Foundation; either +# version 2 of the License, or (at your option) any later version. +# +# This library is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU +# Lesser General Public License for more details. +# +# You should have received a copy of the GNU Lesser General Public +# License along with this library. If not, see . +# +# Authors: Tristan Van Berkom +# Sam Thursfield +# Jürg Billeter +# + import os import pytest from buildstream._exceptions import ErrorDomain from tests.testutils import cli, create_artifact_share, create_element_size -from tests.testutils import generate_junction +from tests.testutils import generate_junction, wait_for_cache_granularity from . import configure_project @@ -327,6 +349,8 @@ def test_recently_pulled_artifact_does_not_expire(cli, datafiles, tmpdir): # Ensure element1 is cached locally assert cli.get_element_state(project, 'element1.bst') == 'cached' + wait_for_cache_granularity() + # Create and build the element3 (of 5 MB) create_element_size('element3.bst', project, element_path, [], int(5e6)) result = cli.run(project=project, args=['build', 'element3.bst']) diff --git a/tests/frontend/workspace.py b/tests/frontend/workspace.py index fff9bf00a..c7af0a70f 100644 --- a/tests/frontend/workspace.py +++ b/tests/frontend/workspace.py @@ -1,9 +1,34 @@ +# +# Copyright (C) 2018 Codethink Limited +# Copyright (C) 2018 Bloomberg Finance LP +# +# This program is free software; you can redistribute it and/or +# modify it under the terms of the GNU Lesser General Public +# License as published by the Free Software Foundation; either +# version 2 of the License, or (at your option) any later version. +# +# This library is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU +# Lesser General Public License for more details. +# +# You should have received a copy of the GNU Lesser General Public +# License along with this library. If not, see . +# +# Authors: Tristan Van Berkom +# Tristan Maat +# Chandan Singh +# Phillip Smyth +# Jonathan Maw +# Richard Maw +# + import os import pytest import shutil import subprocess from ruamel.yaml.comments import CommentedSet -from tests.testutils import cli, create_repo, ALL_REPO_KINDS +from tests.testutils import cli, create_repo, ALL_REPO_KINDS, wait_for_cache_granularity from buildstream import _yaml from buildstream._exceptions import ErrorDomain, LoadError, LoadErrorReason @@ -507,6 +532,8 @@ def test_detect_modifications(cli, tmpdir, datafiles, modification, strict): assert cli.get_element_state(project, element_name) == 'cached' assert cli.get_element_key(project, element_name) != "{:?<64}".format('') + wait_for_cache_granularity() + # Modify the workspace in various different ways, ensuring we # properly detect the changes. # diff --git a/tests/testutils/__init__.py b/tests/testutils/__init__.py index e9db94989..c2fae1cc4 100644 --- a/tests/testutils/__init__.py +++ b/tests/testutils/__init__.py @@ -1,5 +1,31 @@ +# +# Copyright (C) 2018 Codethink Limited +# Copyright (C) 2018 Bloomberg Finance LP +# +# This program is free software; you can redistribute it and/or +# modify it under the terms of the GNU Lesser General Public +# License as published by the Free Software Foundation; either +# version 2 of the License, or (at your option) any later version. +# +# This library is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU +# Lesser General Public License for more details. +# +# You should have received a copy of the GNU Lesser General Public +# License along with this library. If not, see . +# +# Authors: Tristan Van Berkom +# Tristan Maat +# Sam Thursfield +# James Ennis +# Valentin David +# William Salmon +# + from .runcli import cli, cli_integration from .repo import create_repo, ALL_REPO_KINDS from .artifactshare import create_artifact_share from .element_generators import create_element_size from .junction import generate_junction +from .runner_integration import wait_for_cache_granularity diff --git a/tests/testutils/runner_integration.py b/tests/testutils/runner_integration.py new file mode 100644 index 000000000..9c49b2fb0 --- /dev/null +++ b/tests/testutils/runner_integration.py @@ -0,0 +1,32 @@ +# +# Copyright (C) 2018 Bloomberg Finance LP +# +# This program is free software; you can redistribute it and/or +# modify it under the terms of the GNU Lesser General Public +# License as published by the Free Software Foundation; either +# version 2 of the License, or (at your option) any later version. +# +# This library is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU +# Lesser General Public License for more details. +# +# You should have received a copy of the GNU Lesser General Public +# License along with this library. If not, see . +# +# Authors: +# Will Salmon + +import time + + +def wait_for_cache_granularity(): + # This isn't called very often so has minimal impact on test runtime. + # If this changes it may be worth while adding a more sophisticated approach. + """ + Mitigate the coarse granularity of the gitlab runners mtime + + This function waits for the mtime to increment so that the cache can sort by mtime and + get the most recent results. + """ + time.sleep(1.1) -- cgit v1.2.1