Trying to mitigate a mtime granularity braking the cache testswillsalmon/CacheExpiryTest

This patch mitigates the granularity of the mtimes used on the default
gitlab runners, Allowing the test suite to pass on these runners.

5 files changed, 133 insertions, 3 deletions

diff --git a/tests/artifactcache/expiry.py b/tests/artifactcache/expiry.py
index 9c74eb1c4..9b77fbe3e 100644
--- a/tests/artifactcache/expiry.py
+++ b/tests/artifactcache/expiry.py
@@ -1,3 +1,22 @@
+#
+#  Copyright (C) 2018 Codethink Limited
+#
+#  This program is free software; you can redistribute it and/or
+#  modify it under the terms of the GNU Lesser General Public
+#  License as published by the Free Software Foundation; either
+#  version 2 of the License, or (at your option) any later version.
+#
+#  This library is distributed in the hope that it will be useful,
+#  but WITHOUT ANY WARRANTY; without even the implied warranty of
+#  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+#  Lesser General Public License for more details.
+#
+#  You should have received a copy of the GNU Lesser General Public
+#  License along with this library. If not, see <http://www.gnu.org/licenses/>.
+#
+#  Authors: Tristan Maat <tristan.maat@codethink.co.uk>
+#
+
 import os
 
 import pytest
@@ -5,7 +24,7 @@ import pytest
 from buildstream import _yaml
 from buildstream._exceptions import ErrorDomain, LoadErrorReason
 
-from tests.testutils import cli, create_element_size
+from tests.testutils import cli, create_element_size, wait_for_cache_granularity
 
 
 DATA_DIR = os.path.join(
@@ -108,6 +127,8 @@ def test_expiry_order(cli, datafiles, tmpdir):
     res = cli.run(project=project, args=['build', 'target2.bst'])
     res.assert_success()
 
+    wait_for_cache_granularity()
+
     # Now extract dep.bst
     res = cli.run(project=project, args=['checkout', 'dep.bst', checkout])
     res.assert_success()
diff --git a/tests/frontend/push.py b/tests/frontend/push.py
index 6ee301e6d..f351e33be 100644
--- a/tests/frontend/push.py
+++ b/tests/frontend/push.py
@@ -1,9 +1,31 @@
+#
+#  Copyright (C) 2018 Codethink Limited
+#  Copyright (C) 2018 Bloomberg Finance LP
+#
+#  This program is free software; you can redistribute it and/or
+#  modify it under the terms of the GNU Lesser General Public
+#  License as published by the Free Software Foundation; either
+#  version 2 of the License, or (at your option) any later version.
+#
+#  This library is distributed in the hope that it will be useful,
+#  but WITHOUT ANY WARRANTY; without even the implied warranty of
+#  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+#  Lesser General Public License for more details.
+#
+#  You should have received a copy of the GNU Lesser General Public
+#  License along with this library. If not, see <http://www.gnu.org/licenses/>.
+#
+#  Authors: Tristan Van Berkom <tristan.vanberkom@codethink.co.uk>
+#           Sam Thursfield <sam.thursfield@codethink.co.uk>
+#           Jürg Billeter <juerg.billeter@codethink.co.uk>
+#
+
 import os
 import pytest
 
 from buildstream._exceptions import ErrorDomain
 from tests.testutils import cli, create_artifact_share, create_element_size
-from tests.testutils import generate_junction
+from tests.testutils import generate_junction, wait_for_cache_granularity
 from . import configure_project
 
 
@@ -327,6 +349,8 @@ def test_recently_pulled_artifact_does_not_expire(cli, datafiles, tmpdir):
         # Ensure element1 is cached locally
         assert cli.get_element_state(project, 'element1.bst') == 'cached'
 
+        wait_for_cache_granularity()
+
         # Create and build the element3 (of 5 MB)
         create_element_size('element3.bst', project, element_path, [], int(5e6))
         result = cli.run(project=project, args=['build', 'element3.bst'])
diff --git a/tests/frontend/workspace.py b/tests/frontend/workspace.py
index fff9bf00a..c7af0a70f 100644
--- a/tests/frontend/workspace.py
+++ b/tests/frontend/workspace.py
@@ -1,9 +1,34 @@
+#
+#  Copyright (C) 2018 Codethink Limited
+#  Copyright (C) 2018 Bloomberg Finance LP
+#
+#  This program is free software; you can redistribute it and/or
+#  modify it under the terms of the GNU Lesser General Public
+#  License as published by the Free Software Foundation; either
+#  version 2 of the License, or (at your option) any later version.
+#
+#  This library is distributed in the hope that it will be useful,
+#  but WITHOUT ANY WARRANTY; without even the implied warranty of
+#  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+#  Lesser General Public License for more details.
+#
+#  You should have received a copy of the GNU Lesser General Public
+#  License along with this library. If not, see <http://www.gnu.org/licenses/>.
+#
+#  Authors: Tristan Van Berkom <tristan.vanberkom@codethink.co.uk>
+#           Tristan Maat <tristan.maat@codethink.co.uk>
+#           Chandan Singh <csingh43@bloomberg.net>
+#           Phillip Smyth <phillip.smyth@codethink.co.uk>
+#           Jonathan Maw <jonathan.maw@codethink.co.uk>
+#           Richard Maw <richard.maw@codethink.co.uk>
+#
+
 import os
 import pytest
 import shutil
 import subprocess
 from ruamel.yaml.comments import CommentedSet
-from tests.testutils import cli, create_repo, ALL_REPO_KINDS
+from tests.testutils import cli, create_repo, ALL_REPO_KINDS, wait_for_cache_granularity
 
 from buildstream import _yaml
 from buildstream._exceptions import ErrorDomain, LoadError, LoadErrorReason
@@ -507,6 +532,8 @@ def test_detect_modifications(cli, tmpdir, datafiles, modification, strict):
     assert cli.get_element_state(project, element_name) == 'cached'
     assert cli.get_element_key(project, element_name) != "{:?<64}".format('')
 
+    wait_for_cache_granularity()
+
     # Modify the workspace in various different ways, ensuring we
     # properly detect the changes.
     #
diff --git a/tests/testutils/__init__.py b/tests/testutils/__init__.py
index e9db94989..c2fae1cc4 100644
--- a/tests/testutils/__init__.py
+++ b/tests/testutils/__init__.py
@@ -1,5 +1,31 @@
+#
+#  Copyright (C) 2018 Codethink Limited
+#  Copyright (C) 2018 Bloomberg Finance LP
+#
+#  This program is free software; you can redistribute it and/or
+#  modify it under the terms of the GNU Lesser General Public
+#  License as published by the Free Software Foundation; either
+#  version 2 of the License, or (at your option) any later version.
+#
+#  This library is distributed in the hope that it will be useful,
+#  but WITHOUT ANY WARRANTY; without even the implied warranty of
+#  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+#  Lesser General Public License for more details.
+#
+#  You should have received a copy of the GNU Lesser General Public
+#  License along with this library. If not, see <http://www.gnu.org/licenses/>.
+#
+#  Authors: Tristan Van Berkom <tristan.vanberkom@codethink.co.uk>
+#           Tristan Maat <tristan.maat@codethink.co.uk>
+#           Sam Thursfield <sam.thursfield@codethink.co.uk>
+#           James Ennis <james.ennis@codethink.co.uk>
+#           Valentin David <valentin.david@codethink.co.uk>
+#           William Salmon <will.salmon@codethink.co.uk>
+#
+
 from .runcli import cli, cli_integration
 from .repo import create_repo, ALL_REPO_KINDS
 from .artifactshare import create_artifact_share
 from .element_generators import create_element_size
 from .junction import generate_junction
+from .runner_integration import wait_for_cache_granularity
diff --git a/tests/testutils/runner_integration.py b/tests/testutils/runner_integration.py
new file mode 100644
index 000000000..9c49b2fb0
--- /dev/null
+++ b/tests/testutils/runner_integration.py
@@ -0,0 +1,32 @@
+#
+#  Copyright (C) 2018 Bloomberg Finance LP
+#
+#  This program is free software; you can redistribute it and/or
+#  modify it under the terms of the GNU Lesser General Public
+#  License as published by the Free Software Foundation; either
+#  version 2 of the License, or (at your option) any later version.
+#
+#  This library is distributed in the hope that it will be useful,
+#  but WITHOUT ANY WARRANTY; without even the implied warranty of
+#  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+#  Lesser General Public License for more details.
+#
+#  You should have received a copy of the GNU Lesser General Public
+#  License along with this library. If not, see <http://www.gnu.org/licenses/>.
+#
+#  Authors:
+#         Will Salmon <will.salmon@codethink.co.uk>
+
+import time
+
+
+def wait_for_cache_granularity():
+    # This isn't called very often so has minimal impact on test runtime.
+    # If this changes it may be worth while adding a more sophisticated approach.
+    """
+    Mitigate the coarse granularity of the gitlab runners mtime
+
+    This function waits for the mtime to increment so that the cache can sort by mtime and
+    get the most recent results.
+    """
+    time.sleep(1.1)