diff options
author | Darius Makovsky <traveltissues@protonmail.com> | 2019-10-30 17:25:52 +0000 |
---|---|---|
committer | Darius Makovsky <traveltissues@protonmail.com> | 2019-11-04 12:54:55 +0000 |
commit | 861f9636a503f6dbd5acc132e2f8ba6743762216 (patch) | |
tree | dcf3d5954781af028519296150a335843ab289cf | |
parent | 1993ff0d96c51ee6aa1d4a601ad976b20f175a46 (diff) | |
download | buildstream-861f9636a503f6dbd5acc132e2f8ba6743762216.tar.gz |
Split dependencies in the cache key dict into strong and weak
30 files changed, 51 insertions, 36 deletions
diff --git a/src/buildstream/_artifactelement.py b/src/buildstream/_artifactelement.py index 48c3d1769..5ba0f5ed2 100644 --- a/src/buildstream/_artifactelement.py +++ b/src/buildstream/_artifactelement.py @@ -17,13 +17,13 @@ # Authors: # James Ennis <james.ennis@codethink.co.uk> -from typing import TYPE_CHECKING +from typing import TYPE_CHECKING, Optional from . import Element from . import _cachekey from ._exceptions import ArtifactElementError from ._loader.metaelement import MetaElement -from .types import Scope +from .types import Scope, _KeyStrength, List, Union if TYPE_CHECKING: from typing import Dict @@ -147,7 +147,9 @@ class ArtifactElement(Element): sandbox.set_output_directory(install_root) # Override Element._calculate_cache_key - def _calculate_cache_key(self, dependencies=None): + def _calculate_cache_key(self, + dependencies: List[Union[str, None]], + dep_strength: int = _KeyStrength.STRONG) -> Optional[str]: return self._key # Override Element._get_cache_key() diff --git a/src/buildstream/element.py b/src/buildstream/element.py index a6b2af110..0efb6b48c 100644 --- a/src/buildstream/element.py +++ b/src/buildstream/element.py @@ -83,7 +83,7 @@ from functools import partial from itertools import chain import tempfile import string -from typing import cast, TYPE_CHECKING, Any, Dict, Iterator, List, Optional +from typing import cast, TYPE_CHECKING, Any, Dict, Iterator, List, Optional, Union from pyroaring import BitMap # pylint: disable=no-name-in-module @@ -217,7 +217,8 @@ class Element(Plugin): def __init__(self, context: 'Context', project: 'Project', meta: 'MetaElement', plugin_conf: Dict[str, Any]): - self.__cache_key_dict = None # Dict for cache key calculation + # Dict for cache key calculation + self.__cache_key_dict = None # type: Any self.__cache_key = None # Our cached cache key super().__init__(meta.name, context, project, meta.provenance, "element") @@ -2126,12 +2127,18 @@ class Element(Plugin): # # Calculates the cache key # + # Args: + # dependencies (list): dependency keys + # strength (int): strength of dependency keys + # # Returns: # (str): A hex digest cache key for this Element, or None # # None is returned if information for the cache key is missing. # - def _calculate_cache_key(self, dependencies): + def _calculate_cache_key(self, + dependencies: List[Union[str, None]], + dep_strength: int = _KeyStrength.STRONG) -> Optional[str]: # No cache keys for dependencies which have no cache keys if None in dependencies: return None @@ -2167,7 +2174,13 @@ class Element(Plugin): self.__cache_key_dict['fatal-warnings'] = sorted(project._fatal_warnings) cache_key_dict = self.__cache_key_dict.copy() - cache_key_dict['dependencies'] = dependencies + cache_key_dict['dependency-keys-strong'] = [] + cache_key_dict['dependency-keys-weak'] = [] + + if dep_strength == _KeyStrength.WEAK: + cache_key_dict['dependency-keys-weak'] = dependencies + else: + cache_key_dict['dependency-keys-strong'] = dependencies return _cachekey.generate_key(cache_key_dict) @@ -3088,7 +3101,7 @@ class Element(Plugin): for e in self.dependencies(Scope.BUILD) ] - self.__weak_cache_key = self._calculate_cache_key(dependencies) + self.__weak_cache_key = self._calculate_cache_key(dependencies, dep_strength=_KeyStrength.WEAK) if self.__weak_cache_key is None: # Weak cache key could not be calculated yet, therefore diff --git a/tests/cachekey/project/elements/build1.expected b/tests/cachekey/project/elements/build1.expected index e3653d845..4044df2c9 100644 --- a/tests/cachekey/project/elements/build1.expected +++ b/tests/cachekey/project/elements/build1.expected @@ -1 +1 @@ -87391ccead1223d376cbc5261a42a163762f895d5c035430084b9e419b004174
\ No newline at end of file +5428a5de8a9b26513aacc1f319723f601ea15ab9287ce1797cf8104029d22e8d
\ No newline at end of file diff --git a/tests/cachekey/project/elements/build2.expected b/tests/cachekey/project/elements/build2.expected index e5f7e165c..85f8ffd44 100644 --- a/tests/cachekey/project/elements/build2.expected +++ b/tests/cachekey/project/elements/build2.expected @@ -1 +1 @@ -55eff1fd81182b3a441d5e587d99ceff025d866bbe614260b3aa8b8356eb79aa
\ No newline at end of file +f1eae5a7b0ebb147fc3e910d9f802553c79242af6d2901b7e9386768531a0951
\ No newline at end of file diff --git a/tests/cachekey/project/elements/compose1.expected b/tests/cachekey/project/elements/compose1.expected index 3ceee5957..c13c9ddd6 100644 --- a/tests/cachekey/project/elements/compose1.expected +++ b/tests/cachekey/project/elements/compose1.expected @@ -1 +1 @@ -a5bcbca1752d8378cbd1a85ea93f4ecf7a74476fa9cd81f899c2cdc2a5f4fb43
\ No newline at end of file +21eab9c3f5f64f06a86b206b489282c36d2730af751790e25c32db288530ac51
\ No newline at end of file diff --git a/tests/cachekey/project/elements/compose2.expected b/tests/cachekey/project/elements/compose2.expected index c906f88f5..ace86d0ca 100644 --- a/tests/cachekey/project/elements/compose2.expected +++ b/tests/cachekey/project/elements/compose2.expected @@ -1 +1 @@ -07f80eaf2c6de2e019b64ecccae1dd45696427d6a6d05680e429785fa341e318
\ No newline at end of file +428aadf565b5677c4d08bcba0d61f61877dce32c85d4b1c361839c5c8f95f230
\ No newline at end of file diff --git a/tests/cachekey/project/elements/compose3.expected b/tests/cachekey/project/elements/compose3.expected index b795a5bd3..5d818c0d3 100644 --- a/tests/cachekey/project/elements/compose3.expected +++ b/tests/cachekey/project/elements/compose3.expected @@ -1 +1 @@ -1fed855571867d0b1bd7f84128ad21aab60550a7bd6b32c679b093c1d3f8fdb8
\ No newline at end of file +f4ba0e2ca52dcada647e1d67f4d08211b2136833ac88dc390f4cade069893540
\ No newline at end of file diff --git a/tests/cachekey/project/elements/compose4.expected b/tests/cachekey/project/elements/compose4.expected index c9986deed..73c199427 100644 --- a/tests/cachekey/project/elements/compose4.expected +++ b/tests/cachekey/project/elements/compose4.expected @@ -1 +1 @@ -8f1a61a35a40e968acfaf298403f8546f80d63597954a974a261d8aea1a715e7
\ No newline at end of file +df638236726329e1bccb268b2b222559f6bb5e38b1d8621caef040c4ef9336c4
\ No newline at end of file diff --git a/tests/cachekey/project/elements/compose5.expected b/tests/cachekey/project/elements/compose5.expected index 214600317..8f8c72977 100644 --- a/tests/cachekey/project/elements/compose5.expected +++ b/tests/cachekey/project/elements/compose5.expected @@ -1 +1 @@ -96622c2c119879119c54412437eba4d8b4e906c5d27bae1e7bffc1da3a2a6b5e
\ No newline at end of file +7ad4ce5aedcaa7e0f25cb31c4e6016fb547df90b62dc1e1c6cf9eb37dbf12462
\ No newline at end of file diff --git a/tests/cachekey/project/elements/import1.expected b/tests/cachekey/project/elements/import1.expected index 78f4972fa..ad2e9a9d8 100644 --- a/tests/cachekey/project/elements/import1.expected +++ b/tests/cachekey/project/elements/import1.expected @@ -1 +1 @@ -7596dd8b5e930d95b0d93f8aef1fb41b4ee0ec952dccb5cd1a4364cd6d63dd41
\ No newline at end of file +8f05d68de71fbd70661eb4a3516189d754c9ee76af42eff93ed9cde6c441e257
\ No newline at end of file diff --git a/tests/cachekey/project/elements/import2.expected b/tests/cachekey/project/elements/import2.expected index 52b074d6c..179a86778 100644 --- a/tests/cachekey/project/elements/import2.expected +++ b/tests/cachekey/project/elements/import2.expected @@ -1 +1 @@ -1895fc7749e6839c26b881f1957e8778e7a459ef1ffaca8cf409d45324330362
\ No newline at end of file +df9c8856a0e652bacbee483ec115e61336bad7d008bcbb8b31cc604a819c9b3a
\ No newline at end of file diff --git a/tests/cachekey/project/elements/import3.expected b/tests/cachekey/project/elements/import3.expected index 9a0a4ae66..10b9216f5 100644 --- a/tests/cachekey/project/elements/import3.expected +++ b/tests/cachekey/project/elements/import3.expected @@ -1 +1 @@ -a51f965f654e70ef4630d2f6cbaeb8c9f27bc190d3c7827e0e95feb4fc83b944
\ No newline at end of file +7afb1275c1d5f2e019ab6319dcd19447fcebd2f587a4aba9f25d86fd8df93d75
\ No newline at end of file diff --git a/tests/cachekey/project/elements/script1.expected b/tests/cachekey/project/elements/script1.expected index 51e5b97e1..ec6dc0a01 100644 --- a/tests/cachekey/project/elements/script1.expected +++ b/tests/cachekey/project/elements/script1.expected @@ -1 +1 @@ -00788f98d62eb5fafac1fc1bd52530fcb8f03091edf205b308c61d2b621bebe3
\ No newline at end of file +249d98e96abd7e23fc5587eab5a5f94eee116c35a02a6968b28ff7845a9d8b0d
\ No newline at end of file diff --git a/tests/cachekey/project/sources/bzr1.expected b/tests/cachekey/project/sources/bzr1.expected index 253ebc601..069881cfb 100644 --- a/tests/cachekey/project/sources/bzr1.expected +++ b/tests/cachekey/project/sources/bzr1.expected @@ -1 +1 @@ -673bb938cc3fabe0be55e98c6b8b80853168becc86b4fa102fc0c538879bf83a +cb1e72ea1b5d842010d056498a86bc3c3302a1b9919f7285da9329fffe556187
\ No newline at end of file diff --git a/tests/cachekey/project/sources/git1.expected b/tests/cachekey/project/sources/git1.expected index c8e11137a..482ed50c9 100644 --- a/tests/cachekey/project/sources/git1.expected +++ b/tests/cachekey/project/sources/git1.expected @@ -1 +1 @@ -53a367133fb8f3ca86ba772801ea62681414271da9582800dd56a62e9c6d7e5d +b61a46d4effd5501f0553b1eedfd24220912102370e9be7b489159cd1e02d47f
\ No newline at end of file diff --git a/tests/cachekey/project/sources/git2.expected b/tests/cachekey/project/sources/git2.expected index 2e320c5ff..1591ef3c5 100644 --- a/tests/cachekey/project/sources/git2.expected +++ b/tests/cachekey/project/sources/git2.expected @@ -1 +1 @@ -86e0cf4f3154fa006899acf64317930ebf08ca6d01113abfa35ccceed2961fcd +878922a1112f47701bf05a9ee752b331463fca98580e88dcac5c4d90897a181a
\ No newline at end of file diff --git a/tests/cachekey/project/sources/git3.expected b/tests/cachekey/project/sources/git3.expected index fd1b1d8d0..dbe658b90 100644 --- a/tests/cachekey/project/sources/git3.expected +++ b/tests/cachekey/project/sources/git3.expected @@ -1 +1 @@ -ad5dff8a422c9de7c3d02773aeed7b425d43501ee5c2d5d13064b6f1e1ed9dec +4363cd830fd28b8b20354b8e2643e2f55e99afc0328f26ae3af8a049b5fdafb2
\ No newline at end of file diff --git a/tests/cachekey/project/sources/local1.expected b/tests/cachekey/project/sources/local1.expected index 78f4972fa..ad2e9a9d8 100644 --- a/tests/cachekey/project/sources/local1.expected +++ b/tests/cachekey/project/sources/local1.expected @@ -1 +1 @@ -7596dd8b5e930d95b0d93f8aef1fb41b4ee0ec952dccb5cd1a4364cd6d63dd41
\ No newline at end of file +8f05d68de71fbd70661eb4a3516189d754c9ee76af42eff93ed9cde6c441e257
\ No newline at end of file diff --git a/tests/cachekey/project/sources/local2.expected b/tests/cachekey/project/sources/local2.expected index e32afb042..713581851 100644 --- a/tests/cachekey/project/sources/local2.expected +++ b/tests/cachekey/project/sources/local2.expected @@ -1 +1 @@ -427439145ad4a9c3b839f6db8fc81d693d0163da5f3e48f36e6f1e39fc5d816b
\ No newline at end of file +4c2f914ab71bae44c21c72768bed96a7a27bd4426ac651093ad07566e948a62d
\ No newline at end of file diff --git a/tests/cachekey/project/sources/patch1.expected b/tests/cachekey/project/sources/patch1.expected index f4a5b6318..d13e6c13f 100644 --- a/tests/cachekey/project/sources/patch1.expected +++ b/tests/cachekey/project/sources/patch1.expected @@ -1 +1 @@ -4d2d8e8e92a20255a38d167abd93b5e6843f7b2738cdfe11ce64bc662fcaa886 +2bfdd9b55896b4287bac41cc57974eb266bacd21dc1e8a7eb78417cdaf17dd3d
\ No newline at end of file diff --git a/tests/cachekey/project/sources/patch2.expected b/tests/cachekey/project/sources/patch2.expected index 99a1923a2..8826ab3f5 100644 --- a/tests/cachekey/project/sources/patch2.expected +++ b/tests/cachekey/project/sources/patch2.expected @@ -1 +1 @@ -21a53c232671f21cd717a4c94274e2decdba2c916dde56e030f944fe92ae785e +666d87ff1fcf238c1cc621d850be8d7b955990579c63c18797fb022ea46e0150
\ No newline at end of file diff --git a/tests/cachekey/project/sources/patch3.expected b/tests/cachekey/project/sources/patch3.expected index b44d06530..83f4292f9 100644 --- a/tests/cachekey/project/sources/patch3.expected +++ b/tests/cachekey/project/sources/patch3.expected @@ -1 +1 @@ -5ec3023b14bb2a44c94e205d4edc0e366d187357d6661bbc699f73e014b0630b +9bb2454bd6a8a2af627d304045d48d5587cca32cd802216b8b67c83c7da049b5
\ No newline at end of file diff --git a/tests/cachekey/project/sources/pip1.expected b/tests/cachekey/project/sources/pip1.expected index e5ba434ab..d09d8d6aa 100644 --- a/tests/cachekey/project/sources/pip1.expected +++ b/tests/cachekey/project/sources/pip1.expected @@ -1 +1 @@ -60d6200ba331e3cff4b3255cb218569e387c571ee57761f6b3883b1283a937a2 +877660711746453066c7a780b23aeee81961a510eb6405d55d24d9880e18d277
\ No newline at end of file diff --git a/tests/cachekey/project/sources/remote1.expected b/tests/cachekey/project/sources/remote1.expected index 2f42a3356..e25a282b8 100644 --- a/tests/cachekey/project/sources/remote1.expected +++ b/tests/cachekey/project/sources/remote1.expected @@ -1 +1 @@ -46f09da7ea078bf0d630ec2e14a668f8144df5175ee1c19c9af367873047b482 +ddeb5f82ef95533d863883546d1579e1cd3882459aec8e0bb8e9948143204d00
\ No newline at end of file diff --git a/tests/cachekey/project/sources/remote2.expected b/tests/cachekey/project/sources/remote2.expected index 1dbaf7739..5c68f919c 100644 --- a/tests/cachekey/project/sources/remote2.expected +++ b/tests/cachekey/project/sources/remote2.expected @@ -1 +1 @@ -aaa2d0c22b40d2f9b87d40ff24c37769240edba4902c50fa948e8ab6c9848f6f +fb7a50cf67554963126b698ce0f9c175c8992472b3d1e34ed27b10e6db2b465b
\ No newline at end of file diff --git a/tests/cachekey/project/sources/tar1.expected b/tests/cachekey/project/sources/tar1.expected index 153138437..0ea93ca03 100644 --- a/tests/cachekey/project/sources/tar1.expected +++ b/tests/cachekey/project/sources/tar1.expected @@ -1 +1 @@ -6b550e20ab7b8a11912ca14171e39c76badf7fa161a01c83d817c789b84e45c3 +4609c3c0f007ac68d801e2893cd8ccce57b4b374fe800c8eef26d5dc95993d0d
\ No newline at end of file diff --git a/tests/cachekey/project/sources/tar2.expected b/tests/cachekey/project/sources/tar2.expected index 6440dfaed..84ffefa8b 100644 --- a/tests/cachekey/project/sources/tar2.expected +++ b/tests/cachekey/project/sources/tar2.expected @@ -1 +1 @@ -f890b611cc83036b9c52dddf4eb2a02ccac5a73ae3ddcb34586406d7deba5a11 +1b5a123bc483b3350c1e26573a70311956c7c2ccadb14ddc758b3329020c3ee0
\ No newline at end of file diff --git a/tests/cachekey/project/sources/zip1.expected b/tests/cachekey/project/sources/zip1.expected index 1207db1a7..ecb6c5167 100644 --- a/tests/cachekey/project/sources/zip1.expected +++ b/tests/cachekey/project/sources/zip1.expected @@ -1 +1 @@ -5393d513abcc88bd1cdbf03cff65d470285a906a43cf2e192ce0770fbceb933d +45f3248e0d4d4c8872927d6a76f730afce3655b5204b484a10eedaa3a39cdd4a
\ No newline at end of file diff --git a/tests/cachekey/project/sources/zip2.expected b/tests/cachekey/project/sources/zip2.expected index 4fa2e563d..68ffc9fc3 100644 --- a/tests/cachekey/project/sources/zip2.expected +++ b/tests/cachekey/project/sources/zip2.expected @@ -1 +1 @@ -a03196c4878e0a585c54c0e75cabe069068d5e37b49f07ca95f5aeb6e3b1cf5b +9c041e467499a1849f02e78cec5af73c5d8e8440bdab3abba131fa620379239b
\ No newline at end of file diff --git a/tests/cachekey/project/target.expected b/tests/cachekey/project/target.expected index 17d7376dc..cf0e262ae 100644 --- a/tests/cachekey/project/target.expected +++ b/tests/cachekey/project/target.expected @@ -1 +1 @@ -9fba84d9e7fbbb03121e4bc24fb58dd9a101c88abdb0437c82402e342d46b88f
\ No newline at end of file +bf4f53d434bf0c140007a270c9787fe3b67ce3e09dd2fe168de742990d900e01
\ No newline at end of file |