Cache Keys: Update to use JSON rather than pickledanielsilverstone-ct/json-cache-key

This affects the cache key version (updated to 7) and introduces
a dependency on `ujson` which is BSD licenced as of the version
locked in `requirements.txt`

Signed-off-by: Daniel Silverstone <daniel.silverstone@codethink.co.uk>

33 files changed, 36 insertions, 33 deletions

diff --git a/buildstream/_cachekey.py b/buildstream/_cachekey.py
index fe407e96f..953263aa9 100644
--- a/buildstream/_cachekey.py
+++ b/buildstream/_cachekey.py
@@ -19,7 +19,8 @@
 
 
 import hashlib
-import pickle
+
+import ujson
 
 from . import _yaml
 
@@ -38,5 +39,5 @@ from . import _yaml
 #
 def generate_key(value):
     ordered = _yaml.node_sanitize(value)
-    string = pickle.dumps(ordered)
-    return hashlib.sha256(string).hexdigest()
+    ustring = ujson.dumps(ordered, sort_keys=True, escape_forward_slashes=False).encode('utf-8')
+    return hashlib.sha256(ustring).hexdigest()
diff --git a/buildstream/_versions.py b/buildstream/_versions.py
index 2563fa504..f24a74714 100644
--- a/buildstream/_versions.py
+++ b/buildstream/_versions.py
@@ -33,4 +33,4 @@ BST_FORMAT_VERSION = 21
 # or if buildstream was changed in a way which can cause
 # the same cache key to produce something that is no longer
 # the same.
-BST_CORE_ARTIFACT_VERSION = 6
+BST_CORE_ARTIFACT_VERSION = 7
diff --git a/requirements/requirements.in b/requirements/requirements.in
index 45e7b8075..4f08969bb 100644
--- a/requirements/requirements.in
+++ b/requirements/requirements.in
@@ -14,3 +14,4 @@ psutil
 ruamel.yaml >= 0.15.41, < 0.15.52
 setuptools
 pyroaring
+ujson
diff --git a/requirements/requirements.txt b/requirements/requirements.txt
index 115879ad9..d46f92553 100644
--- a/requirements/requirements.txt
+++ b/requirements/requirements.txt
@@ -14,6 +14,7 @@ psutil==5.4.8
 ruamel.yaml==0.15.51
 setuptools==39.0.1
 pyroaring==0.2.6
+ujson==1.35
 ## The following requirements were added by pip freeze:
 MarkupSafe==1.1.0
 six==1.12.0
diff --git a/tests/cachekey/project/elements/build1.expected b/tests/cachekey/project/elements/build1.expected
index c2982f939..901f6acf7 100644
--- a/tests/cachekey/project/elements/build1.expected
+++ b/tests/cachekey/project/elements/build1.expected
@@ -1 +1 @@
-298bb7caab56c60bc12f59df15db2b1107a604fdb0ce3fa5729aa23d6d37f6e4
\ No newline at end of file
+0eddcb5c12b2bdb4ce83bef7b997f25c89a9396c6321fe471b31dabba8eba928
\ No newline at end of file
diff --git a/tests/cachekey/project/elements/build2.expected b/tests/cachekey/project/elements/build2.expected
index bc663989f..cfff92ad0 100644
--- a/tests/cachekey/project/elements/build2.expected
+++ b/tests/cachekey/project/elements/build2.expected
@@ -1 +1 @@
-c745f07458a2488180ae7f5b0601a38853dcdfd2b6771c18ecbb622f1af10cf1
\ No newline at end of file
+d54339437d5495f573a01434c61b8f0496cef0f833566b9c9f0a6cb362075edb
\ No newline at end of file
diff --git a/tests/cachekey/project/elements/compose1.expected b/tests/cachekey/project/elements/compose1.expected
index 9abf5b751..d8f256190 100644
--- a/tests/cachekey/project/elements/compose1.expected
+++ b/tests/cachekey/project/elements/compose1.expected
@@ -1 +1 @@
-d759c8b7262c4782c12b425bfca0bc8f4c30656d91c98e309c10f11d4179b101
\ No newline at end of file
+d14a3d59da4fcb18c81f0cd6b5e893dc5f25e1d5eecc1a0845fb9b2ef499d878
\ No newline at end of file
diff --git a/tests/cachekey/project/elements/compose2.expected b/tests/cachekey/project/elements/compose2.expected
index 4c3881b8e..f7de6d183 100644
--- a/tests/cachekey/project/elements/compose2.expected
+++ b/tests/cachekey/project/elements/compose2.expected
@@ -1 +1 @@
-01e22d37b82c9b35c35d3d8485e05fe6de526d3af00137bbff51a309b1fa3401
\ No newline at end of file
+29857451bb3797e258911e485f94c7d59e13f239f3ac904e7524c4d0a187e240
\ No newline at end of file
diff --git a/tests/cachekey/project/elements/compose3.expected b/tests/cachekey/project/elements/compose3.expected
index e66de59f5..0cf041baf 100644
--- a/tests/cachekey/project/elements/compose3.expected
+++ b/tests/cachekey/project/elements/compose3.expected
@@ -1 +1 @@
-22b846d1928155b5a21497e8048af2945ee03a826338f91d4f00d74597cc3ef7
\ No newline at end of file
+2dbb668ebc50445dc03cd44d6d1110cc14d50093a37ef4a25499055318835174
\ No newline at end of file
diff --git a/tests/cachekey/project/elements/compose4.expected b/tests/cachekey/project/elements/compose4.expected
index abc56f234..db0879543 100644
--- a/tests/cachekey/project/elements/compose4.expected
+++ b/tests/cachekey/project/elements/compose4.expected
@@ -1 +1 @@
-0e00f203ce9bc4163be0ed304fcf2abb6ee39aad4ee9a12d80719708a4327337
\ No newline at end of file
+7bc4f89c2a4c27999626c9c8a28fd2865695588375660236e3aec49e847705c1
\ No newline at end of file
diff --git a/tests/cachekey/project/elements/compose5.expected b/tests/cachekey/project/elements/compose5.expected
index 83345705e..8300b0302 100644
--- a/tests/cachekey/project/elements/compose5.expected
+++ b/tests/cachekey/project/elements/compose5.expected
@@ -1 +1 @@
-3d8dd9fef508c27961edeb21dd469d04e7ea097c86a98e7c3bf725975548748f
\ No newline at end of file
+12753c08f60587f22f4ef877b636f6e237f958233ea9b512471de213c84ff927
\ No newline at end of file
diff --git a/tests/cachekey/project/elements/import1.expected b/tests/cachekey/project/elements/import1.expected
index fd349363e..32f292fd0 100644
--- a/tests/cachekey/project/elements/import1.expected
+++ b/tests/cachekey/project/elements/import1.expected
@@ -1 +1 @@
-02f863aedaf175f4b4bb9979a538bd1fb6eabd0ef9d77a633b141c072d957df5
\ No newline at end of file
+bbe25b17a5190532ab36663eb08d3f855a45e09108a80ef619a1198e9d730dcf
\ No newline at end of file
diff --git a/tests/cachekey/project/elements/import2.expected b/tests/cachekey/project/elements/import2.expected
index e68e2b505..f271519e0 100644
--- a/tests/cachekey/project/elements/import2.expected
+++ b/tests/cachekey/project/elements/import2.expected
@@ -1 +1 @@
-80ee70ad27ee4640400533585d1b2313b1abdb0034f4159dad42f545310d9c24
\ No newline at end of file
+532277cf8cf2b5d609bbd690dd3cf2eb2e1042329cdd4b25f7a03f77209bf316
\ No newline at end of file
diff --git a/tests/cachekey/project/elements/import3.expected b/tests/cachekey/project/elements/import3.expected
index 1cab93718..075d6af90 100644
--- a/tests/cachekey/project/elements/import3.expected
+++ b/tests/cachekey/project/elements/import3.expected
@@ -1 +1 @@
-b30d79b763ba3048945784d06482ee378d2642dff712ecffae737aa6f9391b9f
\ No newline at end of file
+5b5911dad04b5f592cedd29b46aa55b87428c31b2bf6539f9c2b2e51a932adbe
\ No newline at end of file
diff --git a/tests/cachekey/project/elements/script1.expected b/tests/cachekey/project/elements/script1.expected
index 18977ecc0..58f79dde5 100644
--- a/tests/cachekey/project/elements/script1.expected
+++ b/tests/cachekey/project/elements/script1.expected
@@ -1 +1 @@
-fb69c9237156bdeb0f948078f735189e9d8a4c4dca3c67b6f1ed5fa314f88707
\ No newline at end of file
+818bda0985ef569204c392de0ae2e7ebefd8fcac94f0c33226270936978d52ee
\ No newline at end of file
diff --git a/tests/cachekey/project/sources/bzr1.expected b/tests/cachekey/project/sources/bzr1.expected
index 68ef31975..dd527da83 100644
--- a/tests/cachekey/project/sources/bzr1.expected
+++ b/tests/cachekey/project/sources/bzr1.expected
@@ -1 +1 @@
-5a8144078d50ff5507d7260c2aa7c62fc6272554767392cd827277e224385cb9
\ No newline at end of file
+767d481ad88648c7a926190ac6697c2d24c563c5133abab9e6ab16c6156941d3
\ No newline at end of file
diff --git a/tests/cachekey/project/sources/git1.expected b/tests/cachekey/project/sources/git1.expected
index 271faafdf..9a6d7c991 100644
--- a/tests/cachekey/project/sources/git1.expected
+++ b/tests/cachekey/project/sources/git1.expected
@@ -1 +1 @@
-21968271111bb43e57417d24481742c2fe52fddd798af4aacca0f604c54b6b94
\ No newline at end of file
+4ccd8fdfcca973bd9079960f260c22df8bef5df7f598009f0ee896d3b5905b6e
\ No newline at end of file
diff --git a/tests/cachekey/project/sources/git2.expected b/tests/cachekey/project/sources/git2.expected
index 8abecc0c0..54cc4bff3 100644
--- a/tests/cachekey/project/sources/git2.expected
+++ b/tests/cachekey/project/sources/git2.expected
@@ -1 +1 @@
-f3d0b8c1179f7b7a22beb4cef7b33a13af047bd882e7e3c34cea55c85dc0aae4
\ No newline at end of file
+444e3a9bd1f3cd8805370a9e560b5c3720d58e778dc4ae812d9023d3b06d77fc
\ No newline at end of file
diff --git a/tests/cachekey/project/sources/git3.expected b/tests/cachekey/project/sources/git3.expected
index 55b02f242..2c6477e57 100644
--- a/tests/cachekey/project/sources/git3.expected
+++ b/tests/cachekey/project/sources/git3.expected
@@ -1 +1 @@
-d3c796a610f0ce13c7f4dc99ce5d0f0a8674c7bbd8c53de28248711b44d427a1
\ No newline at end of file
+f2c3ab6ee644ba9507f3ddd33880fa78f92c80ff872bc1ad676e0a5c40ff3493
\ No newline at end of file
diff --git a/tests/cachekey/project/sources/local1.expected b/tests/cachekey/project/sources/local1.expected
index fd349363e..32f292fd0 100644
--- a/tests/cachekey/project/sources/local1.expected
+++ b/tests/cachekey/project/sources/local1.expected
@@ -1 +1 @@
-02f863aedaf175f4b4bb9979a538bd1fb6eabd0ef9d77a633b141c072d957df5
\ No newline at end of file
+bbe25b17a5190532ab36663eb08d3f855a45e09108a80ef619a1198e9d730dcf
\ No newline at end of file
diff --git a/tests/cachekey/project/sources/local2.expected b/tests/cachekey/project/sources/local2.expected
index 6aa588c9e..f0e548eef 100644
--- a/tests/cachekey/project/sources/local2.expected
+++ b/tests/cachekey/project/sources/local2.expected
@@ -1 +1 @@
-b9c5019375c715cb6a239714c78a1bd702d4acf81518f0c05da0217a74c365df
\ No newline at end of file
+36e4110f09d9a4ba06fb5c18c8633a7d9a482da28a51e44873624014a2b87438
\ No newline at end of file
diff --git a/tests/cachekey/project/sources/ostree1.expected b/tests/cachekey/project/sources/ostree1.expected
index d3fb12439..de7d92294 100644
--- a/tests/cachekey/project/sources/ostree1.expected
+++ b/tests/cachekey/project/sources/ostree1.expected
@@ -1 +1 @@
-8e9f5115d54df9feff28accf061f18b77458816897fdd841d7399bc0d5375633
\ No newline at end of file
+dfb4df92f5afb7fd4856db0d028ece5fc465040045f6043e8081a63683297c8a
\ No newline at end of file
diff --git a/tests/cachekey/project/sources/patch1.expected b/tests/cachekey/project/sources/patch1.expected
index 85cd460ec..026dbdf55 100644
--- a/tests/cachekey/project/sources/patch1.expected
+++ b/tests/cachekey/project/sources/patch1.expected
@@ -1 +1 @@
-52f6d22e92bb627d48f6957d0c55d57d0da5e228ea0d43d108283425b3cb13f4
\ No newline at end of file
+a85bc578b9c12f0b380e064cbc835d675ac1d0d3c74f821d20c7ace2b802351f
\ No newline at end of file
diff --git a/tests/cachekey/project/sources/patch2.expected b/tests/cachekey/project/sources/patch2.expected
index 54636aa88..9f5b12437 100644
--- a/tests/cachekey/project/sources/patch2.expected
+++ b/tests/cachekey/project/sources/patch2.expected
@@ -1 +1 @@
-1937060b2df8c11f37b031751beacdd28b471324734f372b970d29ba3d605b0d
\ No newline at end of file
+ec64492225ef13d90236385b6d3e19aa127b8db1e33837cce180e7dd1657d95c
\ No newline at end of file
diff --git a/tests/cachekey/project/sources/patch3.expected b/tests/cachekey/project/sources/patch3.expected
index bc3f10148..ca1243717 100644
--- a/tests/cachekey/project/sources/patch3.expected
+++ b/tests/cachekey/project/sources/patch3.expected
@@ -1 +1 @@
-cdccd2228c0329b99d22703aac874b4a3a78753adb759c4fae8add533e933d2f
\ No newline at end of file
+d6eb497e7760976b8e9e41f3f80c0f152361188876662eb703c0ddc4984945a6
\ No newline at end of file
diff --git a/tests/cachekey/project/sources/pip1.expected b/tests/cachekey/project/sources/pip1.expected
index 7fcee4c9f..05b7cda23 100644
--- a/tests/cachekey/project/sources/pip1.expected
+++ b/tests/cachekey/project/sources/pip1.expected
@@ -1 +1 @@
-24dcac92e44ef96217b625941d7571dd03fea4a86f34177e280ba9e3b20dea32
\ No newline at end of file
+ff8b62cdb1eb65e84082e7af4c745a42e0d0259c25ab0101f42f812e873a53a8
\ No newline at end of file
diff --git a/tests/cachekey/project/sources/remote1.expected b/tests/cachekey/project/sources/remote1.expected
index f9ceb1bf3..a86a0f419 100644
--- a/tests/cachekey/project/sources/remote1.expected
+++ b/tests/cachekey/project/sources/remote1.expected
@@ -1 +1 @@
-4eec6ce665ea7c970b4df61d2033dd5a49764e9e97855054c475c066fc29efc7
\ No newline at end of file
+009d779779361f1546d808945aaa05765e7d17b6b8b7a383d658ceaeddcff0db
\ No newline at end of file
diff --git a/tests/cachekey/project/sources/remote2.expected b/tests/cachekey/project/sources/remote2.expected
index cc235357c..43b5ce0c8 100644
--- a/tests/cachekey/project/sources/remote2.expected
+++ b/tests/cachekey/project/sources/remote2.expected
@@ -1 +1 @@
-15f1af6230e981740c7103f7b8327e53a74acbbcd40579fc9990bd39ae8f049a
\ No newline at end of file
+d4f791a9185a4f2c18c3046c41efb26c0a833b73fc52abb3cddd4fe3ac130196
\ No newline at end of file
diff --git a/tests/cachekey/project/sources/tar1.expected b/tests/cachekey/project/sources/tar1.expected
index d61f558d4..8fcec06f6 100644
--- a/tests/cachekey/project/sources/tar1.expected
+++ b/tests/cachekey/project/sources/tar1.expected
@@ -1 +1 @@
-3601b0102ce629bba95734d5136e4589348dde3d74402185777c2aef6ebe9c82
\ No newline at end of file
+81542513d239d19a2b1ba826fb1c650cbc106e9457cbadb0ae49c9af07aeb263
\ No newline at end of file
diff --git a/tests/cachekey/project/sources/tar2.expected b/tests/cachekey/project/sources/tar2.expected
index 40c46cf50..e0c85d6bd 100644
--- a/tests/cachekey/project/sources/tar2.expected
+++ b/tests/cachekey/project/sources/tar2.expected
@@ -1 +1 @@
-9a1230d0f68187d35bd083115e545bc153b2b06879c661ac30a73a03f876ac01
\ No newline at end of file
+29f5688daecb262a4d818ec00a1bfc071fa9e54081aa098f9ef36f7e5dbdb065
\ No newline at end of file
diff --git a/tests/cachekey/project/sources/zip1.expected b/tests/cachekey/project/sources/zip1.expected
index 04b82aadf..2f6a7d72c 100644
--- a/tests/cachekey/project/sources/zip1.expected
+++ b/tests/cachekey/project/sources/zip1.expected
@@ -1 +1 @@
-a00fbe9c48fcf071e2a84fb2e39d7939c13eac0f8b07262fb52b7746f0a6f160
\ No newline at end of file
+0b8b08e15b1d2ac12d7c9c4f4f9977b9c1363e017f938f6a871516f443d985b8
\ No newline at end of file
diff --git a/tests/cachekey/project/sources/zip2.expected b/tests/cachekey/project/sources/zip2.expected
index 4c3e6e71e..59fae2cce 100644
--- a/tests/cachekey/project/sources/zip2.expected
+++ b/tests/cachekey/project/sources/zip2.expected
@@ -1 +1 @@
-fe0e42a62eba9f4f96016eb6b0a3807401e9f9f8ded3c1ec3e500a431de0e2eb
\ No newline at end of file
+943e869041c5a7319d63dacf4d10b717f43863d7754f5417d6f784a2f287d1be
\ No newline at end of file
diff --git a/tests/cachekey/project/target.expected b/tests/cachekey/project/target.expected
index 7d793fade..9be3be5f5 100644
--- a/tests/cachekey/project/target.expected
+++ b/tests/cachekey/project/target.expected
@@ -1 +1 @@
-5accdddcdf28c48173381d9bd5d178dfe9b4b69b9ca456995d0a0d0908654275
\ No newline at end of file
+1b1f2ab00c24e9c1da970922d2f80a4cee71c5e578a0c5c0b2bca24249d5e822
\ No newline at end of file