Merge branch 'tristan/bst-1/json-cache-keys' into 'bst-1'

Backport json cache keys (bst-1)

See merge request BuildStream/buildstream!1518

34 files changed, 38 insertions, 30 deletions

diff --git a/NEWS b/NEWS
index 6227faa17..49901731b 100644
--- a/NEWS
+++ b/NEWS
@@ -30,6 +30,8 @@ buildstream 1.3.2
                                declaration but does not exist at the
 			       given ref in the git repository.
 
+  o BuildStream now depends on python3 ujson (for some internal serializations)
+
 =================
 buildstream 1.3.1
 =================
diff --git a/buildstream/_cachekey.py b/buildstream/_cachekey.py
index fe407e96f..953263aa9 100644
--- a/buildstream/_cachekey.py
+++ b/buildstream/_cachekey.py
@@ -19,7 +19,8 @@
 
 
 import hashlib
-import pickle
+
+import ujson
 
 from . import _yaml
 
@@ -38,5 +39,5 @@ from . import _yaml
 #
 def generate_key(value):
     ordered = _yaml.node_sanitize(value)
-    string = pickle.dumps(ordered)
-    return hashlib.sha256(string).hexdigest()
+    ustring = ujson.dumps(ordered, sort_keys=True, escape_forward_slashes=False).encode('utf-8')
+    return hashlib.sha256(ustring).hexdigest()
diff --git a/buildstream/_versions.py b/buildstream/_versions.py
index e55296513..383429902 100644
--- a/buildstream/_versions.py
+++ b/buildstream/_versions.py
@@ -34,4 +34,4 @@ BST_FORMAT_VERSION = 17
 # the same cache key to produce something that is no longer
 # the same.
 
-BST_CORE_ARTIFACT_VERSION = ('bst-1.2', 4)
+BST_CORE_ARTIFACT_VERSION = ('bst-1.2', 5)
diff --git a/requirements/requirements.in b/requirements/requirements.in
index 1743e1c3f..537031128 100644
--- a/requirements/requirements.in
+++ b/requirements/requirements.in
@@ -6,3 +6,4 @@ protobuf >= 3.5
 psutil
 ruamel.yaml < 0.15.52
 setuptools
+ujson
diff --git a/requirements/requirements.txt b/requirements/requirements.txt
index 96645cb4b..99c8bffbc 100644
--- a/requirements/requirements.txt
+++ b/requirements/requirements.txt
@@ -5,6 +5,7 @@ protobuf==3.6.1
 psutil==5.4.8
 ruamel.yaml==0.15.51
 setuptools==40.6.2
+ujson==1.35
 ## The following requirements were added by pip freeze:
 Jinja2==2.10
 MarkupSafe==1.1.0
diff --git a/tests/cachekey/project/elements/build1.expected b/tests/cachekey/project/elements/build1.expected
index 70d29bffd..234e58ca4 100644
--- a/tests/cachekey/project/elements/build1.expected
+++ b/tests/cachekey/project/elements/build1.expected
@@ -1 +1 @@
-9195e7c5f1326b878275c3953e39f87fa9909b9eecfe1b07cb23dcbf61e52a70
\ No newline at end of file
+36d193b074051c3315afc943cdf4cfb835becac3ea9f18cd1458130bb39e320c
\ No newline at end of file
diff --git a/tests/cachekey/project/elements/build2.expected b/tests/cachekey/project/elements/build2.expected
index 2401ea01a..3bdc8d831 100644
--- a/tests/cachekey/project/elements/build2.expected
+++ b/tests/cachekey/project/elements/build2.expected
@@ -1 +1 @@
-beb17868e92bdf5203d6a9263cffc91c3252f329116afc04855fe8ef529d77c4
\ No newline at end of file
+7104190005529a3fe8a1ae6beb794b6a371312b915b35bec360af40a52bd3778
\ No newline at end of file
diff --git a/tests/cachekey/project/elements/compose1.expected b/tests/cachekey/project/elements/compose1.expected
index 71c199104..3e2b1e635 100644
--- a/tests/cachekey/project/elements/compose1.expected
+++ b/tests/cachekey/project/elements/compose1.expected
@@ -1 +1 @@
-ef47f8a74e6c96f3d8386ee9b4e4999b91e3fa44a063ee480505c4ae3b9af55a
\ No newline at end of file
+2d251a155b377f20923dc66092dd7d942f1a0343a063d86341f8e761caa50bdb
\ No newline at end of file
diff --git a/tests/cachekey/project/elements/compose2.expected b/tests/cachekey/project/elements/compose2.expected
index 08af0b347..493ea68c8 100644
--- a/tests/cachekey/project/elements/compose2.expected
+++ b/tests/cachekey/project/elements/compose2.expected
@@ -1 +1 @@
-739450b3cd7b6cd870ef136ef94880f824f381834b1d5a8c7f616a6590eb34ef
\ No newline at end of file
+7dca9347141d00754c0cc2bbe67695062331002a6a21cd9969435fb70a45600a
\ No newline at end of file
diff --git a/tests/cachekey/project/elements/compose3.expected b/tests/cachekey/project/elements/compose3.expected
index e77e7af48..d93e53d00 100644
--- a/tests/cachekey/project/elements/compose3.expected
+++ b/tests/cachekey/project/elements/compose3.expected
@@ -1 +1 @@
-c250ea677c5be87114ae1199ebd3ee5f0ceefc6e417b09c8be94448a942a15ac
\ No newline at end of file
+b7c821b8fd006069ae2b55b855be9dde6f3f3ce141dd546a2bce64b0bde9e93a
\ No newline at end of file
diff --git a/tests/cachekey/project/elements/compose4.expected b/tests/cachekey/project/elements/compose4.expected
index d656e6e20..8461a7438 100644
--- a/tests/cachekey/project/elements/compose4.expected
+++ b/tests/cachekey/project/elements/compose4.expected
@@ -1 +1 @@
-729f40643a6b14b08ed70108ccd9efeb2024ccf4fcd7102996857bc4a75c8eb7
\ No newline at end of file
+7df77bfd64828e4d3a6f08eaf5760f85526f6c1adee82b61701f26c4a3432a86
\ No newline at end of file
diff --git a/tests/cachekey/project/elements/compose5.expected b/tests/cachekey/project/elements/compose5.expected
index d9c69186a..8875ad638 100644
--- a/tests/cachekey/project/elements/compose5.expected
+++ b/tests/cachekey/project/elements/compose5.expected
@@ -1 +1 @@
-7e59a1292236d81ac1b79d5d8f188c1457488608944842e45b0c2f813bf08fee
\ No newline at end of file
+504fe6417dc90a3c3dc7d809745b1f7a3cc2f3c3a1c2408408d871be027d49c7
\ No newline at end of file
diff --git a/tests/cachekey/project/elements/import1.expected b/tests/cachekey/project/elements/import1.expected
index fdb2911e5..936764243 100644
--- a/tests/cachekey/project/elements/import1.expected
+++ b/tests/cachekey/project/elements/import1.expected
@@ -1 +1 @@
-92a12959aba0d4123f383e8d285e1989877f04074d248b3d5bae801d14801fa0
\ No newline at end of file
+f915b632d26b2e2ccc339677635a9783d0f00dd6e3cdc2fee08cc7b3e649a3a0
\ No newline at end of file
diff --git a/tests/cachekey/project/elements/import2.expected b/tests/cachekey/project/elements/import2.expected
index c7c325048..99392fcec 100644
--- a/tests/cachekey/project/elements/import2.expected
+++ b/tests/cachekey/project/elements/import2.expected
@@ -1 +1 @@
-f3d07c82b0bea080dc1783a07234910129cc4f95edab6868482395d77503e445
\ No newline at end of file
+58f07139c4b6c814f04131b5d0f1dc7ebfc5c9db8669be59ee7c58e3b8fe8111
\ No newline at end of file
diff --git a/tests/cachekey/project/elements/import3.expected b/tests/cachekey/project/elements/import3.expected
index 4b943859d..9ac178a8b 100644
--- a/tests/cachekey/project/elements/import3.expected
+++ b/tests/cachekey/project/elements/import3.expected
@@ -1 +1 @@
-647fe25d06131191ff1b2898f6c57f88f206f6e11297fa25f30610b77fc22e6e
\ No newline at end of file
+76c6de73910086f2697ea748757a4f5e67c6e069131f28e3e54c380a7d6e8e18
\ No newline at end of file
diff --git a/tests/cachekey/project/elements/script1.expected b/tests/cachekey/project/elements/script1.expected
index f7951113c..311b7b38d 100644
--- a/tests/cachekey/project/elements/script1.expected
+++ b/tests/cachekey/project/elements/script1.expected
@@ -1 +1 @@
-f9715017e60fe29557e37402e70eed9119f84324f880e5bf6f25bc78a4113647
\ No newline at end of file
+7e4e7e8ec36d9b661b357637cb4d2fb9adc37481764d645382d6b0ccb6ad7c71
\ No newline at end of file
diff --git a/tests/cachekey/project/sources/bzr1.expected b/tests/cachekey/project/sources/bzr1.expected
index 25cc7c99b..463dfb0a8 100644
--- a/tests/cachekey/project/sources/bzr1.expected
+++ b/tests/cachekey/project/sources/bzr1.expected
@@ -1 +1 @@
-edf2530d3395461ad3e6802944bb023699e0b0a2afb0d4b65f5f35c3c5936154
\ No newline at end of file
+410a5606ccc28108db7bb9a57e017ede7435dbad27a8006e1c791b3253e9a56f
\ No newline at end of file
diff --git a/tests/cachekey/project/sources/git1.expected b/tests/cachekey/project/sources/git1.expected
index 9d6f05770..e75d79cb2 100644
--- a/tests/cachekey/project/sources/git1.expected
+++ b/tests/cachekey/project/sources/git1.expected
@@ -1 +1 @@
-74bae7aaf8af955c0a93148830dee097382d806e6aba3a30bdfec62617661b46
\ No newline at end of file
+3dafe98cceea0434cf8a0317db110ce8d7715cf55148f7d01db62c87513f22a5
\ No newline at end of file
diff --git a/tests/cachekey/project/sources/git2.expected b/tests/cachekey/project/sources/git2.expected
index 24f95c7a8..65110ea18 100644
--- a/tests/cachekey/project/sources/git2.expected
+++ b/tests/cachekey/project/sources/git2.expected
@@ -1 +1 @@
-46fae4a8d6195aa6680d82824167eaa453f8945625869fb4aa0bb774bdbe8bf5
\ No newline at end of file
+5170e2d5e00d5c82089e345b496a302ab3f718422557ef993ab6aa21ca6957f6
\ No newline at end of file
diff --git a/tests/cachekey/project/sources/git3.expected b/tests/cachekey/project/sources/git3.expected
new file mode 100644
index 000000000..2c6477e57
--- /dev/null
+++ b/tests/cachekey/project/sources/git3.expected
@@ -0,0 +1 @@
+f2c3ab6ee644ba9507f3ddd33880fa78f92c80ff872bc1ad676e0a5c40ff3493
\ No newline at end of file
diff --git a/tests/cachekey/project/sources/local1.expected b/tests/cachekey/project/sources/local1.expected
index fdb2911e5..936764243 100644
--- a/tests/cachekey/project/sources/local1.expected
+++ b/tests/cachekey/project/sources/local1.expected
@@ -1 +1 @@
-92a12959aba0d4123f383e8d285e1989877f04074d248b3d5bae801d14801fa0
\ No newline at end of file
+f915b632d26b2e2ccc339677635a9783d0f00dd6e3cdc2fee08cc7b3e649a3a0
\ No newline at end of file
diff --git a/tests/cachekey/project/sources/local2.expected b/tests/cachekey/project/sources/local2.expected
index 69215eceb..95ed4400c 100644
--- a/tests/cachekey/project/sources/local2.expected
+++ b/tests/cachekey/project/sources/local2.expected
@@ -1 +1 @@
-68b17676ea7156e3c267d07a93041f375c868c10015e8742de365dbe561d810e
\ No newline at end of file
+33239f183d3cbaca957e9693637fca6cb91053d1c22f5c4dea577b50d0acc23d
\ No newline at end of file
diff --git a/tests/cachekey/project/sources/ostree1.expected b/tests/cachekey/project/sources/ostree1.expected
index eb4c8ddd5..88722b5df 100644
--- a/tests/cachekey/project/sources/ostree1.expected
+++ b/tests/cachekey/project/sources/ostree1.expected
@@ -1 +1 @@
-f11dc9d54b07608742a6204adbf6dfa74f737b023c4519cc301b5790e3dcf16c
\ No newline at end of file
+8bccdd60fc3a89eb9ed807dd31e868ea48c98411c564ca4aefc354b9956675de
\ No newline at end of file
diff --git a/tests/cachekey/project/sources/patch1.expected b/tests/cachekey/project/sources/patch1.expected
index 3e00ad4e3..8e20f4761 100644
--- a/tests/cachekey/project/sources/patch1.expected
+++ b/tests/cachekey/project/sources/patch1.expected
@@ -1 +1 @@
-5982cee04b8a10d4fec13fc6463b8be664323c1c44b802146f483f9dd53ca5f6
\ No newline at end of file
+4408a9652884249b5b709d8defd726d3f5aa3279418fd303180bc12d084172ef
\ No newline at end of file
diff --git a/tests/cachekey/project/sources/patch2.expected b/tests/cachekey/project/sources/patch2.expected
index 256470fd6..707fb8249 100644
--- a/tests/cachekey/project/sources/patch2.expected
+++ b/tests/cachekey/project/sources/patch2.expected
@@ -1 +1 @@
-95188e54e87a869450049f5b7e865e3f3762d4d387b930c39199ef3b7864fad8
\ No newline at end of file
+af4213e2a92ed6e447f7a932fffc36ace134c03ee6ddb2eca12f5feb194cc625
\ No newline at end of file
diff --git a/tests/cachekey/project/sources/patch3.expected b/tests/cachekey/project/sources/patch3.expected
index 351f35ec8..0f2fd08c7 100644
--- a/tests/cachekey/project/sources/patch3.expected
+++ b/tests/cachekey/project/sources/patch3.expected
@@ -1 +1 @@
-43c215e5be4402eff02f23cd5d1b83659a942b17a56a4c373c36333fcc07fd76
\ No newline at end of file
+f40eb8bab43e2ff8deeb6b5d360d911506be914ef302d8a1de5a431f3d32c222
\ No newline at end of file
diff --git a/tests/cachekey/project/sources/pip1.expected b/tests/cachekey/project/sources/pip1.expected
index 7f0cd5d31..0d4305821 100644
--- a/tests/cachekey/project/sources/pip1.expected
+++ b/tests/cachekey/project/sources/pip1.expected
@@ -1 +1 @@
-77241a36a6dc2b89b99e8e8dc52dd057f6f78319a7ff913291aa44419b6afab9
\ No newline at end of file
+492115a2d3a54eb5fd4a301f1cdfd328b2d3dd918e709386cea8133f8cac53eb
\ No newline at end of file
diff --git a/tests/cachekey/project/sources/remote1.expected b/tests/cachekey/project/sources/remote1.expected
new file mode 100644
index 000000000..a86a0f419
--- /dev/null
+++ b/tests/cachekey/project/sources/remote1.expected
@@ -0,0 +1 @@
+009d779779361f1546d808945aaa05765e7d17b6b8b7a383d658ceaeddcff0db
\ No newline at end of file
diff --git a/tests/cachekey/project/sources/remote2.expected b/tests/cachekey/project/sources/remote2.expected
new file mode 100644
index 000000000..43b5ce0c8
--- /dev/null
+++ b/tests/cachekey/project/sources/remote2.expected
@@ -0,0 +1 @@
+d4f791a9185a4f2c18c3046c41efb26c0a833b73fc52abb3cddd4fe3ac130196
\ No newline at end of file
diff --git a/tests/cachekey/project/sources/tar1.expected b/tests/cachekey/project/sources/tar1.expected
index 1c892d4a7..3af3cd8e1 100644
--- a/tests/cachekey/project/sources/tar1.expected
+++ b/tests/cachekey/project/sources/tar1.expected
@@ -1 +1 @@
-2a75001c338b720fac8d5d30933c9ffa14f7e9ef1ebd22c9cf67b6dfc7e94dae
\ No newline at end of file
+f2360a3551f3cc5293cf9939773102bca6b6d0aa261bff78ef5f69aac3265afb
\ No newline at end of file
diff --git a/tests/cachekey/project/sources/tar2.expected b/tests/cachekey/project/sources/tar2.expected
index 4eca37ece..9cef3affa 100644
--- a/tests/cachekey/project/sources/tar2.expected
+++ b/tests/cachekey/project/sources/tar2.expected
@@ -1 +1 @@
-0056ada224a0e4c25be48c2b0940b7431af8c79b56cbd5556b7e96c9840060e2
\ No newline at end of file
+15abf0e81387289c4c1635b386afcb162a3672ed2f757266cc6a6273170ac608
\ No newline at end of file
diff --git a/tests/cachekey/project/sources/zip1.expected b/tests/cachekey/project/sources/zip1.expected
index 101b3f704..9b482f466 100644
--- a/tests/cachekey/project/sources/zip1.expected
+++ b/tests/cachekey/project/sources/zip1.expected
@@ -1 +1 @@
-74daf0d1bf5cd07ba420fc059addf2b6e3cb58596b61ebead5ec15830ec8dc63
\ No newline at end of file
+5a6ac152f8bd88add55172dca76225c1cf01c821e1037184ff54f06baf7fb028
\ No newline at end of file
diff --git a/tests/cachekey/project/sources/zip2.expected b/tests/cachekey/project/sources/zip2.expected
index 5bcc7f2b8..4ca71e0d0 100644
--- a/tests/cachekey/project/sources/zip2.expected
+++ b/tests/cachekey/project/sources/zip2.expected
@@ -1 +1 @@
-7ebc6f246a7cba793116c7c1ebeba02eab3abadd527f6b3a239f025380f3682b
\ No newline at end of file
+0016e926397c238c574679cfa4c21b44c4c56cbd1328e92327d088fa319eb206
\ No newline at end of file
diff --git a/tests/cachekey/project/target.expected b/tests/cachekey/project/target.expected
index acf8ab631..b7eb7ea6f 100644
--- a/tests/cachekey/project/target.expected
+++ b/tests/cachekey/project/target.expected
@@ -1 +1 @@
-1d3465903d45365032bff340ec8a8bf0e7d69f4332bb3c525d228a04009dcff7
\ No newline at end of file
+06c4372d43e3ac92c5948af6b24b4451a9411a9d6b35b3ea8fc58ce139801c0d
\ No newline at end of file