summaryrefslogtreecommitdiff
diff options
context:
space:
mode:
authorAngelos Evripiotis <jevripiotis@bloomberg.net>2019-02-26 15:33:44 +0000
committerAngelos Evripiotis <jevripiotis@bloomberg.net>2019-02-26 17:51:53 +0000
commitfe76ac9780a0951b9d85100bbd0c06d997a8e0ad (patch)
treea34a7df7f526533182b9a58f3e95340ba57394d8
parent5a1a5814e77152bf2195f917147d5dc46f7201d7 (diff)
downloadbuildstream-aevri/json_artifact_meta.tar.gz
element: use json for artifact metadata for speedaevri/json_artifact_meta
Change public.yaml files in artifacts to JSON instead of YAML. Also bump the BST_CORE_ARTIFACT_VERSION, as we don't provide compatability with artifacts created by an older version of BuildStream. Note that we could have made a similar change that would not have required a version bump - JSON is a subset of YAML. We deliberately choose not to in order to force folks to take the feature of increased speed for the next major version of BuildStream. This means that the files might be less human-readable, but they are faster and simpler for machines to read. This 'YAML for human readability, JSON for simplicity' split is observed by the authors of the YAML spec: https://yaml.org/spec/1.2/spec.html#id2759572 This change saves 3-4 seconds off a 12-13 second invocation of bst shell like this: time bst shell --build --use-buildtree never target.bst echo hello After the change, parsing artifact info doesn't take long enough to really show up in a py-spy profile of the invocation. My use-case has around 400 elements in it's build plan, as calculated by: bst show --deps build --format BUILDDEP target.bst | grep BUILDDEP | wc -l
-rw-r--r--NEWS3
-rw-r--r--buildstream/_versions.py2
-rw-r--r--buildstream/element.py8
3 files changed, 10 insertions, 3 deletions
diff --git a/NEWS b/NEWS
index 0ff7702aa..968f06287 100644
--- a/NEWS
+++ b/NEWS
@@ -11,6 +11,9 @@ buildstream 1.3.1
to this, `--tar` is no longer a flag, it is a mutually incompatible option
to `--directory`. For example, `bst artifact checkout foo.bst --tar foo.tar.gz`.
+ o The core artifact version was increased, due to switching the format of
+ artifact metadata from YAML to JSON. This was for greatly improved loading
+ speed.
o Added `bst artifact log` subcommand for viewing build logs.
diff --git a/buildstream/_versions.py b/buildstream/_versions.py
index 56fd95223..6b35b924f 100644
--- a/buildstream/_versions.py
+++ b/buildstream/_versions.py
@@ -33,4 +33,4 @@ BST_FORMAT_VERSION = 23
# or if buildstream was changed in a way which can cause
# the same cache key to produce something that is no longer
# the same.
-BST_CORE_ARTIFACT_VERSION = 8
+BST_CORE_ARTIFACT_VERSION = 9
diff --git a/buildstream/element.py b/buildstream/element.py
index 365931e27..80217858f 100644
--- a/buildstream/element.py
+++ b/buildstream/element.py
@@ -85,6 +85,8 @@ import tempfile
import shutil
import string
+import ujson
+
from . import _yaml
from ._variables import Variables
from ._versions import BST_CORE_ARTIFACT_VERSION
@@ -1738,7 +1740,8 @@ class Element(Plugin):
shutil.copyfile(log_filename, self._build_log_path)
# Store public data
- _yaml.dump(_yaml.node_sanitize(self.__dynamic_public), os.path.join(metadir, 'public.yaml'))
+ with open(os.path.join(metadir, 'public.json'), 'w') as datafile:
+ ujson.dump(_yaml.node_sanitize(self.__dynamic_public), datafile)
# Store result
build_result_dict = {"success": self.__build_result[0], "description": self.__build_result[1]}
@@ -2834,7 +2837,8 @@ class Element(Plugin):
# Load the public data from the artifact
artifact_base, _ = self.__extract()
metadir = os.path.join(artifact_base, 'meta')
- self.__dynamic_public = _yaml.load(os.path.join(metadir, 'public.yaml'))
+ with open(os.path.join(metadir, 'public.json')) as datafile:
+ self.__dynamic_public = ujson.load(datafile)
def __load_build_result(self, keystrength):
self.__assert_cached(keystrength=keystrength)