# # Copyright (C) 2017 Codethink Limited # Copyright (C) 2019 Bloomberg Finance LP # # This program is free software; you can redistribute it and/or # modify it under the terms of the GNU Lesser General Public # License as published by the Free Software Foundation; either # version 2 of the License, or (at your option) any later version. # # This library is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU # Lesser General Public License for more details. # # You should have received a copy of the GNU Lesser General Public # License along with this library. If not, see . # # Authors: # Tristan Van Berkom # James Ennis # Benjamin Schubert import contextlib import cProfile import pstats import os import datetime import time from ._exceptions import ProfileError # Use the topic values here to decide what to profile # by setting them in the BST_PROFILE environment variable. # # Multiple topics can be set with the ':' separator. # # E.g.: # # BST_PROFILE=circ-dep-check:sort-deps bst # # The special 'all' value will enable all profiles. class Topics: CIRCULAR_CHECK = "circ-dep-check" SORT_DEPENDENCIES = "sort-deps" LOAD_CONTEXT = "load-context" LOAD_PROJECT = "load-project" LOAD_PIPELINE = "load-pipeline" LOAD_SELECTION = "load-selection" SCHEDULER = "scheduler" ALL = "all" class _Profile: def __init__(self, key, message): self.profiler = cProfile.Profile() self._additional_pstats_files = [] self.key = key self.message = message self.start_time = time.time() filename_template = os.path.join( os.getcwd(), "profile-{}-{}".format( datetime.datetime.fromtimestamp(self.start_time).strftime("%Y%m%dT%H%M%S"), self.key.replace("/", "-").replace(".", "-"), ), ) self.log_filename = "{}.log".format(filename_template) self.cprofile_filename = "{}.cprofile".format(filename_template) def __enter__(self): self.start() def __exit__(self, _exc_type, _exc_value, traceback): self.stop() self.save() def merge(self, profile): self._additional_pstats_files.append(profile.cprofile_filename) def start(self): self.profiler.enable() def stop(self): self.profiler.disable() def save(self): heading = "\n".join( [ "-" * 64, "Profile for key: {}".format(self.key), "Started at: {}".format(self.start_time), "\n\t{}".format(self.message) if self.message else "", "-" * 64, "", # for a final new line ] ) with open(self.log_filename, "a") as fp: stats = pstats.Stats(self.profiler, *self._additional_pstats_files, stream=fp) # Create the log file fp.write(heading) stats.sort_stats("cumulative") stats.print_stats() # Dump the cprofile stats.dump_stats(self.cprofile_filename) class _Profiler: def __init__(self, settings): self.active_topics = set() self.enabled_topics = set() self._active_profilers = [] self._valid_topics = False if settings: self.enabled_topics = set(settings.split(":")) @contextlib.contextmanager def profile(self, topic, key, message=None): # Check if the user enabled topics are valid # NOTE: This is done in the first PROFILER.profile() call and # not __init__ to ensure we handle the exception. This also means # we cannot test for the exception due to the early instantiation and # how the environment is set in the test invocation. if not self._valid_topics: self._check_valid_topics() if not self._is_profile_enabled(topic): yield return if self._active_profilers: # we are in a nested profiler, stop the parent self._active_profilers[-1].stop() key = "{}-{}".format(topic, key) assert key not in self.active_topics self.active_topics.add(key) profiler = _Profile(key, message) self._active_profilers.append(profiler) with profiler: yield self.active_topics.remove(key) # Remove the last profiler from the list self._active_profilers.pop() if self._active_profilers: # We were in a previous profiler, add the previous results to it # and reenable it. parent_profiler = self._active_profilers[-1] parent_profiler.merge(profiler) parent_profiler.start() def _is_profile_enabled(self, topic): return topic in self.enabled_topics or Topics.ALL in self.enabled_topics def _check_valid_topics(self): non_valid_topics = [topic for topic in self.enabled_topics if topic not in vars(Topics).values()] if non_valid_topics: raise ProfileError("Provided BST_PROFILE topics do not exist: {}".format(", ".join(non_valid_topics))) self._valid_topics = True # Export a profiler to be used by BuildStream PROFILER = _Profiler(os.getenv("BST_PROFILE"))