diff options
author | Pedro Alvarez <pedro.alvarez@codethink.co.uk> | 2016-03-20 21:08:22 +0000 |
---|---|---|
committer | Pedro Alvarez <pedro.alvarez@codethink.co.uk> | 2016-03-27 11:53:02 +0000 |
commit | 1f77de80cdf34ea99d9537bad5c6e7f9fcd4d147 (patch) | |
tree | b4283fccb49f0cddb5a4d9b1bd19203648691fad | |
parent | a086310ae56d11ad6b6a1400e5960a901550f08f (diff) | |
download | morph-1f77de80cdf34ea99d9537bad5c6e7f9fcd4d147.tar.gz |
wip multiple builds
Change-Id: If43345ff48c8b590ca70305e01b3c32be285885b
-rw-r--r-- | gear/client.py | 69 |
1 files changed, 52 insertions, 17 deletions
diff --git a/gear/client.py b/gear/client.py index a7870240..ec756b6d 100644 --- a/gear/client.py +++ b/gear/client.py @@ -20,8 +20,10 @@ logging.basicConfig() gear.Server() class SingleBuildController(): - def __init__(self): + def __init__(self, requests_controller, request_id): self.lock = threading.Lock() + self.request_id = request_id + self.requests_controller = requests_controller self.graph_client = BuildGraphClient(self) self.builder_client = BuilderClient(self) self.cache_client = CacheRequestClient(self) @@ -75,13 +77,25 @@ class SingleBuildController(): def _process_cache_response(self, cache_response): response = json.loads( cache_response) - for key, value in response.iteritems(): - if value: - cache_key, kind, name = key.split('.') - self._mark_artifact_as_built(cache_key, name=name) - else: - print key - print ".. wasnt built" + #for key, value in response.iteritems(): + # if value: + # cache_key, kind, name = key.split('.') + # self._mark_artifact_as_built(cache_key, name=name) + # else: + # print key + # print ".. wasnt built" + + # Mark things as built that are now built. We only check the unbuilt + # artifacts, so 'cache_state' will have no info for things we already + # thought were built. + with self.lock: + def update_state(artifact): + if artifact.state == UNBUILT: + is_in_cache = response[artifact.basename()] + if is_in_cache: + artifact.state = BUILT + self._map_build_graph(self.artifact, update_state) + self.build_started = True def _map_build_graph(self, artifact, callback, components=[]): @@ -181,15 +195,21 @@ class SingleBuildController(): def _mark_artifact_as_built(self, cache_key, name=None): - def set_state(a): if a.cache_key == artifact.cache_key: a.state = BUILT + self.requests_controller.mark_as_built(self.request_id, + a.cache_key, + a.kind, a.name) artifact = self._find_artifact(cache_key, name) with self.lock: artifact.state = BUILT + self.requests_controller.mark_as_built(self.request_id, + artifact.cache_key, + artifact.kind, + artifact.name) if not name and artifact.kind == 'chunk': # Building a single chunk artifact # yields all chunk artifacts for the given source @@ -198,6 +218,7 @@ class SingleBuildController(): self._map_build_graph(self.artifact, set_state) + class CacheRequestClient(gear.Client): def __init__(self, controller): super(CacheRequestClient, self).__init__() @@ -269,24 +290,38 @@ class BuilderClient(gear.Client): class RequestsController(): def __init__(self): + self.next_id = 1 + self.new_request_lock = threading.Lock() self.build_requests = [] + self.build_status_lock = threading.Lock() def add_request(self, request): json_request = json.dumps(request) request_data = {} - request_data['controller'] = SingleBuildController() - # TODO: is this the right place to do this? - request_data['controller'].start_build(json_request) - request_data['request'] = request + with self.new_request_lock: + request_data['id'] = self.next_id + request_data['controller'] = SingleBuildController(self, self.next_id) + # TODO: is this the right place to do this? + request_data['controller'].start_build(json_request) + request_data['request'] = request + self.next_id += 1 self.build_requests.append(request_data) def queue_if_possible(self): # TODO: check all of them in a loop? controller = self.build_requests[0]['controller'] - if controller.artifact != None and controller.build_started == True: - to_build = controller.find_artifacts_that_are_ready_to_build( - controller.artifact) - controller._queue_worker_builds(to_build) + with self.build_status_lock: + if controller.artifact != None and controller.build_started == True: + to_build = controller.find_artifacts_that_are_ready_to_build( + controller.artifact) + controller._queue_worker_builds(to_build) + + # TODO: Block execution until we receive a mark_as_built + + def mark_as_built(self, request_id, cache_key, kind, name): + with self.build_status_lock: + print "TO %s: Artifact %s built" % (request_id, name) + |