diff options
Diffstat (limited to 'buildtools')
-rw-r--r-- | buildtools/wafadmin/3rdparty/build_file_tracker.py | 54 | ||||
-rwxr-xr-x | buildtools/wafadmin/3rdparty/prefork.py | 276 | ||||
-rw-r--r-- | buildtools/wafadmin/Node.py | 2 | ||||
-rw-r--r-- | buildtools/wafadmin/TaskGen.py | 2 | ||||
-rw-r--r-- | buildtools/wafadmin/Tools/tex.py | 2 | ||||
-rw-r--r-- | buildtools/wafadmin/Utils.py | 12 |
6 files changed, 339 insertions, 9 deletions
diff --git a/buildtools/wafadmin/3rdparty/build_file_tracker.py b/buildtools/wafadmin/3rdparty/build_file_tracker.py new file mode 100644 index 00000000000..9c48928fc02 --- /dev/null +++ b/buildtools/wafadmin/3rdparty/build_file_tracker.py @@ -0,0 +1,54 @@ +#! /usr/bin/env python +# encoding: utf-8 +# Thomas Nagy, 2015 + +""" +Force tasks to use file timestamps to force partial rebuilds when touch-ing build files + +touch out/libfoo.a +... rebuild what depends on libfoo.a + +to use:: + def options(opt): + opt.tool_options('build_file_tracker') +""" + +import os +import Task, Utils + +def signature(self): + try: return self.cache_sig[0] + except AttributeError: pass + + self.m = Utils.md5() + + # explicit deps + exp_sig = self.sig_explicit_deps() + + # env vars + var_sig = self.sig_vars() + + # implicit deps + imp_sig = Task.SIG_NIL + if self.scan: + try: + imp_sig = self.sig_implicit_deps() + except ValueError: + return self.signature() + + # timestamp dependency on build files only (source files are hashed) + buf = [] + for k in self.inputs + getattr(self, 'dep_nodes', []) + self.generator.bld.node_deps.get(self.unique_id(), []): + if k.id & 3 == 3: + t = os.stat(k.abspath(self.env)).st_mtime + buf.append(t) + self.m.update(str(buf)) + + # we now have the signature (first element) and the details (for debugging) + ret = self.m.digest() + self.cache_sig = (ret, exp_sig, imp_sig, var_sig) + return ret + +Task.Task.signature_bak = Task.Task.signature # unused, kept just in case +Task.Task.signature = signature # overridden + diff --git a/buildtools/wafadmin/3rdparty/prefork.py b/buildtools/wafadmin/3rdparty/prefork.py new file mode 100755 index 00000000000..1c760c27bba --- /dev/null +++ b/buildtools/wafadmin/3rdparty/prefork.py @@ -0,0 +1,276 @@ +#! /usr/bin/env python +# encoding: utf-8 +# Thomas Nagy, 2015 (ita) +# +# prefer the waf 1.8 version + +""" +The full samba build can be faster by ~10%, but there are a few limitations: +* only one build process should be run at a time as the servers would use the same ports +* only one build command is going to be called ("waf build configure build" would not work) + +def build(bld): + + mod = Utils.load_tool('prefork') + mod.build(bld) + ... + (build declarations after) +""" + +import os, re, socket, threading, sys, subprocess, time, atexit, traceback +try: + import SocketServer +except ImportError: + import socketserver as SocketServer +try: + from queue import Queue +except ImportError: + from Queue import Queue +try: + import cPickle +except ImportError: + import pickle as cPickle + +DEFAULT_PORT = 51200 + +HEADER_SIZE = 128 + +REQ = 'REQ' +RES = 'RES' +BYE = 'BYE' + +def make_header(params): + header = ','.join(params) + if sys.hexversion > 0x3000000: + header = header.encode('iso8859-1') + header = header.ljust(HEADER_SIZE) + assert(len(header) == HEADER_SIZE) + return header + + +re_valid_query = re.compile('^[a-zA-Z0-9_, ]+$') +class req(SocketServer.StreamRequestHandler): + def handle(self): + while 1: + try: + self.process_command() + except Exception as e: + print(e) + break + + def process_command(self): + query = self.rfile.read(HEADER_SIZE) + if not query: + return + #print(len(query)) + assert(len(query) == HEADER_SIZE) + if sys.hexversion > 0x3000000: + query = query.decode('iso8859-1') + #print "%r" % query + if not re_valid_query.match(query): + raise ValueError('Invalid query %r' % query) + + query = query.strip().split(',') + + if query[0] == REQ: + self.run_command(query[1:]) + elif query[0] == BYE: + raise ValueError('Exit') + else: + raise ValueError('Invalid query %r' % query) + + def run_command(self, query): + + size = int(query[0]) + data = self.rfile.read(size) + assert(len(data) == size) + kw = cPickle.loads(data) + + # run command + ret = out = err = exc = None + cmd = kw['cmd'] + del kw['cmd'] + #print(cmd) + + try: + if kw['stdout'] or kw['stderr']: + p = subprocess.Popen(cmd, **kw) + (out, err) = p.communicate() + ret = p.returncode + else: + ret = subprocess.Popen(cmd, **kw).wait() + except Exception as e: + ret = -1 + exc = str(e) + traceback.format_exc() + + # write the results + if out or err or exc: + data = (out, err, exc) + data = cPickle.dumps(data, -1) + else: + data = '' + + params = [RES, str(ret), str(len(data))] + + self.wfile.write(make_header(params)) + + if data: + self.wfile.write(data) + +def create_server(conn, cls): + #SocketServer.ThreadingTCPServer.allow_reuse_address = True + #server = SocketServer.ThreadingTCPServer(conn, req) + + SocketServer.TCPServer.allow_reuse_address = True + server = SocketServer.TCPServer(conn, req) + #server.timeout = 6000 # seconds + server.serve_forever(poll_interval=0.001) + +if __name__ == '__main__': + if len(sys.argv) > 1: + port = int(sys.argv[1]) + else: + port = DEFAULT_PORT + #conn = (socket.gethostname(), port) + conn = ("127.0.0.1", port) + #print("listening - %r %r\n" % conn) + create_server(conn, req) +else: + + import Runner, Utils + + def init_task_pool(self): + # lazy creation, and set a common pool for all task consumers + pool = self.pool = [] + for i in range(self.numjobs): + consumer = Runner.get_pool() + pool.append(consumer) + consumer.idx = i + self.ready = Queue(0) + def setq(consumer): + consumer.ready = self.ready + try: + threading.current_thread().idx = consumer.idx + except Exception as e: + print(e) + for x in pool: + x.ready.put(setq) + return pool + Runner.Parallel.init_task_pool = init_task_pool + + PORT = 51200 + + def make_server(idx): + port = PORT + idx + cmd = [sys.executable, os.path.abspath(__file__), str(port)] + proc = subprocess.Popen(cmd) + proc.port = port + return proc + + def make_conn(srv): + #port = PORT + idx + port = srv.port + conn = socket.socket(socket.AF_INET, socket.SOCK_STREAM) + conn.connect(('127.0.0.1', port)) + return conn + + SERVERS = [] + CONNS = [] + def close_all(): + while CONNS: + conn = CONNS.pop() + try: + conn.close() + except: + pass + while SERVERS: + srv = SERVERS.pop() + try: + srv.kill() + except: + pass + atexit.register(close_all) + + def put_data(conn, data): + conn.send(data) + + def read_data(conn, siz): + ret = conn.recv(siz) + if not ret: + print("closed connection?") + + assert(len(ret) == siz) + return ret + + def exec_command(cmd, **kw): + if 'log' in kw: + log = kw['log'] + kw['stdout'] = kw['stderr'] = subprocess.PIPE + del(kw['log']) + else: + kw['stdout'] = kw['stderr'] = None + kw['shell'] = isinstance(cmd, str) + + idx = threading.current_thread().idx + kw['cmd'] = cmd + + data = cPickle.dumps(kw, -1) + params = [REQ, str(len(data))] + header = make_header(params) + + conn = CONNS[idx] + + put_data(conn, header) + put_data(conn, data) + + data = read_data(conn, HEADER_SIZE) + if sys.hexversion > 0x3000000: + data = data.decode('iso8859-1') + + lst = data.split(',') + ret = int(lst[1]) + dlen = int(lst[2]) + + out = err = None + if dlen: + data = read_data(conn, dlen) + (out, err, exc) = cPickle.loads(data) + if exc: + raise Utils.WafError('Execution failure: %s' % exc) + + if out: + log.write(out) + if err: + log.write(err) + + return ret + + def __init__(self): + threading.Thread.__init__(self) + + # identifier of the current thread + self.idx = len(SERVERS) + + # create a server and wait for the connection + srv = make_server(self.idx) + SERVERS.append(srv) + + conn = None + for x in range(30): + try: + conn = make_conn(srv) + break + except socket.error: + time.sleep(0.01) + if not conn: + raise ValueError('Could not start the server!') + CONNS.append(conn) + + self.setDaemon(1) + self.start() + Runner.TaskConsumer.__init__ = __init__ + + def build(bld): + # dangerous, there is no other command hopefully + Utils.exec_command = exec_command + diff --git a/buildtools/wafadmin/Node.py b/buildtools/wafadmin/Node.py index 236dd0d2b3f..4fa205c96be 100644 --- a/buildtools/wafadmin/Node.py +++ b/buildtools/wafadmin/Node.py @@ -663,7 +663,7 @@ class Node(object): def update_build_dir(self, env=None): if not env: - for env in bld.all_envs: + for env in self.bld.all_envs: self.update_build_dir(env) return diff --git a/buildtools/wafadmin/TaskGen.py b/buildtools/wafadmin/TaskGen.py index ae1834a10a4..52d0236c850 100644 --- a/buildtools/wafadmin/TaskGen.py +++ b/buildtools/wafadmin/TaskGen.py @@ -567,7 +567,7 @@ def exec_rule(self): if getattr(self, 'cwd', None): tsk.cwd = self.cwd - if getattr(self, 'on_results', None): + if getattr(self, 'on_results', None) or getattr(self, 'update_outputs', None): Task.update_outputs(cls) if getattr(self, 'always', None): diff --git a/buildtools/wafadmin/Tools/tex.py b/buildtools/wafadmin/Tools/tex.py index 2dd748b2320..700c54fc6e4 100644 --- a/buildtools/wafadmin/Tools/tex.py +++ b/buildtools/wafadmin/Tools/tex.py @@ -154,7 +154,7 @@ def tex_build(task, command='LATEX'): task.env.SRCFILE = srcfile ret = fun(task) if ret: - error('error when calling %s %s' % (command, latex_compile_cmd)) + error('error when calling %s %s' % (command, latex_fun)) return ret return None # ok diff --git a/buildtools/wafadmin/Utils.py b/buildtools/wafadmin/Utils.py index 91ded93b6e7..cff6719261d 100644 --- a/buildtools/wafadmin/Utils.py +++ b/buildtools/wafadmin/Utils.py @@ -416,15 +416,15 @@ def pprint(col, str, label='', sep='\n'): "print messages in color" sys.stderr.write("%s%s%s %s%s" % (Logs.colors(col), str, Logs.colors.NORMAL, label, sep)) -def check_dir(dir): +def check_dir(path): """If a folder doesn't exists, create it.""" - try: - os.lstat(dir) - except OSError: + if not os.path.isdir(path): try: - os.makedirs(dir) + os.makedirs(path) except OSError, e: - raise WafError("Cannot create folder '%s' (original error: %s)" % (dir, e)) + if not os.path.isdir(path): + raise Errors.WafError('Cannot create the folder %r' % path, ex=e) + def cmd_output(cmd, **kw): |