third_party: Update waf to version 2.0.22

New in waf 2.0.22

* Fix stdin propagation with faulty vcvarsall scripts #2315
* Enable mixing Unix-style paths with destdir on Windows platforms #2337
* Fix shell escaping unit test parameters #2314
* Improve extras/clang_compilation_database and extras/swig compatibility #2336
* Propagate C++ flags to the Cuda compiler in extras/cuda #2311
* Fix detection of Qt 5.0.0 (preparation for Qt6) #2331
* Enable Haxe processing #2308
* Fix regression in MACOSX_DEPLOYMENT_TARGET caused by distutils #2330
* Fix extras/wafcache concurrent trimming issues #2312
* Fix extras/wafcache symlink handling #2327

The import was done like this:

./third_party/waf/update.sh

Then changing buildtools/bin/waf and buildtools/wafsamba/wafsamba.py
by hand.

Pair-Programmed-With: Stefan Metzmacher <metze@samba.org>

Signed-off-by: Andreas Schneider <asn@samba.org>
Signed-off-by: Stefan Metzmacher <metze@samba.org>
Reviewed-by: Andrew Bartlett <abartlet@samba.org>

Autobuild-User(master): Andrew Bartlett <abartlet@samba.org>
Autobuild-Date(master): Thu Sep  2 21:22:17 UTC 2021 on sn-devel-184

10 files changed, 213 insertions, 42 deletions

diff --git a/third_party/waf/waflib/Build.py b/third_party/waf/waflib/Build.py
index 52837618577..b49dd8302b1 100644
--- a/third_party/waf/waflib/Build.py
+++ b/third_party/waf/waflib/Build.py
@@ -1066,9 +1066,9 @@ class inst(Task.Task):
 		else:
 			dest = os.path.normpath(Utils.subst_vars(self.install_to, self.env))
 		if not os.path.isabs(dest):
-		    dest = os.path.join(self.env.PREFIX, dest)
+			dest = os.path.join(self.env.PREFIX, dest)
 		if destdir and Options.options.destdir:
-			dest = os.path.join(Options.options.destdir, os.path.splitdrive(dest)[1].lstrip(os.sep))
+			dest = Options.options.destdir.rstrip(os.sep) + os.sep + os.path.splitdrive(dest)[1].lstrip(os.sep)
 		return dest
 
 	def copy_fun(self, src, tgt):
diff --git a/third_party/waf/waflib/Context.py b/third_party/waf/waflib/Context.py
index 0ce9df6e91f..07ee1201f03 100644
--- a/third_party/waf/waflib/Context.py
+++ b/third_party/waf/waflib/Context.py
@@ -18,13 +18,13 @@ else:
 	import imp
 
 # the following 3 constants are updated on each new release (do not touch)
-HEXVERSION=0x2001500
+HEXVERSION=0x2001600
 """Constant updated on new releases"""
 
-WAFVERSION="2.0.21"
+WAFVERSION="2.0.22"
 """Constant updated on new releases"""
 
-WAFREVISION="edde20a6425a5c3eb6b47d5f3f5c4fbc93fed5f4"
+WAFREVISION="816d5bc48ba2abc4ac22f2b44d94d322bf992b9c"
 """Git revision when the waf version is updated"""
 
 WAFNAME="waf"
diff --git a/third_party/waf/waflib/Tools/msvc.py b/third_party/waf/waflib/Tools/msvc.py
index 37233be8242..0c4703aaee9 100644
--- a/third_party/waf/waflib/Tools/msvc.py
+++ b/third_party/waf/waflib/Tools/msvc.py
@@ -193,7 +193,7 @@ echo PATH=%%PATH%%
 echo INCLUDE=%%INCLUDE%%
 echo LIB=%%LIB%%;%%LIBPATH%%
 """ % (vcvars,target))
-	sout = conf.cmd_and_log(['cmd.exe', '/E:on', '/V:on', '/C', batfile.abspath()])
+	sout = conf.cmd_and_log(['cmd.exe', '/E:on', '/V:on', '/C', batfile.abspath()], stdin=getattr(Utils.subprocess, 'DEVNULL', None))
 	lines = sout.splitlines()
 
 	if not lines[0]:
diff --git a/third_party/waf/waflib/Tools/python.py b/third_party/waf/waflib/Tools/python.py
index b1c8dd01285..07442561dff 100644
--- a/third_party/waf/waflib/Tools/python.py
+++ b/third_party/waf/waflib/Tools/python.py
@@ -327,7 +327,7 @@ def check_python_headers(conf, features='pyembed pyext'):
 	dct = dict(zip(v, lst))
 	x = 'MACOSX_DEPLOYMENT_TARGET'
 	if dct[x]:
-		env[x] = conf.environ[x] = dct[x]
+		env[x] = conf.environ[x] = str(dct[x])
 	env.pyext_PATTERN = '%s' + dct['SO'] # not a mistake
 
 
diff --git a/third_party/waf/waflib/Tools/qt5.py b/third_party/waf/waflib/Tools/qt5.py
index cff2028174f..82c83e18c8a 100644
--- a/third_party/waf/waflib/Tools/qt5.py
+++ b/third_party/waf/waflib/Tools/qt5.py
@@ -566,7 +566,7 @@ def find_qt5_binaries(self):
 	# at the end, try to find qmake in the paths given
 	# keep the one with the highest version
 	cand = None
-	prev_ver = ['5', '0', '0']
+	prev_ver = ['0', '0', '0']
 	for qmk in ('qmake-qt5', 'qmake5', 'qmake'):
 		try:
 			qmake = self.find_program(qmk, path_list=paths)
@@ -580,7 +580,7 @@ def find_qt5_binaries(self):
 			else:
 				if version:
 					new_ver = version.split('.')
-					if new_ver > prev_ver:
+					if new_ver[0] == '5' and new_ver > prev_ver:
 						cand = qmake
 						prev_ver = new_ver
 
@@ -783,7 +783,7 @@ def set_qt5_libs_to_check(self):
 			pat = self.env.cxxstlib_PATTERN
 		if Utils.unversioned_sys_platform() == 'darwin':
 			pat = r"%s\.framework"
-		re_qt = re.compile(pat%'Qt5?(?P<name>.*)'+'$')
+		re_qt = re.compile(pat % 'Qt5?(?P<name>\\D+)' + '$')
 		for x in dirlst:
 			m = re_qt.match(x)
 			if m:
diff --git a/third_party/waf/waflib/Tools/waf_unit_test.py b/third_party/waf/waflib/Tools/waf_unit_test.py
index dc66fe9c184..8cff89bdeb9 100644
--- a/third_party/waf/waflib/Tools/waf_unit_test.py
+++ b/third_party/waf/waflib/Tools/waf_unit_test.py
@@ -206,7 +206,7 @@ class utest(Task.Task):
 		self.ut_exec = getattr(self.generator, 'ut_exec', [self.inputs[0].abspath()])
 		ut_cmd = getattr(self.generator, 'ut_cmd', False)
 		if ut_cmd:
-			self.ut_exec = shlex.split(ut_cmd % ' '.join(self.ut_exec))
+			self.ut_exec = shlex.split(ut_cmd % Utils.shell_escape(self.ut_exec))
 
 		return self.exec_command(self.ut_exec)
 
diff --git a/third_party/waf/waflib/Utils.py b/third_party/waf/waflib/Utils.py
index fc64fa05154..669490ca908 100644
--- a/third_party/waf/waflib/Utils.py
+++ b/third_party/waf/waflib/Utils.py
@@ -11,7 +11,7 @@ through Python versions 2.5 to 3.X and across different platforms (win32, linux,
 
 from __future__ import with_statement
 
-import atexit, os, sys, errno, inspect, re, datetime, platform, base64, signal, functools, time
+import atexit, os, sys, errno, inspect, re, datetime, platform, base64, signal, functools, time, shlex
 
 try:
 	import cPickle
@@ -577,10 +577,13 @@ def quote_define_name(s):
 	fu = fu.upper()
 	return fu
 
-re_sh = re.compile('\\s|\'|"')
-"""
-Regexp used for shell_escape below
-"""
+# shlex.quote didn't exist until python 3.3. Prior to that it was a non-documented
+# function in pipes.
+try:
+	shell_quote = shlex.quote
+except AttributeError:
+	import pipes
+	shell_quote = pipes.quote
 
 def shell_escape(cmd):
 	"""
@@ -589,7 +592,7 @@ def shell_escape(cmd):
 	"""
 	if isinstance(cmd, str):
 		return cmd
-	return ' '.join(repr(x) if re_sh.search(x) else x for x in cmd)
+	return ' '.join(shell_quote(x) for x in cmd)
 
 def h_list(lst):
 	"""
diff --git a/third_party/waf/waflib/extras/clang_compilation_database.py b/third_party/waf/waflib/extras/clang_compilation_database.py
index ff71f22ecfd..17f66949376 100644
--- a/third_party/waf/waflib/extras/clang_compilation_database.py
+++ b/third_party/waf/waflib/extras/clang_compilation_database.py
@@ -29,22 +29,9 @@ from waflib import Logs, TaskGen, Task, Build, Scripting
 
 Task.Task.keep_last_cmd = True
 
-@TaskGen.feature('c', 'cxx')
-@TaskGen.after_method('process_use')
-def collect_compilation_db_tasks(self):
-	"Add a compilation database entry for compiled tasks"
-	if not isinstance(self.bld, ClangDbContext):
-		return
-
-	tup = tuple(y for y in [Task.classes.get(x) for x in ('c', 'cxx')] if y)
-	for task in getattr(self, 'compiled_tasks', []):
-		if isinstance(task, tup):
-			self.bld.clang_compilation_database_tasks.append(task)
-
 class ClangDbContext(Build.BuildContext):
 	'''generates compile_commands.json by request'''
 	cmd = 'clangdb'
-	clang_compilation_database_tasks = []
 
 	def write_compilation_database(self):
 		"""
@@ -78,6 +65,8 @@ class ClangDbContext(Build.BuildContext):
 		Build dry run
 		"""
 		self.restore()
+		self.cur_tasks = []
+		self.clang_compilation_database_tasks = []
 
 		if not self.all_envs:
 			self.load_envs()
@@ -103,8 +92,21 @@ class ClangDbContext(Build.BuildContext):
 					lst = [tg]
 				else: lst = tg.tasks
 				for tsk in lst:
+					if tsk.__class__.__name__ == "swig":
+						tsk.runnable_status()
+						if hasattr(tsk, 'more_tasks'):
+							lst.extend(tsk.more_tasks)
+					# Not all dynamic tasks can be processed, in some cases
+					# one may have to call the method "run()" like this:
+					#elif tsk.__class__.__name__ == 'src2c':
+					#	tsk.run()
+					#	if hasattr(tsk, 'more_tasks'):
+					#		lst.extend(tsk.more_tasks)
+
 					tup = tuple(y for y in [Task.classes.get(x) for x in ('c', 'cxx')] if y)
 					if isinstance(tsk, tup):
+						self.clang_compilation_database_tasks.append(tsk)
+						tsk.nocache = True
 						old_exec = tsk.exec_command
 						tsk.exec_command = exec_command
 						tsk.run()
diff --git a/third_party/waf/waflib/extras/haxe.py b/third_party/waf/waflib/extras/haxe.py
new file mode 100644
index 00000000000..cb3ba6a949c
--- /dev/null
+++ b/third_party/waf/waflib/extras/haxe.py
@@ -0,0 +1,131 @@
+import os, re
+from waflib import Utils, Task, Errors
+from waflib.TaskGen import extension, taskgen_method, feature
+from waflib.Configure import conf
+
+@conf
+def libname_haxe(self, libname):
+	return libname
+
+@conf
+def check_lib_haxe(self, libname, uselib_store=None):
+	haxe_libs = [node.name for node in self.root.find_node('haxe_libraries').ant_glob()]
+	changed = False
+	self.start_msg('Checking for library %s' % libname)
+	if libname + '.hxml' in haxe_libs:
+		self.end_msg('yes')
+	else:
+		changed = True
+		try:
+			cmd = self.env.LIX + ['+lib', libname]
+			res = self.cmd_and_log(cmd)
+			if (res):
+				raise Errors.WafError(res)
+			else:
+				self.end_msg('downloaded', color = 'YELLOW')
+		except Errors.WafError as e:
+			self.end_msg('no', color = 'RED')
+			self.fatal('Getting %s has failed' % libname)
+
+	postfix = uselib_store if uselib_store else libname.upper()
+	self.env['LIB_' + postfix] += [self.libname_haxe(libname)]
+	return changed
+
+@conf
+def check_libs_haxe(self, libnames, uselib_store=None):
+	changed = False
+	for libname in Utils.to_list(libnames):
+		if self.check_lib_haxe(libname, uselib_store):
+			changed = True
+	return changed
+
+@conf
+def ensure_lix_pkg(self, *k, **kw):
+	if kw.get('compiler') == 'hx':
+		if isinstance(kw.get('libs'), list) and len(kw.get('libs')):
+			changed = self.check_libs_haxe(kw.get('libs'), kw.get('uselib_store'))
+			if changed:
+				try:
+					cmd = self.env.LIX + ['download']
+					res = self.cmd_and_log(cmd)
+					if (res):
+						raise Errors.WafError(res)
+				except Errors.WafError as e:
+					self.fatal('lix download has failed')
+		else:
+			self.check_lib_haxe(kw.get('lib'), kw.get('uselib_store'))
+
+@conf
+def haxe(bld, *k, **kw):
+	task_gen = bld(*k, **kw)
+
+class haxe(Task.Task):
+	vars = ['HAXE', 'HAXE_VERSION', 'HAXEFLAGS']
+	ext_out = ['.hl', '.c', '.h']
+
+	def run(self):
+		cmd = self.env.HAXE + self.env.HAXEFLAGS
+		return self.exec_command(cmd, stdout = open(os.devnull, 'w'))
+
+@taskgen_method
+def init_haxe_task(self, node):
+	def addflags(flags):
+		self.env.append_value('HAXEFLAGS', flags)
+
+	if node.suffix() == '.hxml':
+		addflags(self.path.abspath() + '/' + node.name)
+	else:
+		addflags(['-main', node.name])
+	addflags(['-hl', self.path.get_bld().make_node(self.target).abspath()])
+	addflags(['-cp', self.path.abspath()])
+	addflags(['-D', 'resourcesPath=%s' % getattr(self, 'res', '')])
+	if hasattr(self, 'use'):
+		for dep in self.use:
+			if self.env['LIB_' + dep]:
+				for lib in self.env['LIB_' + dep]: addflags(['-lib', lib])
+
+@extension('.hx', '.hxml')
+def haxe_file(self, node):
+	if len(self.source) > 1:
+		self.bld.fatal('Use separate task generators for multiple files')
+
+	try:
+		haxetask = self.haxetask
+	except AttributeError:
+		haxetask = self.haxetask = self.create_task('haxe')
+		self.init_haxe_task(node)
+
+	haxetask.inputs.append(node)
+	haxetask.outputs.append(self.path.get_bld().make_node(self.target))
+
+@conf
+def find_haxe(self, min_version):
+	npx = self.env.NPX = self.find_program('npx')
+	self.env.LIX = npx + ['lix']
+	npx_haxe = self.env.HAXE = npx + ['haxe']
+	try:
+		output = self.cmd_and_log(npx_haxe + ['-version'])
+	except Errors.WafError:
+		haxe_version = None
+	else:
+		ver = re.search(r'\d+.\d+.\d+', output).group().split('.')
+		haxe_version = tuple([int(x) for x in ver])
+
+	self.msg('Checking for haxe version',
+	         haxe_version, haxe_version and haxe_version >= min_version)
+	if npx_haxe and haxe_version < min_version:
+		self.fatal('haxe version %r is too old, need >= %r' % (haxe_version, min_version))
+
+	self.env.HAXE_VERSION = haxe_version
+	return npx_haxe
+
+@conf
+def check_haxe(self, min_version=(4,1,4)):
+	if self.env.HAXE_MINVER:
+		min_version = self.env.HAXE_MINVER
+	find_haxe(self, min_version)
+
+def configure(self):
+	self.env.HAXEFLAGS = []
+	self.check_haxe()
+	self.add_os_flags('HAXEFLAGS', dup = False)
diff --git a/third_party/waf/waflib/extras/wafcache.py b/third_party/waf/waflib/extras/wafcache.py
index 088fd0d098d..cc23fcd6673 100644
--- a/third_party/waf/waflib/extras/wafcache.py
+++ b/third_party/waf/waflib/extras/wafcache.py
@@ -31,6 +31,7 @@ The following environment variables may be set:
     gsutil cp gs://mybucket/bb/bbbbb/2 build/somefile
 * WAFCACHE_NO_PUSH: if set, disables pushing to the cache
 * WAFCACHE_VERBOSITY: if set, displays more detailed cache operations
+* WAFCACHE_STATS: if set, displays cache usage statistics on exit
 
 File cache specific options:
   Files are copied using hard links by default; if the cache is located
@@ -69,6 +70,7 @@ EVICT_INTERVAL_MINUTES = int(os.environ.get('WAFCACHE_EVICT_INTERVAL_MINUTES', 3
 EVICT_MAX_BYTES = int(os.environ.get('WAFCACHE_EVICT_MAX_BYTES', 10**10))
 WAFCACHE_NO_PUSH = 1 if os.environ.get('WAFCACHE_NO_PUSH') else 0
 WAFCACHE_VERBOSITY = 1 if os.environ.get('WAFCACHE_VERBOSITY') else 0
+WAFCACHE_STATS = 1 if os.environ.get('WAFCACHE_STATS') else 0
 OK = "ok"
 
 re_waf_cmd = re.compile('(?P<src>%{SRC})|(?P<tgt>%{TGT})')
@@ -93,6 +95,9 @@ def can_retrieve_cache(self):
 	sig = self.signature()
 	ssig = Utils.to_hex(self.uid() + sig)
 
+	if WAFCACHE_STATS:
+		self.generator.bld.cache_reqs += 1
+
 	files_to = [node.abspath() for node in self.outputs]
 	err = cache_command(ssig, [], files_to)
 	if err.startswith(OK):
@@ -100,6 +105,8 @@ def can_retrieve_cache(self):
 			Logs.pprint('CYAN', '  Fetched %r from cache' % files_to)
 		else:
 			Logs.debug('wafcache: fetched %r from cache', files_to)
+		if WAFCACHE_STATS:
+			self.generator.bld.cache_hits += 1
 	else:
 		if WAFCACHE_VERBOSITY:
 			Logs.pprint('YELLOW', '  No cache entry %s' % files_to)
@@ -117,11 +124,17 @@ def put_files_cache(self):
 	if WAFCACHE_NO_PUSH or getattr(self, 'cached', None) or not self.outputs:
 		return
 
+	files_from = []
+	for node in self.outputs:
+		path = node.abspath()
+		if not os.path.isfile(path):
+			return
+		files_from.append(path)
+
 	bld = self.generator.bld
 	sig = self.signature()
 	ssig = Utils.to_hex(self.uid() + sig)
 
-	files_from = [node.abspath() for node in self.outputs]
 	err = cache_command(ssig, files_from, [])
 
 	if err.startswith(OK):
@@ -129,6 +142,8 @@ def put_files_cache(self):
 			Logs.pprint('CYAN', '  Successfully uploaded %s to cache' % files_from)
 		else:
 			Logs.debug('wafcache: Successfully uploaded %r to cache', files_from)
+		if WAFCACHE_STATS:
+			self.generator.bld.cache_puts += 1
 	else:
 		if WAFCACHE_VERBOSITY:
 			Logs.pprint('RED', '  Error caching step results %s: %s' % (files_from, err))
@@ -193,6 +208,10 @@ def make_cached(cls):
 	if getattr(cls, 'nocache', None) or getattr(cls, 'has_cache', False):
 		return
 
+	full_name = "%s.%s" % (cls.__module__, cls.__name__)
+	if full_name in ('waflib.Tools.ccroot.vnum', 'waflib.Build.inst'):
+		return
+
 	m1 = getattr(cls, 'run', None)
 	def run(self):
 		if getattr(self, 'nocache', False):
@@ -208,9 +227,6 @@ def make_cached(cls):
 			return m2(self)
 		ret = m2(self)
 		self.put_files_cache()
-		if hasattr(self, 'chmod'):
-			for node in self.outputs:
-				os.chmod(node.abspath(), self.chmod)
 		return ret
 	cls.post_run = post_run
 	cls.has_cache = True
@@ -257,6 +273,19 @@ def build(bld):
 	for x in reversed(list(Task.classes.values())):
 		make_cached(x)
 
+	if WAFCACHE_STATS:
+		# Init counter for statistics and hook to print results at the end
+		bld.cache_reqs = bld.cache_hits = bld.cache_puts = 0
+
+		def printstats(bld):
+			hit_ratio = 0
+			if bld.cache_reqs > 0:
+				hit_ratio = (bld.cache_hits / bld.cache_reqs) * 100
+			Logs.pprint('CYAN', '  wafcache stats: requests: %s, hits, %s, ratio: %.2f%%, writes %s' %
+					 (bld.cache_reqs, bld.cache_hits, hit_ratio, bld.cache_puts) )
+
+		bld.add_post_fun(printstats)
+
 def cache_command(sig, files_from, files_to):
 	"""
 	Create a command for cache worker processes, returns a pickled
@@ -320,7 +349,10 @@ def lru_trim():
 
 				size = 0
 				for fname in os.listdir(path):
-					size += os.lstat(os.path.join(path, fname)).st_size
+					try:
+						size += os.lstat(os.path.join(path, fname)).st_size
+					except OSError:
+						pass
 				lst.append((os.stat(path).st_mtime, size, path))
 
 	lst.sort(key=lambda x: x[0])
@@ -331,7 +363,7 @@ def lru_trim():
 		_, tmp_size, path = lst.pop()
 		tot -= tmp_size
 
-		tmp = path + '.tmp'
+		tmp = path + '.remove'
 		try:
 			shutil.rmtree(tmp)
 		except OSError:
@@ -339,12 +371,12 @@ def lru_trim():
 		try:
 			os.rename(path, tmp)
 		except OSError:
-			sys.stderr.write('Could not rename %r to %r' % (path, tmp))
+			sys.stderr.write('Could not rename %r to %r\n' % (path, tmp))
 		else:
 			try:
 				shutil.rmtree(tmp)
 			except OSError:
-				sys.stderr.write('Could not remove %r' % tmp)
+				sys.stderr.write('Could not remove %r\n' % tmp)
 	sys.stderr.write("Cache trimmed: %r bytes in %r folders left\n" % (tot, len(lst)))
 
 
@@ -371,8 +403,8 @@ def lru_evict():
 			try:
 				fcntl.flock(fd, fcntl.LOCK_EX | fcntl.LOCK_NB)
 			except EnvironmentError:
-				sys.stderr.write('another process is running!\n')
-				pass
+				if WAFCACHE_VERBOSITY:
+					sys.stderr.write('wafcache: another cleaning process is running\n')
 			else:
 				# now dow the actual cleanup
 				lru_trim()
@@ -443,7 +475,10 @@ class fcache(object):
 		else:
 			# attempt trimming if caching was successful:
 			# we may have things to trim!
-			lru_evict()
+			try:
+				lru_evict()
+			except Exception:
+				return traceback.format_exc()
 		return OK
 
 	def copy_from_cache(self, sig, files_from, files_to):
@@ -481,7 +516,7 @@ class bucket_cache(object):
 		out, err = proc.communicate()
 		if proc.returncode:
 			raise OSError('Error copy %r to %r using: %r (exit %r):\n  out:%s\n  err:%s' % (
-				source, target, cmd, proc.returncode, out.decode(), err.decode()))
+				source, target, cmd, proc.returncode, out.decode(errors='replace'), err.decode(errors='replace')))
 
 	def copy_to_cache(self, sig, files_from, files_to):
 		try: