mesonlib.split_args/quote_arg/join_args

17 files changed, 241 insertions, 69 deletions

diff --git a/mesonbuild/backend/ninjabackend.py b/mesonbuild/backend/ninjabackend.py
index b948e25ea..98f244dad 100644
--- a/mesonbuild/backend/ninjabackend.py
+++ b/mesonbuild/backend/ninjabackend.py
@@ -14,7 +14,6 @@
 from typing import List
 import os
 import re
-import shlex
 import pickle
 import subprocess
 from collections import OrderedDict
@@ -32,7 +31,7 @@ from .. import compilers
 from ..compilers import Compiler, CompilerArgs, CCompiler, VisualStudioLikeCompiler, FortranCompiler
 from ..linkers import ArLinker
 from ..mesonlib import (
-    File, LibType, MachineChoice, MesonException, OrderedSet, PerMachine, ProgressBar
+    File, LibType, MachineChoice, MesonException, OrderedSet, PerMachine, ProgressBar, quote_arg
 )
 from ..mesonlib import get_compiler_for_source, has_path_sep
 from .backends import CleanTrees
@@ -44,11 +43,14 @@ FORTRAN_SUBMOD_PAT = r"^\s*\bsubmodule\b\s*\((\w+:?\w+)\)\s*(\w+)"
 FORTRAN_USE_PAT = r"^\s*use,?\s*(?:non_intrinsic)?\s*(?:::)?\s*(\w+)"
 
 if mesonlib.is_windows():
+    # FIXME: can't use quote_arg on Windows just yet; there are a number of existing workarounds
+    # throughout the codebase that cumulatively make the current code work (see, e.g. Backend.escape_extra_args
+    # and NinjaBuildElement.write below) and need to be properly untangled before attempting this
     quote_func = lambda s: '"{}"'.format(s)
     execute_wrapper = ['cmd', '/c']
     rmfile_prefix = ['del', '/f', '/s', '/q', '{}', '&&']
 else:
-    quote_func = shlex.quote
+    quote_func = quote_arg
     execute_wrapper = []
     rmfile_prefix = ['rm', '-f', '{}', '&&']
 
diff --git a/mesonbuild/compilers/compilers.py b/mesonbuild/compilers/compilers.py
index 4218775ba..38bf2408a 100644
--- a/mesonbuild/compilers/compilers.py
+++ b/mesonbuild/compilers/compilers.py
@@ -12,9 +12,9 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-import contextlib, enum, os.path, re, tempfile, shlex
+import contextlib, enum, os.path, re, tempfile
 import typing
-from typing import List, Optional, Tuple
+from typing import Optional, Tuple, List
 
 from ..linkers import StaticLinker, GnuLikeDynamicLinkerMixin
 from .. import coredata
@@ -22,7 +22,7 @@ from .. import mlog
 from .. import mesonlib
 from ..mesonlib import (
     EnvironmentException, MachineChoice, MesonException, OrderedSet,
-    Popen_safe
+    Popen_safe, split_args
 )
 from ..envconfig import (
     Properties,
@@ -799,7 +799,7 @@ class Compiler:
         env_compile_flags = os.environ.get(cflags_mapping[lang])
         log_var(cflags_mapping[lang], env_compile_flags)
         if env_compile_flags is not None:
-            compile_flags += shlex.split(env_compile_flags)
+            compile_flags += split_args(env_compile_flags)
 
         # Link flags (same for all languages)
         if self.use_ldflags():
@@ -820,7 +820,7 @@ class Compiler:
             env_preproc_flags = os.environ.get('CPPFLAGS')
             log_var('CPPFLAGS', env_preproc_flags)
             if env_preproc_flags is not None:
-                compile_flags += shlex.split(env_preproc_flags)
+                compile_flags += split_args(env_preproc_flags)
 
         return compile_flags, link_flags
 
@@ -830,10 +830,10 @@ class Compiler:
         opts.update({
             self.language + '_args': coredata.UserArrayOption(
                 description + ' compiler',
-                [], shlex_split=True, user_input=True, allow_dups=True),
+                [], split_args=True, user_input=True, allow_dups=True),
             self.language + '_link_args': coredata.UserArrayOption(
                 description + ' linker',
-                [], shlex_split=True, user_input=True, allow_dups=True),
+                [], split_args=True, user_input=True, allow_dups=True),
         })
 
         return opts
diff --git a/mesonbuild/compilers/mixins/islinker.py b/mesonbuild/compilers/mixins/islinker.py
index 4c1a4769d..dca20d092 100644
--- a/mesonbuild/compilers/mixins/islinker.py
+++ b/mesonbuild/compilers/mixins/islinker.py
@@ -21,7 +21,6 @@ classes for those cases.
 """
 
 import os
-import shlex
 import typing
 
 from ... import mesonlib
@@ -39,7 +38,7 @@ class LinkerEnvVarsMixin:
         flags = os.environ.get('LDFLAGS')
         if not flags:
             return []
-        return shlex.split(flags)
+        return mesonlib.split_args(flags)
 
 
 class BasicLinkerIsCompilerMixin:
diff --git a/mesonbuild/coredata.py b/mesonbuild/coredata.py
index fe14c3519..579637779 100644
--- a/mesonbuild/coredata.py
+++ b/mesonbuild/coredata.py
@@ -13,14 +13,14 @@
 # limitations under the License.
 
 from . import mlog
-import pickle, os, uuid, shlex
+import pickle, os, uuid
 import sys
 from itertools import chain
 from pathlib import PurePath
 from collections import OrderedDict
 from .mesonlib import (
     MesonException, MachineChoice, PerMachine,
-    default_libdir, default_libexecdir, default_prefix
+    default_libdir, default_libexecdir, default_prefix, split_args
 )
 from .wrap import WrapMode
 import ast
@@ -163,9 +163,9 @@ class UserComboOption(UserOption[str]):
         return value
 
 class UserArrayOption(UserOption[List[str]]):
-    def __init__(self, description, value, shlex_split=False, user_input=False, allow_dups=False, **kwargs):
+    def __init__(self, description, value, split_args=False, user_input=False, allow_dups=False, **kwargs):
         super().__init__(description, kwargs.get('choices', []), yielding=kwargs.get('yielding', None))
-        self.shlex_split = shlex_split
+        self.split_args = split_args
         self.allow_dups = allow_dups
         self.value = self.validate_value(value, user_input=user_input)
 
@@ -183,8 +183,8 @@ class UserArrayOption(UserOption[List[str]]):
             elif value == '':
                 newvalue = []
             else:
-                if self.shlex_split:
-                    newvalue = shlex.split(value)
+                if self.split_args:
+                    newvalue = split_args(value)
                 else:
                     newvalue = [v.strip() for v in value.split(',')]
         elif isinstance(value, list):
diff --git a/mesonbuild/dependencies/base.py b/mesonbuild/dependencies/base.py
index e517fea86..b0f24c86b 100644
--- a/mesonbuild/dependencies/base.py
+++ b/mesonbuild/dependencies/base.py
@@ -34,7 +34,7 @@ from ..compilers import clib_langs
 from ..environment import BinaryTable, Environment, MachineInfo
 from ..cmake import CMakeExecutor, CMakeTraceParser, CMakeException
 from ..mesonlib import MachineChoice, MesonException, OrderedSet, PerMachine
-from ..mesonlib import Popen_safe, version_compare_many, version_compare, listify, stringlistify, extract_as_list
+from ..mesonlib import Popen_safe, version_compare_many, version_compare, listify, stringlistify, extract_as_list, split_args
 from ..mesonlib import Version, LibType
 
 # These must be defined in this file to avoid cyclical references.
@@ -490,16 +490,13 @@ class ConfigToolDependency(ExternalDependency):
 
     def get_config_value(self, args, stage):
         p, out, err = Popen_safe(self.config + args)
-        # This is required to keep shlex from stripping path separators on
-        # Windows. Also, don't put escape sequences in config values, okay?
-        out = out.replace('\\', '\\\\')
         if p.returncode != 0:
             if self.required:
                 raise DependencyException(
                     'Could not generate {} for {}.\n{}'.format(
                         stage, self.name, err))
             return []
-        return shlex.split(out)
+        return split_args(out)
 
     @staticmethod
     def get_methods():
@@ -697,6 +694,11 @@ class PkgConfigDependency(ExternalDependency):
             converted.append(arg)
         return converted
 
+    def _split_args(self, cmd):
+        # pkg-config paths follow Unix conventions, even on Windows; split the
+        # output using shlex.split rather than mesonlib.split_args
+        return shlex.split(cmd)
+
     def _set_cargs(self):
         env = None
         if self.language == 'fortran':
@@ -708,7 +710,7 @@ class PkgConfigDependency(ExternalDependency):
         if ret != 0:
             raise DependencyException('Could not generate cargs for %s:\n\n%s' %
                                       (self.name, out))
-        self.compile_args = self._convert_mingw_paths(shlex.split(out))
+        self.compile_args = self._convert_mingw_paths(self._split_args(out))
 
     def _search_libs(self, out, out_raw):
         '''
@@ -737,7 +739,7 @@ class PkgConfigDependency(ExternalDependency):
         # always searched first.
         prefix_libpaths = OrderedSet()
         # We also store this raw_link_args on the object later
-        raw_link_args = self._convert_mingw_paths(shlex.split(out_raw))
+        raw_link_args = self._convert_mingw_paths(self._split_args(out_raw))
         for arg in raw_link_args:
             if arg.startswith('-L') and not arg.startswith(('-L-l', '-L-L')):
                 path = arg[2:]
@@ -746,7 +748,7 @@ class PkgConfigDependency(ExternalDependency):
                     path = os.path.join(self.env.get_build_dir(), path)
                 prefix_libpaths.add(path)
         system_libpaths = OrderedSet()
-        full_args = self._convert_mingw_paths(shlex.split(out))
+        full_args = self._convert_mingw_paths(self._split_args(out))
         for arg in full_args:
             if arg.startswith(('-L-l', '-L-L')):
                 # These are D language arguments, not library paths
diff --git a/mesonbuild/dependencies/misc.py b/mesonbuild/dependencies/misc.py
index 53c374738..23a283fae 100644
--- a/mesonbuild/dependencies/misc.py
+++ b/mesonbuild/dependencies/misc.py
@@ -18,11 +18,11 @@ from pathlib import Path
 import functools
 import os
 import re
-import shlex
 import sysconfig
 
 from .. import mlog
 from .. import mesonlib
+from ..mesonlib import split_args
 from ..environment import detect_cpu_family
 
 from .base import (
@@ -277,7 +277,7 @@ class MPIDependency(ExternalDependency):
                 mlog.debug(mlog.bold('Standard output\n'), o)
                 mlog.debug(mlog.bold('Standard error\n'), e)
                 return
-            cargs = shlex.split(o)
+            cargs = split_args(o)
 
             cmd = prog.get_command() + ['--showme:link']
             p, o, e = mesonlib.Popen_safe(cmd)
@@ -287,7 +287,7 @@ class MPIDependency(ExternalDependency):
                 mlog.debug(mlog.bold('Standard output\n'), o)
                 mlog.debug(mlog.bold('Standard error\n'), e)
                 return
-            libs = shlex.split(o)
+            libs = split_args(o)
 
             cmd = prog.get_command() + ['--showme:version']
             p, o, e = mesonlib.Popen_safe(cmd)
@@ -316,7 +316,7 @@ class MPIDependency(ExternalDependency):
                 mlog.debug(mlog.bold('Standard output\n'), o)
                 mlog.debug(mlog.bold('Standard error\n'), e)
                 return
-            args = shlex.split(o)
+            args = split_args(o)
 
             version = None
 
diff --git a/mesonbuild/envconfig.py b/mesonbuild/envconfig.py
index f7a43a06e..0c9f03ffd 100644
--- a/mesonbuild/envconfig.py
+++ b/mesonbuild/envconfig.py
@@ -12,11 +12,11 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-import configparser, os, shlex, subprocess
+import configparser, os, subprocess
 import typing
 
 from . import mesonlib
-from .mesonlib import EnvironmentException
+from .mesonlib import EnvironmentException, split_args
 from . import mlog
 
 _T = typing.TypeVar('_T')
@@ -361,7 +361,7 @@ This is probably wrong, it should always point to the native compiler.''' % evar
             evar = self.evarMap.get(name, "")
             command = os.environ.get(evar)
             if command is not None:
-                command = shlex.split(command)
+                command = split_args(command)
 
         # Do not return empty or blank string entries
         if command is not None and (len(command) == 0 or len(command[0].strip()) == 0):
diff --git a/mesonbuild/environment.py b/mesonbuild/environment.py
index 7e1ca9dc1..cf386dab5 100644
--- a/mesonbuild/environment.py
+++ b/mesonbuild/environment.py
@@ -12,7 +12,7 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-import os, platform, re, sys, shlex, shutil, subprocess, typing
+import os, platform, re, sys, shutil, subprocess, typing
 import tempfile
 
 from . import coredata
@@ -20,7 +20,7 @@ from .linkers import ArLinker, ArmarLinker, VisualStudioLinker, DLinker, CcrxLin
 from . import mesonlib
 from .mesonlib import (
     MesonException, EnvironmentException, MachineChoice, Popen_safe,
-    PerMachineDefaultable, PerThreeMachineDefaultable
+    PerMachineDefaultable, PerThreeMachineDefaultable, split_args, quote_arg
 )
 from . import mlog
 
@@ -120,7 +120,7 @@ def detect_gcovr(min_version='3.3', new_rootdir_version='4.2', log=False):
     found = search_version(found)
     if p.returncode == 0 and mesonlib.version_compare(found, '>=' + min_version):
         if log:
-            mlog.log('Found gcovr-{} at {}'.format(found, shlex.quote(shutil.which(gcovr_exe))))
+            mlog.log('Found gcovr-{} at {}'.format(found, quote_arg(shutil.which(gcovr_exe))))
         return gcovr_exe, mesonlib.version_compare(found, '>=' + new_rootdir_version)
     return None, None
 
@@ -158,7 +158,7 @@ def detect_ninja(version: str = '1.5', log: bool = False) -> str:
                     name = 'ninja'
                 if name == 'samu':
                     name = 'samurai'
-                mlog.log('Found {}-{} at {}'.format(name, found, shlex.quote(n)))
+                mlog.log('Found {}-{} at {}'.format(name, found, quote_arg(n)))
             return n
 
 def detect_native_windows_arch():
@@ -1322,7 +1322,7 @@ class Environment:
             if isinstance(compiler, compilers.CudaCompiler):
                 linkers = [self.cuda_static_linker, self.default_static_linker]
             elif evar in os.environ:
-                linkers = [shlex.split(os.environ[evar])]
+                linkers = [split_args(os.environ[evar])]
             elif isinstance(compiler, compilers.VisualStudioLikeCompiler):
                 linkers = [self.vs_static_linker, self.clang_cl_static_linker]
             elif isinstance(compiler, compilers.GnuCompiler):
diff --git a/mesonbuild/linkers.py b/mesonbuild/linkers.py
index 6cce78b87..8de254bfd 100644
--- a/mesonbuild/linkers.py
+++ b/mesonbuild/linkers.py
@@ -14,7 +14,6 @@
 
 import abc
 import os
-import shlex
 import typing
 
 from . import mesonlib
@@ -279,7 +278,7 @@ class DynamicLinker(metaclass=abc.ABCMeta):
         flags = os.environ.get('LDFLAGS')
         if not flags:
             return []
-        return shlex.split(flags)
+        return mesonlib.split_args(flags)
 
     def get_option_args(self, options: 'OptionDictType') -> typing.List[str]:
         return []
diff --git a/mesonbuild/mesonlib.py b/mesonbuild/mesonlib.py
index 133302781..d5646ed0f 100644
--- a/mesonbuild/mesonlib.py
+++ b/mesonbuild/mesonlib.py
@@ -17,7 +17,7 @@ from pathlib import Path
 import sys
 import stat
 import time
-import platform, subprocess, operator, os, shutil, re
+import platform, subprocess, operator, os, shlex, shutil, re
 import collections
 from enum import Enum
 from functools import lru_cache
@@ -729,6 +729,84 @@ def has_path_sep(name, sep='/\\'):
             return True
     return False
 
+
+if is_windows():
+    # shlex.split is not suitable for splitting command line on Window (https://bugs.python.org/issue1724822);
+    # shlex.quote is similarly problematic. Below are "proper" implementations of these functions according to
+    # https://docs.microsoft.com/en-us/cpp/c-language/parsing-c-command-line-arguments and
+    # https://blogs.msdn.microsoft.com/twistylittlepassagesallalike/2011/04/23/everyone-quotes-command-line-arguments-the-wrong-way/
+
+    _whitespace = ' \t\n\r'
+    _find_unsafe_char = re.compile(r'[{}"]'.format(_whitespace)).search
+
+    def quote_arg(arg):
+        if arg and not _find_unsafe_char(arg):
+            return arg
+
+        result = '"'
+        num_backslashes = 0
+        for c in arg:
+            if c == '\\':
+                num_backslashes += 1
+            else:
+                if c == '"':
+                    # Escape all backslashes and the following double quotation mark
+                    num_backslashes = num_backslashes * 2 + 1
+
+                result += num_backslashes * '\\' + c
+                num_backslashes = 0
+
+        # Escape all backslashes, but let the terminating double quotation
+        # mark we add below be interpreted as a metacharacter
+        result += (num_backslashes * 2) * '\\' + '"'
+        return result
+
+    def split_args(cmd):
+        result = []
+        arg = ''
+        num_backslashes = 0
+        num_quotes = 0
+        in_quotes = False
+        for c in cmd:
+            if c == '\\':
+                num_backslashes += 1
+            else:
+                if c == '"' and not (num_backslashes % 2):
+                    # unescaped quote, eat it
+                    arg += (num_backslashes // 2) * '\\'
+                    num_quotes += 1
+                    in_quotes = not in_quotes
+                elif c in _whitespace and not in_quotes:
+                    if arg or num_quotes:
+                        # reached the end of the argument
+                        result.append(arg)
+                        arg = ''
+                        num_quotes = 0
+                else:
+                    if c == '"':
+                        # escaped quote
+                        num_backslashes = (num_backslashes - 1) // 2
+
+                    arg += num_backslashes * '\\' + c
+
+                num_backslashes = 0
+
+        if arg or num_quotes:
+            result.append(arg)
+
+        return result
+else:
+    def quote_arg(arg):
+        return shlex.quote(arg)
+
+    def split_args(cmd):
+        return shlex.split(cmd)
+
+
+def join_args(args):
+    return ' '.join([quote_arg(x) for x in args])
+
+
 def do_replacement(regex, line, variable_format, confdata):
     missing_variables = set()
     start_tag = '@'
diff --git a/mesonbuild/modules/gnome.py b/mesonbuild/modules/gnome.py
index 4e97d3a75..5c9d3dd29 100644
--- a/mesonbuild/modules/gnome.py
+++ b/mesonbuild/modules/gnome.py
@@ -17,7 +17,6 @@ functionality such as gobject-introspection, gresources and gtk-doc'''
 
 import os
 import copy
-import shlex
 import subprocess
 
 from .. import build
@@ -29,7 +28,7 @@ from . import get_include_args
 from . import ExtensionModule
 from . import ModuleReturnValue
 from ..mesonlib import (
-    MachineChoice, MesonException, OrderedSet, Popen_safe, extract_as_list
+    MachineChoice, MesonException, OrderedSet, Popen_safe, extract_as_list, join_args
 )
 from ..dependencies import Dependency, PkgConfigDependency, InternalDependency
 from ..interpreterbase import noKwargs, permittedKwargs, FeatureNew, FeatureNewKwargs
@@ -1079,12 +1078,12 @@ This will become a hard error in the future.''')
         ldflags.extend(compiler_flags[1])
         ldflags.extend(compiler_flags[2])
         if compiler:
-            args += ['--cc=%s' % ' '.join([shlex.quote(x) for x in compiler.get_exelist()])]
-            args += ['--ld=%s' % ' '.join([shlex.quote(x) for x in compiler.get_linker_exelist()])]
+            args += ['--cc=%s' % join_args(compiler.get_exelist())]
+            args += ['--ld=%s' % join_args(compiler.get_linker_exelist())]
         if cflags:
-            args += ['--cflags=%s' % ' '.join([shlex.quote(x) for x in cflags])]
+            args += ['--cflags=%s' % join_args(cflags)]
         if ldflags:
-            args += ['--ldflags=%s' % ' '.join([shlex.quote(x) for x in ldflags])]
+            args += ['--ldflags=%s' % join_args(ldflags)]
 
         return args
 
diff --git a/mesonbuild/modules/pkgconfig.py b/mesonbuild/modules/pkgconfig.py
index 78fffb1c6..60d4b3f0d 100644
--- a/mesonbuild/modules/pkgconfig.py
+++ b/mesonbuild/modules/pkgconfig.py
@@ -240,7 +240,7 @@ class PkgConfigModule(ExtensionModule):
 
     def _escape(self, value):
         '''
-        We cannot use shlex.quote because it quotes with ' and " which does not
+        We cannot use quote_arg because it quotes with ' and " which does not
         work with pkg-config and pkgconf at all.
         '''
         # We should always write out paths with / because pkg-config requires
diff --git a/mesonbuild/mtest.py b/mesonbuild/mtest.py
index e11c8e47c..70585f48e 100644
--- a/mesonbuild/mtest.py
+++ b/mesonbuild/mtest.py
@@ -29,7 +29,6 @@ import pickle
 import platform
 import random
 import re
-import shlex
 import signal
 import subprocess
 import sys
@@ -41,7 +40,7 @@ from . import build
 from . import environment
 from . import mlog
 from .dependencies import ExternalProgram
-from .mesonlib import MesonException, get_wine_shortpath
+from .mesonlib import MesonException, get_wine_shortpath, split_args
 
 if typing.TYPE_CHECKING:
     from .backend.backends import TestSerialisation
@@ -88,7 +87,7 @@ def add_arguments(parser: argparse.ArgumentParser) -> None:
                         help='Run test under gdb.')
     parser.add_argument('--list', default=False, dest='list', action='store_true',
                         help='List available tests.')
-    parser.add_argument('--wrapper', default=None, dest='wrapper', type=shlex.split,
+    parser.add_argument('--wrapper', default=None, dest='wrapper', type=split_args,
                         help='wrapper to run tests with (e.g. Valgrind)')
     parser.add_argument('-C', default='.', dest='wd',
                         help='directory to cd into before running')
@@ -116,7 +115,7 @@ def add_arguments(parser: argparse.ArgumentParser) -> None:
                         ' more time to execute.')
     parser.add_argument('--setup', default=None, dest='setup',
                         help='Which test setup to use.')
-    parser.add_argument('--test-args', default=[], type=shlex.split,
+    parser.add_argument('--test-args', default=[], type=split_args,
                         help='Arguments to pass to the specified test(s) or all tests')
     parser.add_argument('args', nargs='*',
                         help='Optional list of tests to run')
diff --git a/mesonbuild/scripts/gtkdochelper.py b/mesonbuild/scripts/gtkdochelper.py
index 4998e17ed..ddcc8c0bc 100644
--- a/mesonbuild/scripts/gtkdochelper.py
+++ b/mesonbuild/scripts/gtkdochelper.py
@@ -14,10 +14,9 @@
 
 import sys, os
 import subprocess
-import shlex
 import shutil
 import argparse
-from ..mesonlib import MesonException, Popen_safe, is_windows
+from ..mesonlib import MesonException, Popen_safe, is_windows, split_args
 from . import destdir_join
 
 parser = argparse.ArgumentParser()
@@ -149,7 +148,7 @@ def build_gtkdoc(source_root, build_root, doc_subdir, src_subdirs,
                          '--output-dir=' + abs_out]
 
         library_paths = []
-        for ldflag in shlex.split(ldflags):
+        for ldflag in split_args(ldflags):
             if ldflag.startswith('-Wl,-rpath,'):
                 library_paths.append(ldflag[11:])
 
diff --git a/mesonbuild/scripts/scanbuild.py b/mesonbuild/scripts/scanbuild.py
index 51f70f0c3..8c0f423d3 100644
--- a/mesonbuild/scripts/scanbuild.py
+++ b/mesonbuild/scripts/scanbuild.py
@@ -13,12 +13,11 @@
 # limitations under the License.
 
 import os
-import shlex
 import subprocess
 import shutil
 import tempfile
 from ..environment import detect_ninja
-from ..mesonlib import Popen_safe
+from ..mesonlib import Popen_safe, split_args
 
 def scanbuild(exelist, srcdir, blddir, privdir, logdir, args):
     with tempfile.TemporaryDirectory(dir=privdir) as scandir:
@@ -63,7 +62,7 @@ def run(args):
             break
 
     if 'SCANBUILD' in os.environ:
-        exelist = shlex.split(os.environ['SCANBUILD'])
+        exelist = split_args(os.environ['SCANBUILD'])
     else:
         exelist = [toolname]
 
diff --git a/run_tests.py b/run_tests.py
index 06a072736..85a55c313 100755
--- a/run_tests.py
+++ b/run_tests.py
@@ -117,7 +117,7 @@ Backend = Enum('Backend', 'ninja vs xcode')
 
 if 'MESON_EXE' in os.environ:
     import shlex
-    meson_exe = shlex.split(os.environ['MESON_EXE'])
+    meson_exe = mesonlib.split_args(os.environ['MESON_EXE'])
 else:
     meson_exe = None
 
diff --git a/run_unittests.py b/run_unittests.py
index dd5b4341d..f9ba01763 100755
--- a/run_unittests.py
+++ b/run_unittests.py
@@ -14,7 +14,6 @@
 # limitations under the License.
 
 import stat
-import shlex
 import subprocess
 import re
 import json
@@ -51,7 +50,7 @@ from mesonbuild.ast import AstInterpreter
 from mesonbuild.mesonlib import (
     BuildDirLock, LibType, MachineChoice, PerMachine, Version,
     is_windows, is_osx, is_cygwin, is_dragonflybsd, is_openbsd, is_haiku,
-    windows_proof_rmtree, python_command, version_compare,
+    windows_proof_rmtree, python_command, version_compare, split_args, quote_arg
 )
 from mesonbuild.environment import detect_ninja
 from mesonbuild.mesonlib import MesonException, EnvironmentException
@@ -1022,6 +1021,103 @@ class InternalTests(unittest.TestCase):
         self.assertTrue(vctools_ver.startswith(toolset_ver),
                         msg='{!r} does not start with {!r}'.format(vctools_ver, toolset_ver))
 
+    def test_split_args(self):
+        split_args = mesonbuild.mesonlib.split_args
+        join_args = mesonbuild.mesonlib.join_args
+        if is_windows():
+            test_data = [
+                # examples from https://docs.microsoft.com/en-us/cpp/c-language/parsing-c-command-line-arguments
+                (r'"a b c" d e', ['a b c', 'd', 'e'], True),
+                (r'"ab\"c" "\\" d', ['ab"c', '\\', 'd'], False),
+                (r'a\\\b d"e f"g h', [r'a\\\b', 'de fg', 'h'], False),
+                (r'a\\\"b c d', [r'a\"b', 'c', 'd'], False),
+                (r'a\\\\"b c" d e', [r'a\\b c', 'd', 'e'], False),
+                # other basics
+                (r'""', [''], True),
+                (r'a b c d "" e', ['a', 'b', 'c', 'd', '', 'e'], True),
+                (r"'a b c' d e", ["'a", 'b', "c'", 'd', 'e'], True),
+                (r"'a&b&c' d e", ["'a&b&c'", 'd', 'e'], True),
+                (r"a & b & c d e", ['a', '&', 'b', '&', 'c', 'd', 'e'], True),
+                (r"'a & b & c d e'", ["'a", '&', 'b', '&', 'c', 'd', "e'"], True),
+                ('a  b\nc\rd \n\re', ['a', 'b', 'c', 'd', 'e'], False),
+                # more illustrative tests
+                (r'cl test.cpp /O1 /Fe:test.exe', ['cl', 'test.cpp', '/O1', '/Fe:test.exe'], True),
+                (r'cl "test.cpp /O1 /Fe:test.exe"', ['cl', 'test.cpp /O1 /Fe:test.exe'], True),
+                (r'cl /DNAME=\"Bob\" test.cpp', ['cl', '/DNAME="Bob"', 'test.cpp'], False),
+                (r'cl "/DNAME=\"Bob\"" test.cpp', ['cl', '/DNAME="Bob"', 'test.cpp'], True),
+                (r'cl /DNAME=\"Bob, Alice\" test.cpp', ['cl', '/DNAME="Bob,', 'Alice"', 'test.cpp'], False),
+                (r'cl "/DNAME=\"Bob, Alice\"" test.cpp', ['cl', '/DNAME="Bob, Alice"', 'test.cpp'], True),
+                (r'cl C:\path\with\backslashes.cpp', ['cl', r'C:\path\with\backslashes.cpp'], True),
+                (r'cl C:\\path\\with\\double\\backslashes.cpp', ['cl', r'C:\\path\\with\\double\\backslashes.cpp'], True),
+                (r'cl "C:\\path\\with\\double\\backslashes.cpp"', ['cl', r'C:\\path\\with\\double\\backslashes.cpp'], False),
+                (r'cl C:\path with spaces\test.cpp', ['cl', r'C:\path', 'with', r'spaces\test.cpp'], False),
+                (r'cl "C:\path with spaces\test.cpp"', ['cl', r'C:\path with spaces\test.cpp'], True),
+                (r'cl /DPATH="C:\path\with\backslashes test.cpp', ['cl', r'/DPATH=C:\path\with\backslashes test.cpp'], False),
+                (r'cl /DPATH=\"C:\\ends\\with\\backslashes\\\" test.cpp', ['cl', r'/DPATH="C:\\ends\\with\\backslashes\"', 'test.cpp'], False),
+                (r'cl /DPATH="C:\\ends\\with\\backslashes\\" test.cpp', ['cl', '/DPATH=C:\\\\ends\\\\with\\\\backslashes\\', 'test.cpp'], False),
+                (r'cl "/DNAME=\"C:\\ends\\with\\backslashes\\\"" test.cpp', ['cl', r'/DNAME="C:\\ends\\with\\backslashes\"', 'test.cpp'], True),
+                (r'cl "/DNAME=\"C:\\ends\\with\\backslashes\\\\"" test.cpp', ['cl', r'/DNAME="C:\\ends\\with\\backslashes\\ test.cpp'], False),
+                (r'cl "/DNAME=\"C:\\ends\\with\\backslashes\\\\\"" test.cpp', ['cl', r'/DNAME="C:\\ends\\with\\backslashes\\"', 'test.cpp'], True),
+            ]
+        else:
+            test_data = [
+                (r"'a b c' d e", ['a b c', 'd', 'e'], True),
+                (r"a/b/c d e", ['a/b/c', 'd', 'e'], True),
+                (r"a\b\c d e", [r'abc', 'd', 'e'], False),
+                (r"a\\b\\c d e", [r'a\b\c', 'd', 'e'], False),
+                (r'"a b c" d e', ['a b c', 'd', 'e'], False),
+                (r'"a\\b\\c\\" d e', ['a\\b\\c\\', 'd', 'e'], False),
+                (r"'a\b\c\' d e", ['a\\b\\c\\', 'd', 'e'], True),
+                (r"'a&b&c' d e", ['a&b&c', 'd', 'e'], True),
+                (r"a & b & c d e", ['a', '&', 'b', '&', 'c', 'd', 'e'], False),
+                (r"'a & b & c d e'", ['a & b & c d e'], True),
+                (r"abd'e f'g h", [r'abde fg', 'h'], False),
+                ('a  b\nc\rd \n\re', ['a', 'b', 'c', 'd', 'e'], False),
+
+                ('g++ -DNAME="Bob" test.cpp', ['g++', '-DNAME=Bob', 'test.cpp'], False),
+                ("g++ '-DNAME=\"Bob\"' test.cpp", ['g++', '-DNAME="Bob"', 'test.cpp'], True),
+                ('g++ -DNAME="Bob, Alice" test.cpp', ['g++', '-DNAME=Bob, Alice', 'test.cpp'], False),
+                ("g++ '-DNAME=\"Bob, Alice\"' test.cpp", ['g++', '-DNAME="Bob, Alice"', 'test.cpp'], True),
+            ]
+
+        for (cmd, expected, roundtrip) in test_data:
+            self.assertEqual(split_args(cmd), expected)
+            if roundtrip:
+                self.assertEqual(join_args(expected), cmd)
+
+    def test_quote_arg(self):
+        split_args = mesonbuild.mesonlib.split_args
+        quote_arg = mesonbuild.mesonlib.quote_arg
+        if is_windows():
+            test_data = [
+                ('', '""'),
+                ('arg1', 'arg1'),
+                ('/option1', '/option1'),
+                ('/Ovalue', '/Ovalue'),
+                ('/OBob&Alice', '/OBob&Alice'),
+                ('/Ovalue with spaces', r'"/Ovalue with spaces"'),
+                (r'/O"value with spaces"', r'"/O\"value with spaces\""'),
+                (r'/OC:\path with spaces\test.exe', r'"/OC:\path with spaces\test.exe"'),
+                ('/LIBPATH:C:\\path with spaces\\ends\\with\\backslashes\\', r'"/LIBPATH:C:\path with spaces\ends\with\backslashes\\"'),
+                ('/LIBPATH:"C:\\path with spaces\\ends\\with\\backslashes\\\\"', r'"/LIBPATH:\"C:\path with spaces\ends\with\backslashes\\\\\""'),
+                (r'/DMSG="Alice said: \"Let\'s go\""', r'"/DMSG=\"Alice said: \\\"Let\'s go\\\"\""'),
+            ]
+        else:
+            test_data = [
+                ('arg1', 'arg1'),
+                ('--option1', '--option1'),
+                ('-O=value', '-O=value'),
+                ('-O=Bob&Alice', "'-O=Bob&Alice'"),
+                ('-O=value with spaces', "'-O=value with spaces'"),
+                ('-O="value with spaces"', '\'-O=\"value with spaces\"\''),
+                ('-O=/path with spaces/test', '\'-O=/path with spaces/test\''),
+                ('-DMSG="Alice said: \\"Let\'s go\\""', "'-DMSG=\"Alice said: \\\"Let'\"'\"'s go\\\"\"'"),
+            ]
+
+        for (arg, expected) in test_data:
+            self.assertEqual(quote_arg(arg), expected)
+            self.assertEqual(split_args(expected)[0], arg)
+
 
 @unittest.skipIf(is_tarball(), 'Skipping because this is a tarball release')
 class DataTests(unittest.TestCase):
@@ -2033,7 +2129,7 @@ class AllPlatformTests(BasePlatformTests):
         if not execmd or not fxecmd:
             raise Exception('Could not find someexe and somfxe commands')
         # Check include order for 'someexe'
-        incs = [a for a in shlex.split(execmd) if a.startswith("-I")]
+        incs = [a for a in split_args(execmd) if a.startswith("-I")]
         self.assertEqual(len(incs), 9)
         # target private dir
         someexe_id = Target.construct_id_from_path("sub4", "someexe", "@exe")
@@ -2055,7 +2151,7 @@ class AllPlatformTests(BasePlatformTests):
         # custom target include dir
         self.assertPathEqual(incs[8], '-Ictsub')
         # Check include order for 'somefxe'
-        incs = [a for a in shlex.split(fxecmd) if a.startswith('-I')]
+        incs = [a for a in split_args(fxecmd) if a.startswith('-I')]
         self.assertEqual(len(incs), 9)
         # target private dir
         self.assertPathEqual(incs[0], '-Isomefxe@exe')
@@ -2123,7 +2219,7 @@ class AllPlatformTests(BasePlatformTests):
                 else:
                     raise AssertionError('Unknown compiler {!r}'.format(evalue))
                 # Check that we actually used the evalue correctly as the compiler
-                self.assertEqual(ecc.get_exelist(), shlex.split(evalue))
+                self.assertEqual(ecc.get_exelist(), split_args(evalue))
             # Do auto-detection of compiler based on platform, PATH, etc.
             cc = getattr(env, 'detect_{}_compiler'.format(lang))(MachineChoice.HOST)
             self.assertTrue(cc.version)
@@ -2174,14 +2270,14 @@ class AllPlatformTests(BasePlatformTests):
             wrappercc = python_command + [wrapper] + cc.get_exelist() + ['-DSOME_ARG']
             wrappercc_s = ''
             for w in wrappercc:
-                wrappercc_s += shlex.quote(w) + ' '
+                wrappercc_s += quote_arg(w) + ' '
             os.environ[evar] = wrappercc_s
             wcc = getattr(env, 'detect_{}_compiler'.format(lang))(MachineChoice.HOST)
             # Check static linker too
             wrapperlinker = python_command + [wrapper] + linker.get_exelist() + linker.get_always_args()
             wrapperlinker_s = ''
             for w in wrapperlinker:
-                wrapperlinker_s += shlex.quote(w) + ' '
+                wrapperlinker_s += quote_arg(w) + ' '
             os.environ['AR'] = wrapperlinker_s
             wlinker = env.detect_static_linker(wcc)
             # Pop it so we don't use it for the next detection
@@ -2207,7 +2303,7 @@ class AllPlatformTests(BasePlatformTests):
         commands = {'c-asm': {}, 'cpp-asm': {}, 'cpp-c-asm': {}, 'c-cpp-asm': {}}
         for cmd in self.get_compdb():
             # Get compiler
-            split = shlex.split(cmd['command'])
+            split = split_args(cmd['command'])
             if split[0] == 'ccache':
                 compiler = split[1]
             else:
@@ -2272,7 +2368,7 @@ class AllPlatformTests(BasePlatformTests):
         define = 'MESON_TEST_DEFINE_VALUE'
         # NOTE: this list can't have \n, ' or "
         # \n is never substituted by the GNU pre-processor via a -D define
-        # ' and " confuse shlex.split() even when they are escaped
+        # ' and " confuse split_args() even when they are escaped
         # % and # confuse the MSVC preprocessor
         # !, ^, *, and < confuse lcc preprocessor
         value = 'spaces and fun@$&()-=_+{}[]:;>?,./~`'
@@ -3154,7 +3250,7 @@ recommended as it is not supported on some platforms''')
         self.assertEqual(obj.user_options['subp:subp_opt'].value, 'foo')
         self.wipe()
 
-        # c_args value should be parsed with shlex
+        # c_args value should be parsed with split_args
         self.init(testdir, extra_args=['-Dc_args=-Dfoo -Dbar "-Dthird=one two"'])
         obj = mesonbuild.coredata.load(self.builddir)
         self.assertEqual(obj.compiler_options.host['c_args'].value, ['-Dfoo', '-Dbar', '-Dthird=one two'])