wafsamba: introduce require_builtin_deps/provide_builtin_linking/builtin_cflags to SAMBA_{SUBSYSTEM,LIBRARY}

The 'provide_builtin_linking=True' option that allows wscript files
to specify that a SAMBA_{SUBSYSTEM,LIBRARY} will also create a
builtin version of them in addition.

The logic behind this is very similar to what we already have with the
'--builtin-libraries=BUILTIN_LIBRARIES' configure option.

This avoids the need for manual definitions of SAMBA_SUBSYSTEMS() with
like this:

   bld.SAMBA_SUBSYSTEM('replace-hidden',
                       source=REPLACE_SOURCE,
                       group='base_libraries',
                       hide_symbols=True,
                       deps='dl attr' + extra_libs)

The builtin version will also make sure that it will include all
dependecies (of internal code) also in the builtin variant.
Note that this is also possible if the dependency also
provided 'provide_builtin_linking=True' in order to limit
the scope.

We now imply '-D_PUBLIC_=_PRIVATE_' and 'hide_symbols=True' for
builtin libraries and subsystems in order to avoid exporting
the symbols of them.

With 'require_builtin_deps=True' a library can specify that it
is only able to use libraries/subsystems marked with
provide_builtin_linking=True. As a result it won't
link against any other SAMBA_LIBRARY() dependency,
but link in everything internal. Only system libraries
still get linked dynamically.

Use 'git show -w' to see a reduced diff.

BUG: https://bugzilla.samba.org/show_bug.cgi?id=14780

Signed-off-by: Stefan Metzmacher <metze@samba.org>
Reviewed-by: Andrew Bartlett <abartlet@samba.org>
Reviewed-by: Andreas Schneider <asn@samba.org>

4 files changed, 261 insertions, 39 deletions

diff --git a/buildtools/wafsamba/samba_deps.py b/buildtools/wafsamba/samba_deps.py
index 8d76e2e434e..5255d39e3ba 100644
--- a/buildtools/wafsamba/samba_deps.py
+++ b/buildtools/wafsamba/samba_deps.py
@@ -7,7 +7,6 @@ from waflib.Logs import debug
 from waflib.Configure import conf
 from waflib import ConfigSet
 
-from samba_bundled import BUILTIN_LIBRARY
 from samba_utils import LOCAL_CACHE, TO_LIST, get_tgt_list, unique_list
 from samba_autoconf import library_flags
 
@@ -102,7 +101,7 @@ def build_dependencies(self):
         debug('deps: computed dependencies for target %s: uselib=%s uselib_local=%s add_objects=%s',
               self.sname, self.uselib, self.uselib_local, self.add_objects)
 
-    if self.samba_type in ['SUBSYSTEM']:
+    if self.samba_type in ['SUBSYSTEM', 'BUILTIN']:
         # this is needed for the cflags of libs that come from pkg_config
         self.uselib = list(self.final_syslibs)
         self.uselib.extend(list(self.direct_syslibs))
@@ -354,7 +353,7 @@ def show_final_deps(bld, tgt_list):
     targets = LOCAL_CACHE(bld, 'TARGET_TYPE')
 
     for t in tgt_list:
-        if not targets[t.sname] in ['LIBRARY', 'BINARY', 'PYTHON', 'SUBSYSTEM']:
+        if not targets[t.sname] in ['LIBRARY', 'BINARY', 'PYTHON', 'SUBSYSTEM', 'BUILTIN']:
             continue
         debug('deps: final dependencies for target %s: uselib=%s uselib_local=%s add_objects=%s',
               t.sname, t.uselib, getattr(t, 'uselib_local', []), getattr(t, 'add_objects', []))
@@ -376,6 +375,58 @@ def add_samba_attributes(bld, tgt_list):
         t.samba_includes_extended = TO_LIST(t.samba_includes)[:]
         t.cflags = getattr(t, 'samba_cflags', '')
 
+def replace_builtin_subsystem_deps(bld, tgt_list):
+    '''replace dependencies based on builtin subsystems/libraries
+
+    '''
+
+    targets  = LOCAL_CACHE(bld, 'TARGET_TYPE')
+
+    # If either the target or the dependency require builtin linking
+    # we should replace the dependency
+    for t in tgt_list:
+        t_require_builtin_deps = getattr(t, 'samba_require_builtin_deps', False)
+        if t_require_builtin_deps:
+            debug("deps: target %s: requires builtin dependencies..." % (t.sname))
+        else:
+            debug("deps: target %s: does not require builtin dependencies..." % (t.sname))
+
+        replacing = {}
+
+        for dep in t.samba_deps_extended:
+            bld.ASSERT(dep in targets, "target %s: dependency target %s not declared" % (t.sname, dep))
+            dtype = targets[dep]
+            bld.ASSERT(dtype != 'BUILTIN', "target %s: dependency target %s is BUILTIN" % (t.sname, dep))
+            bld.ASSERT(dtype != 'PLUGIN', "target %s: dependency target %s is PLUGIN" % (t.sname, dep))
+            if dtype not in ['SUBSYSTEM', 'LIBRARY']:
+                debug("deps: target %s: keep %s dependency %s" % (t.sname, dtype, dep))
+                continue
+            dt = bld.get_tgen_by_name(dep)
+            bld.ASSERT(dt is not None, "target %s: dependency target %s not found by name" % (t.sname, dep))
+            dt_require_builtin_deps = getattr(dt, 'samba_require_builtin_deps', False)
+            if not dt_require_builtin_deps and not t_require_builtin_deps:
+                # both target and dependency don't require builtin linking
+                continue
+            sdt = getattr(dt, 'samba_builtin_subsystem', None)
+            if not t_require_builtin_deps:
+                if sdt is None:
+                    debug("deps: target %s: dependency %s requires builtin deps only" % (t.sname, dep))
+                    continue
+                debug("deps: target %s: dependency %s requires builtin linking" % (t.sname, dep))
+            bld.ASSERT(sdt is not None, "target %s: dependency target %s is missing samba_builtin_subsystem" % (t.sname, dep))
+            sdep = sdt.sname
+            bld.ASSERT(sdep in targets, "target %s: builtin dependency target %s (from %s) not declared" % (t.sname, sdep, dep))
+            sdt = targets[sdep]
+            bld.ASSERT(sdt == 'BUILTIN', "target %s: builtin dependency target %s (from %s) is not BUILTIN" % (t.sname, sdep, dep))
+            replacing[dep] = sdep
+
+        for i in range(len(t.samba_deps_extended)):
+            dep = t.samba_deps_extended[i]
+            if dep in replacing:
+                sdep = replacing[dep]
+                debug("deps: target %s: replacing dependency %s with builtin subsystem %s" % (t.sname, dep, sdep))
+                t.samba_deps_extended[i] = sdep
+
 def replace_grouping_libraries(bld, tgt_list):
     '''replace dependencies based on grouping libraries
 
@@ -446,7 +497,12 @@ def build_direct_deps(bld, tgt_list):
                 t.direct_syslibs.add(d)
                 if d in syslib_deps:
                     for implied in TO_LIST(syslib_deps[d]):
-                        if BUILTIN_LIBRARY(bld, implied):
+                        if targets[implied] == 'SUBSYSTEM':
+                            it = bld.get_tgen_by_name(implied)
+                            sit = getattr(it, 'samba_builtin_subsystem', None)
+                            if sit:
+                                implied = sit.sname
+                        if targets[implied] == 'BUILTIN':
                             t.direct_objects.add(implied)
                         elif targets[implied] == 'SYSLIB':
                             t.direct_syslibs.add(implied)
@@ -463,8 +519,9 @@ def build_direct_deps(bld, tgt_list):
                 sys.exit(1)
             if t2.samba_type in [ 'LIBRARY', 'MODULE' ]:
                 t.direct_libs.add(d)
-            elif t2.samba_type in [ 'SUBSYSTEM', 'ASN1', 'PYTHON' ]:
+            elif t2.samba_type in [ 'SUBSYSTEM', 'BUILTIN', 'ASN1', 'PYTHON' ]:
                 t.direct_objects.add(d)
+
     debug('deps: built direct dependencies')
 
 
@@ -654,7 +711,7 @@ def break_dependency_loops(bld, tgt_list):
     # expand indirect subsystem and library loops
     for loop in loops.copy():
         t = bld.get_tgen_by_name(loop)
-        if t.samba_type in ['SUBSYSTEM']:
+        if t.samba_type in ['SUBSYSTEM', 'BUILTIN']:
             loops[loop] = loops[loop].union(t.indirect_objects)
             loops[loop] = loops[loop].union(t.direct_objects)
         if t.samba_type in ['LIBRARY','PYTHON']:
@@ -698,6 +755,8 @@ def break_dependency_loops(bld, tgt_list):
 
 def reduce_objects(bld, tgt_list):
     '''reduce objects by looking for indirect object dependencies'''
+    targets  = LOCAL_CACHE(bld, 'TARGET_TYPE')
+
     rely_on = {}
 
     for t in tgt_list:
@@ -719,8 +778,9 @@ def reduce_objects(bld, tgt_list):
                 if dup:
                     # Do not remove duplicates of BUILTINS
                     for d in iter(dup.copy()):
-                        if BUILTIN_LIBRARY(bld, d):
-                            debug('deps: BUILTIN_LIBRARY SKIP: removing dups from %s of type %s: %s also in %s %s',
+                        dtype = targets[d]
+                        if dtype == 'BUILTIN':
+                            debug('deps: BUILTIN SKIP: removing dups from %s of type %s: %s also in %s %s',
                                   t.sname, t.samba_type, d, t2.samba_type, l)
                             dup.remove(d)
                     if len(dup) == 0:
@@ -733,6 +793,19 @@ def reduce_objects(bld, tgt_list):
                     if not l in rely_on:
                         rely_on[l] = set()
                     rely_on[l] = rely_on[l].union(dup)
+            for n in iter(new.copy()):
+                # if we got the builtin version as well
+                # as the native one, we keep using the
+                # builtin one and remove the rest.
+                # Otherwise our check_duplicate_sources()
+                # checks would trigger!
+                if n.endswith('.builtin.objlist'):
+                    unused = n.replace('.builtin.objlist', '.objlist')
+                    if unused in new:
+                        new.remove(unused)
+                    unused = n.replace('.builtin.objlist', '')
+                    if unused in new:
+                        new.remove(unused)
             t.final_objects = new
 
     if not changed:
@@ -871,7 +944,7 @@ def calculate_final_deps(bld, tgt_list, loops):
 
     # add in any syslib dependencies
     for t in tgt_list:
-        if not t.samba_type in ['BINARY','PYTHON','LIBRARY','SUBSYSTEM']:
+        if not t.samba_type in ['BINARY','PYTHON','LIBRARY','SUBSYSTEM','BUILTIN']:
             continue
         syslibs = set()
         for d in t.final_objects:
@@ -1114,6 +1187,10 @@ def check_project_rules(bld):
 
     debug('deps: project rules checking started')
 
+    replace_builtin_subsystem_deps(bld, tgt_list)
+
+    debug("deps: replace_builtin_subsystem_deps: %s" % str(timer))
+
     expand_subsystem_deps(bld)
 
     debug("deps: expand_subsystem_deps: %s" % str(timer))
diff --git a/buildtools/wafsamba/samba_utils.py b/buildtools/wafsamba/samba_utils.py
index e08b55cf71d..cb13746a2c5 100644
--- a/buildtools/wafsamba/samba_utils.py
+++ b/buildtools/wafsamba/samba_utils.py
@@ -658,7 +658,7 @@ def get_tgt_list(bld):
     tgt_list = []
     for tgt in targets:
         type = targets[tgt]
-        if not type in ['SUBSYSTEM', 'MODULE', 'BINARY', 'LIBRARY', 'ASN1', 'PYTHON']:
+        if not type in ['SUBSYSTEM', 'BUILTIN', 'MODULE', 'BINARY', 'LIBRARY', 'ASN1', 'PYTHON']:
             continue
         t = bld.get_tgen_by_name(tgt)
         if t is None:
diff --git a/buildtools/wafsamba/symbols.py b/buildtools/wafsamba/symbols.py
index d3bf9ac1c6b..f1e70c80d18 100644
--- a/buildtools/wafsamba/symbols.py
+++ b/buildtools/wafsamba/symbols.py
@@ -389,7 +389,7 @@ def build_library_names(bld, tgt_list):
         if t.samba_type in [ 'LIBRARY' ]:
             for obj in t.samba_deps_extended:
                 t2 = bld.get_tgen_by_name(obj)
-                if t2 and t2.samba_type in [ 'SUBSYSTEM', 'ASN1' ]:
+                if t2 and t2.samba_type in [ 'SUBSYSTEM', 'BUILTIN', 'ASN1' ]:
                     if not t.sname in t2.in_library:
                         t2.in_library.append(t.sname)
     bld.env.done_build_library_names = True
diff --git a/buildtools/wafsamba/wafsamba.py b/buildtools/wafsamba/wafsamba.py
index 4d7b216d683..4a1a00c62c0 100644
--- a/buildtools/wafsamba/wafsamba.py
+++ b/buildtools/wafsamba/wafsamba.py
@@ -130,11 +130,29 @@ def SAMBA_LIBRARY(bld, libname, source,
                   manpages=None,
                   private_library=False,
                   grouping_library=False,
+                  require_builtin_deps=False,
+                  provide_builtin_linking=False,
+                  builtin_cflags='',
                   allow_undefined_symbols=False,
                   allow_warnings=False,
                   enabled=True):
     '''define a Samba library'''
 
+    # We support:
+    # - LIBRARY: this can be use to link via -llibname
+    # - MODULE:  this is module from SAMBA_MODULE()
+    # - PYTHON:  a python C binding library
+    #
+    if target_type not in ['LIBRARY', 'MODULE', 'PYTHON']:
+        raise Errors.WafError("target_type[%s] not supported in SAMBA_LIBRARY('%s')" %
+                              (target_type, libname))
+
+    if require_builtin_deps:
+        # For now we only support require_builtin_deps only for libraries
+        if target_type not in ['LIBRARY']:
+            raise Errors.WafError("target_type[%s] not supported SAMBA_LIBRARY('%s', require_builtin_deps=True)" %
+                                  (target_type, libname))
+
     if private_library and public_headers:
         raise Errors.WafError("private library '%s' must not have public header files" %
                              libname)
@@ -161,10 +179,26 @@ def SAMBA_LIBRARY(bld, libname, source,
                             target=empty_c)
         source=empty_c
 
+    samba_deps = deps + ' ' + public_deps
+    samba_deps = TO_LIST(samba_deps)
+
     if BUILTIN_LIBRARY(bld, libname):
-        obj_target = libname
+        builtin_target = libname + '.builtin.objlist'
+        builtin_cflags_end = '-D_PUBLIC_=_PRIVATE_'
+        empty_target = libname
+        obj_target = None
     else:
+        if provide_builtin_linking:
+            builtin_target = libname + '.builtin.objlist'
+            builtin_cflags_end = '-D_PUBLIC_=_PRIVATE_'
+        else:
+            builtin_target = None
+        empty_target = None
         obj_target = libname + '.objlist'
+        if require_builtin_deps:
+            # hide the builtin deps from the callers
+            samba_deps = TO_LIST('')
+        dep_target = obj_target
 
     if group == 'libraries':
         subsystem_group = 'main'
@@ -174,27 +208,51 @@ def SAMBA_LIBRARY(bld, libname, source,
     # first create a target for building the object files for this library
     # by separating in this way, we avoid recompiling the C files
     # separately for the install library and the build library
-    bld.SAMBA_SUBSYSTEM(obj_target,
-                        source         = source,
-                        deps           = deps,
-                        public_deps    = public_deps,
-                        includes       = includes,
-                        public_headers = public_headers,
-                        public_headers_install = public_headers_install,
-                        private_headers= private_headers,
-                        header_path    = header_path,
-                        cflags         = cflags,
-                        cflags_end     = cflags_end,
-                        group          = subsystem_group,
-                        autoproto      = autoproto,
-                        autoproto_extra_source=autoproto_extra_source,
-                        depends_on     = depends_on,
-                        hide_symbols   = hide_symbols,
-                        allow_warnings = allow_warnings,
-                        pyembed        = pyembed,
-                        pyext          = pyext,
-                        local_include  = local_include,
-                        global_include = global_include)
+    if builtin_target:
+        __t = __SAMBA_SUBSYSTEM_BUILTIN(bld, builtin_target, source,
+                                        deps=deps,
+                                        public_deps=public_deps,
+                                        includes=includes,
+                                        header_path=header_path,
+                                        builtin_cflags=builtin_cflags,
+                                        builtin_cflags_end=builtin_cflags_end,
+                                        group=group,
+                                        depends_on=depends_on,
+                                        local_include=local_include,
+                                        global_include=global_include,
+                                        allow_warnings=allow_warnings)
+        builtin_subsystem = __t
+    else:
+        builtin_subsystem = None
+    if obj_target:
+        bld.SAMBA_SUBSYSTEM(obj_target,
+                            source         = source,
+                            deps           = deps,
+                            public_deps    = public_deps,
+                            includes       = includes,
+                            public_headers = public_headers,
+                            public_headers_install = public_headers_install,
+                            private_headers= private_headers,
+                            header_path    = header_path,
+                            cflags         = cflags,
+                            cflags_end     = cflags_end,
+                            group          = subsystem_group,
+                            autoproto      = autoproto,
+                            autoproto_extra_source=autoproto_extra_source,
+                            depends_on     = depends_on,
+                            hide_symbols   = hide_symbols,
+                            allow_warnings = allow_warnings,
+                            pyembed        = pyembed,
+                            pyext          = pyext,
+                            local_include  = local_include,
+                            __require_builtin_deps=require_builtin_deps,
+                            global_include = global_include)
+    else:
+        et = bld.SAMBA_SUBSYSTEM(empty_target,
+                                 source=[],
+                                 __force_empty=True,
+                                 __require_builtin_deps=True)
+        et.samba_builtin_subsystem = builtin_subsystem
 
     if BUILTIN_LIBRARY(bld, libname):
         return
@@ -203,9 +261,7 @@ def SAMBA_LIBRARY(bld, libname, source,
         return
 
     # the library itself will depend on that object target
-    deps += ' ' + public_deps
-    deps = TO_LIST(deps)
-    deps.append(obj_target)
+    samba_deps.append(dep_target)
 
     realname = bld.map_shlib_extension(realname, python=(target_type=='PYTHON'))
     link_name = bld.map_shlib_extension(link_name, python=(target_type=='PYTHON'))
@@ -291,7 +347,7 @@ def SAMBA_LIBRARY(bld, libname, source,
         target          = bundled_name,
         depends_on      = depends_on,
         samba_ldflags   = ldflags,
-        samba_deps      = deps,
+        samba_deps      = samba_deps,
         samba_includes  = includes,
         version_script  = vscript,
         version_libname = version_libname,
@@ -309,7 +365,9 @@ def SAMBA_LIBRARY(bld, libname, source,
         abi_vnum        = abi_vnum,
         private_library = private_library,
         grouping_library=grouping_library,
-        allow_undefined_symbols=allow_undefined_symbols
+        allow_undefined_symbols=allow_undefined_symbols,
+        samba_require_builtin_deps=False,
+        samba_builtin_subsystem=builtin_subsystem,
         )
 
     if realname and not link_name:
@@ -565,6 +623,47 @@ def SAMBA_MODULE(bld, modname, source,
 
 Build.BuildContext.SAMBA_MODULE = SAMBA_MODULE
 
+def __SAMBA_SUBSYSTEM_BUILTIN(bld, builtin_target, source,
+                              deps='',
+                              public_deps='',
+                              includes='',
+                              public_headers=None,
+                              public_headers_install=True,
+                              private_headers=None,
+                              header_path=None,
+                              builtin_cflags='',
+                              builtin_cflags_end=None,
+                              group='main',
+                              autoproto=None,
+                              autoproto_extra_source='',
+                              depends_on='',
+                              local_include=True,
+                              global_include=True,
+                              allow_warnings=False):
+
+    bld.ASSERT(builtin_target.endswith('.builtin.objlist'),
+               "builtin_target[%s] does not end with '.builtin.objlist'" %
+               (builtin_target))
+    return bld.SAMBA_SUBSYSTEM(builtin_target, source,
+                               deps=deps,
+                               public_deps=public_deps,
+                               includes=includes,
+                               public_headers=public_headers,
+                               public_headers_install=public_headers_install,
+                               private_headers=private_headers,
+                               header_path=header_path,
+                               cflags=builtin_cflags,
+                               cflags_end=builtin_cflags_end,
+                               hide_symbols=True,
+                               group=group,
+                               target_type='BUILTIN',
+                               autoproto=autoproto,
+                               autoproto_extra_source=autoproto_extra_source,
+                               depends_on=depends_on,
+                               local_include=local_include,
+                               global_include=global_include,
+                               allow_warnings=allow_warnings,
+                               __require_builtin_deps=True)
 
 #################################################################
 def SAMBA_SUBSYSTEM(bld, modname, source,
@@ -579,6 +678,7 @@ def SAMBA_SUBSYSTEM(bld, modname, source,
                     cflags='',
                     cflags_end=None,
                     group='main',
+                    target_type='SUBSYSTEM',
                     init_function_sentinel=None,
                     autoproto=None,
                     autoproto_extra_source='',
@@ -593,11 +693,21 @@ def SAMBA_SUBSYSTEM(bld, modname, source,
                     vars=None,
                     subdir=None,
                     hide_symbols=False,
+                    __require_builtin_deps=False,
+                    provide_builtin_linking=False,
+                    builtin_cflags='',
                     allow_warnings=False,
                     pyext=False,
                     pyembed=False):
     '''define a Samba subsystem'''
 
+    # We support:
+    # - SUBSYSTEM: a normal subsystem from SAMBA_SUBSYSTEM()
+    # - BUILTIN:   a hidden subsystem from __SAMBA_SUBSYSTEM_BUILTIN()
+    if target_type not in ['SUBSYSTEM', 'BUILTIN']:
+        raise Errors.WafError("target_type[%s] not supported in SAMBA_SUBSYSTEM('%s')" %
+                              (target_type, modname))
+
     if not enabled:
         SET_TARGET_TYPE(bld, modname, 'DISABLED')
         return
@@ -613,7 +723,7 @@ def SAMBA_SUBSYSTEM(bld, modname, source,
                             target=empty_c)
         source=empty_c
 
-    if not SET_TARGET_TYPE(bld, modname, 'SUBSYSTEM'):
+    if not SET_TARGET_TYPE(bld, modname, target_type):
         return
 
     source = bld.EXPAND_VARIABLES(source, vars=vars)
@@ -648,6 +758,8 @@ def SAMBA_SUBSYSTEM(bld, modname, source,
         samba_subsystem= subsystem_name,
         samba_use_hostcc = use_hostcc,
         samba_use_global_deps = use_global_deps,
+        samba_require_builtin_deps = __require_builtin_deps,
+        samba_builtin_subsystem = None,
         )
 
     if cflags_end is not None:
@@ -658,6 +770,39 @@ def SAMBA_SUBSYSTEM(bld, modname, source,
     if public_headers is not None:
         bld.PUBLIC_HEADERS(public_headers, header_path=header_path,
                            public_headers_install=public_headers_install)
+
+    if provide_builtin_linking:
+
+        if use_hostcc or pyext or pyembed:
+            raise Errors.WafError("subsystem[%s] provide_builtin_linking=True " +
+                                  "not allowed with use_hostcc=True" %
+                                  modname)
+
+        if pyext or pyembed:
+            raise Errors.WafError("subsystem[%s] provide_builtin_linking=True " +
+                                  "not allowed with pyext=True nor pyembed=True" %
+                                  modname)
+
+        if __require_builtin_deps:
+            raise Errors.WafError("subsystem[%s] provide_builtin_linking=True " +
+                                  "not allowed with __require_builtin_deps=True" %
+                                  modname)
+
+        builtin_target = modname + '.builtin.objlist'
+        tbuiltin = __SAMBA_SUBSYSTEM_BUILTIN(bld, builtin_target, source,
+                                             deps=deps,
+                                             public_deps=public_deps,
+                                             includes=includes,
+                                             header_path=header_path,
+                                             builtin_cflags=builtin_cflags,
+                                             builtin_cflags_end='-D_PUBLIC_=_PRIVATE_',
+                                             group=group,
+                                             depends_on=depends_on,
+                                             local_include=local_include,
+                                             global_include=global_include,
+                                             allow_warnings=allow_warnings)
+        t.samba_builtin_subsystem = tbuiltin
+
     return t