# a waf tool to extract symbols from object files or libraries
# using nm, producing a set of exposed defined/undefined symbols
import os, re, subprocess
from waflib import Utils, Build, Options, Logs, Errors
from waflib.Logs import debug
from samba_utils import TO_LIST, LOCAL_CACHE, get_tgt_list
# these are the data structures used in symbols.py:
#
# bld.env.symbol_map : dictionary mapping public symbol names to list of
# subsystem names where that symbol exists
#
# t.in_library : list of libraries that t is in
#
# bld.env.public_symbols: set of public symbols for each subsystem
# bld.env.used_symbols : set of used symbols for each subsystem
#
# bld.env.syslib_symbols: dictionary mapping system library name to set of symbols
# for that library
# bld.env.library_dict : dictionary mapping built library paths to subsystem names
#
# LOCAL_CACHE(bld, 'TARGET_TYPE') : dictionary mapping subsystem name to target type
def symbols_extract(bld, objfiles, dynamic=False):
'''extract symbols from objfile, returning a dictionary containing
the set of undefined and public symbols for each file'''
ret = {}
# see if we can get some results from the nm cache
if not bld.env.nm_cache:
bld.env.nm_cache = {}
objfiles = set(objfiles).copy()
remaining = set()
for obj in objfiles:
if obj in bld.env.nm_cache:
ret[obj] = bld.env.nm_cache[obj].copy()
else:
remaining.add(obj)
objfiles = remaining
if len(objfiles) == 0:
return ret
cmd = ["nm"]
if dynamic:
# needed for some .so files
cmd.append("-D")
cmd.extend(list(objfiles))
nmpipe = subprocess.Popen(cmd, stdout=subprocess.PIPE).stdout
if len(objfiles) == 1:
filename = list(objfiles)[0]
ret[filename] = { "PUBLIC": set(), "UNDEFINED" : set()}
for line in nmpipe:
line = line.strip()
if line.endswith(b':'):
filename = line[:-1]
ret[filename] = { "PUBLIC": set(), "UNDEFINED" : set() }
continue
cols = line.split(b" ")
if cols == [b'']:
continue
# see if the line starts with an address
if len(cols) == 3:
symbol_type = cols[1]
symbol = cols[2]
else:
symbol_type = cols[0]
symbol = cols[1]
if symbol_type in b"BDGTRVWSi":
# its a public symbol
ret[filename]["PUBLIC"].add(symbol)
elif symbol_type in b"U":
ret[filename]["UNDEFINED"].add(symbol)
# add to the cache
for obj in objfiles:
if obj in ret:
bld.env.nm_cache[obj] = ret[obj].copy()
else:
bld.env.nm_cache[obj] = { "PUBLIC": set(), "UNDEFINED" : set() }
return ret
def real_name(name):
if name.find(".objlist") != -1:
name = name[:-8]
return name
def find_ldd_path(bld, libname, binary):
'''find the path to the syslib we will link against'''
ret = None
if not bld.env.syslib_paths:
bld.env.syslib_paths = {}
if libname in bld.env.syslib_paths:
return bld.env.syslib_paths[libname]
lddpipe = subprocess.Popen(['ldd', binary], stdout=subprocess.PIPE).stdout
for line in lddpipe:
line = line.strip()
cols = line.split(b" ")
if len(cols) < 3 or cols[1] != b"=>":
continue
if cols[0].startswith(b"libc."):
# save this one too
bld.env.libc_path = cols[2]
if cols[0].startswith(libname):
ret = cols[2]
bld.env.syslib_paths[libname] = ret
return ret
# some regular expressions for parsing readelf output
re_sharedlib = re.compile(b'Shared library: \[(.*)\]')
# output from readelf could be `Library rpath` or `Libray runpath`
re_rpath = re.compile(b'Library (rpath|runpath): \[(.*)\]')
def get_libs(bld, binname):
'''find the list of linked libraries for any binary or library
binname is the path to the binary/library on disk
We do this using readelf instead of ldd as we need to avoid recursing
into system libraries
'''
# see if we can get the result from the ldd cache
if not bld.env.lib_cache:
bld.env.lib_cache = {}
if binname in bld.env.lib_cache:
return bld.env.lib_cache[binname].copy()
rpath = []
libs = set()
elfpipe = subprocess.Popen(['readelf', '--dynamic', binname], stdout=subprocess.PIPE).stdout
for line in elfpipe:
m = re_sharedlib.search(line)
if m:
libs.add(m.group(1))
m = re_rpath.search(line)
if m:
# output from Popen is always bytestr even in py3
rpath.extend(m.group(2).split(b":"))
ret = set()
for lib in libs:
found = False
for r in rpath:
path = os.path.join(r, lib)
if os.path.exists(path):
ret.add(os.path.realpath(path))
found = True
break
if not found:
# we didn't find this lib using rpath. It is probably a system
# library, so to find the path to it we either need to use ldd
# or we need to start parsing /etc/ld.so.conf* ourselves. We'll
# use ldd for now, even though it is slow
path = find_ldd_path(bld, lib, binname)
if path:
ret.add(os.path.realpath(path))
bld.env.lib_cache[binname] = ret.copy()
return ret
def get_libs_recursive(bld, binname, seen):
'''find the recursive list of linked libraries for any binary or library
binname is the path to the binary/library on disk. seen is a set used
to prevent loops
'''
if binname in seen:
return set()
ret = get_libs(bld, binname)
seen.add(binname)
for lib in ret:
# we don't want to recurse into system libraries. If a system
# library that we use (eg. libcups) happens to use another library
# (such as libkrb5) which contains common symbols with our own
# libraries, then that is not an error
if lib in bld.env.library_dict:
ret = ret.union(get_libs_recursive(bld, lib, seen))
return ret
def find_syslib_path(bld, libname, deps):
'''find the path to the syslib we will link against'''
# the strategy is to use the targets that depend on the library, and run ldd
# on it to find the real location of the library that is used
linkpath = deps[0].link_task.outputs[0].abspath(bld.env)
if libname == "python":
libname += bld.env.PYTHON_VERSION
return find_ldd_path(bld, "lib%s" % libname.lower(), linkpath)
def build_symbol_sets(bld, tgt_list):
'''build the public_symbols and undefined_symbols attributes for each target'''
if bld.env.public_symbols:
return
objlist = [] # list of object file
objmap = {} # map from object filename to target (subsystem) name
for t in tgt_list:
t.public_symbols = set()
t.undefined_symbols = set()
t.used_symbols = set()
for tsk in getattr(t, 'compiled_tasks', []):
for output in tsk.outputs:
objpath = output.abspath(bld.env)
objlist.append(objpath)
objmap[objpath] = t
symbols = symbols_extract(bld, objlist)
for obj in objlist:
t = objmap[obj]
t.public_symbols = t.public_symbols.union(symbols[obj]["PUBLIC"])
t.undefined_symbols = t.undefined_symbols.union(symbols[obj]["UNDEFINED"])
t.used_symbols = t.used_symbols.union(symbols[obj]["UNDEFINED"])
t.undefined_symbols = t.undefined_symbols.difference(t.public_symbols)
# and the reverse map of public symbols to subsystem name
bld.env.symbol_map = {}
for t in tgt_list:
for s in t.public_symbols:
if not s in bld.env.symbol_map:
bld.env.symbol_map[s] = []
bld.env.symbol_map[s].append(real_name(t.sname))
targets = LOCAL_CACHE(bld, 'TARGET_TYPE')
bld.env.public_symbols = {}
for t in tgt_list:
name = real_name(t.sname)
if name in bld.env.public_symbols:
bld.env.public_symbols[name] = bld.env.public_symbols[name].union(t.public_symbols)
else:
bld.env.public_symbols[name] = t.public_symbols
if t.samba_type == 'LIBRARY':
for dep in t.add_objects:
t2 = bld.get_tgen_by_name(dep)
bld.ASSERT(t2 is not None, "Library '%s' has unknown dependency '%s'" % (name, dep))
bld.env.public_symbols[name] = bld.env.public_symbols[name].union(t2.public_symbols)
bld.env.used_symbols = {}
for t in tgt_list:
name = real_name(t.sname)
if name in bld.env.used_symbols:
bld.env.used_symbols[name] = bld.env.used_symbols[name].union(t.used_symbols)
else:
bld.env.used_symbols[name] = t.used_symbols
if t.samba_type == 'LIBRARY':
for dep in t.add_objects:
t2 = bld.get_tgen_by_name(dep)
bld.ASSERT(t2 is not None, "Library '%s' has unknown dependency '%s'" % (name, dep))
bld.env.used_symbols[name] = bld.env.used_symbols[name].union(t2.used_symbols)
def build_library_dict(bld, tgt_list):
'''build the library_dict dictionary'''
if bld.env.library_dict:
return
bld.env.library_dict = {}
for t in tgt_list:
if t.samba_type in [ 'LIBRARY', 'PYTHON' ]:
linkpath = os.path.realpath(t.link_task.outputs[0].abspath(bld.env))
bld.env.library_dict[linkpath] = t.sname
def build_syslib_sets(bld, tgt_list):
'''build the public_symbols for all syslibs'''
if bld.env.syslib_symbols:
return
# work out what syslibs we depend on, and what targets those are used in
syslibs = {}
objmap = {}
for t in tgt_list:
if getattr(t, 'uselib', []) and t.samba_type in [ 'LIBRARY', 'BINARY', 'PYTHON' ]:
for lib in t.uselib:
if lib in ['PYEMBED', 'PYEXT']:
lib = "python"
if not lib in syslibs:
syslibs[lib] = []
syslibs[lib].append(t)
# work out the paths to each syslib
syslib_paths = []
for lib in syslibs:
path = find_syslib_path(bld, lib, syslibs[lib])
if path is None:
Logs.warn("Unable to find syslib path for %s" % lib)
if path is not None:
syslib_paths.append(path)
objmap[path] = lib.lower()
# add in libc
syslib_paths.append(bld.env.libc_path)
objmap[bld.env.libc_path] = 'c'
symbols = symbols_extract(bld, syslib_paths, dynamic=True)
# keep a map of syslib names to public symbols
bld.env.syslib_symbols = {}
for lib in symbols:
bld.env.syslib_symbols[lib] = symbols[lib]["PUBLIC"]
# add to the map of symbols to dependencies
for lib in symbols:
for sym in symbols[lib]["PUBLIC"]:
if not sym in bld.env.symbol_map:
bld.env.symbol_map[sym] = []
bld.env.symbol_map[sym].append(objmap[lib])
# keep the libc symbols as well, as these are useful for some of the
# sanity checks
bld.env.libc_symbols = symbols[bld.env.libc_path]["PUBLIC"]
# add to the combined map of dependency name to public_symbols
for lib in bld.env.syslib_symbols:
bld.env.public_symbols[objmap[lib]] = bld.env.syslib_symbols[lib]
def build_autodeps(bld, t):
'''build the set of dependencies for a target'''
deps = set()
name = real_name(t.sname)
targets = LOCAL_CACHE(bld, 'TARGET_TYPE')
for sym in t.undefined_symbols:
if sym in t.public_symbols:
continue
if sym in bld.env.symbol_map:
depname = bld.env.symbol_map[sym]
if depname == [ name ]:
# self dependencies aren't interesting
continue
if t.in_library == depname:
# no need to depend on the library we are part of
continue
if depname[0] in ['c', 'python']:
# these don't go into autodeps
continue
if targets[depname[0]] in [ 'SYSLIB' ]:
deps.add(depname[0])
continue
t2 = bld.get_tgen_by_name(depname[0])
if len(t2.in_library) != 1:
deps.add(depname[0])
continue
if t2.in_library == t.in_library:
# if we're part of the same library, we don't need to autodep
continue
deps.add(t2.in_library[0])
t.autodeps = deps
def build_library_names(bld, tgt_list):
'''add a in_library attribute to all targets that are part of a library'''
if bld.env.done_build_library_names:
return
for t in tgt_list:
t.in_library = []
for t in tgt_list:
if t.samba_type in [ 'LIBRARY' ]:
for obj in t.samba_deps_extended:
t2 = bld.get_tgen_by_name(obj)
if t2 and t2.samba_type in [ 'SUBSYSTEM', 'ASN1' ]:
if not t.sname in t2.in_library:
t2.in_library.append(t.sname)
bld.env.done_build_library_names = True
def check_library_deps(bld, t):
'''check that all the autodeps that have mutual dependency of this
target are in the same library as the target'''
name = real_name(t.sname)
if len(t.in_library) > 1:
Logs.warn("WARNING: Target '%s' in multiple libraries: %s" % (t.sname, t.in_library))
for dep in t.autodeps:
t2 = bld.get_tgen_by_name(dep)
if t2 is None:
continue
for dep2 in t2.autodeps:
if dep2 == name and t.in_library != t2.in_library:
Logs.warn("WARNING: mutual dependency %s <=> %s" % (name, real_name(t2.sname)))
Logs.warn("Libraries should match. %s != %s" % (t.in_library, t2.in_library))
# raise Errors.WafError("illegal mutual dependency")
def check_syslib_collisions(bld, tgt_list):
'''check if a target has any symbol collisions with a syslib
We do not want any code in Samba to use a symbol name from a
system library. The chance of that causing problems is just too
high. Note that libreplace uses a rep_XX approach of renaming
symbols via macros
'''
has_error = False
for t in tgt_list:
for lib in bld.env.syslib_symbols:
common = t.public_symbols.intersection(bld.env.syslib_symbols[lib])
if common:
Logs.error("ERROR: Target '%s' has symbols '%s' which is also in syslib '%s'" % (t.sname, common, lib))
has_error = True
if has_error:
raise Errors.WafError("symbols in common with system libraries")
def check_dependencies(bld, t):
'''check for depenencies that should be changed'''
if bld.get_tgen_by_name(t.sname + ".objlist"):
return
targets = LOCAL_CACHE(bld, 'TARGET_TYPE')
remaining = t.undefined_symbols.copy()
remaining = remaining.difference(t.public_symbols)
sname = real_name(t.sname)
deps = set(t.samba_deps)
for d in t.samba_deps:
if targets[d] in [ 'EMPTY', 'DISABLED', 'SYSLIB', 'GENERATOR' ]:
continue
bld.ASSERT(d in bld.env.public_symbols, "Failed to find symbol list for dependency '%s'" % d)
diff = remaining.intersection(bld.env.public_symbols[d])
if not diff and targets[sname] != 'LIBRARY':
Logs.info("Target '%s' has no dependency on %s" % (sname, d))
else:
remaining = remaining.difference(diff)
t.unsatisfied_symbols = set()
needed = {}
for sym in remaining:
if sym in bld.env.symbol_map:
dep = bld.env.symbol_map[sym]
if not dep[0] in needed:
needed[dep[0]] = set()
needed[dep[0]].add(sym)
else:
t.unsatisfied_symbols.add(sym)
for dep in needed:
Logs.info("Target '%s' should add dep '%s' for symbols %s" % (sname, dep, " ".join(needed[dep])))
def check_syslib_dependencies(bld, t):
'''check for syslib depenencies'''
if bld.get_tgen_by_name(t.sname + ".objlist"):
return
sname = real_name(t.sname)
remaining = set()
features = TO_LIST(t.features)
if 'pyembed' in features or 'pyext' in features:
if 'python' in bld.env.public_symbols:
t.unsatisfied_symbols = t.unsatisfied_symbols.difference(bld.env.public_symbols['python'])
needed = {}
for sym in t.unsatisfied_symbols:
if sym in bld.env.symbol_map:
dep = bld.env.symbol_map[sym][0]
if dep == 'c':
continue
if not dep in needed:
needed[dep] = set()
needed[dep].add(sym)
else:
remaining.add(sym)
for dep in needed:
Logs.info("Target '%s' should add syslib dep '%s' for symbols %s" % (sname, dep, " ".join(needed[dep])))
if remaining:
debug("deps: Target '%s' has unsatisfied symbols: %s" % (sname, " ".join(remaining)))
def symbols_symbolcheck(task):
'''check the internal dependency lists'''
bld = task.env.bld
tgt_list = get_tgt_list(bld)
build_symbol_sets(bld, tgt_list)
build_library_names(bld, tgt_list)
for t in tgt_list:
t.autodeps = set()
if getattr(t, 'source', ''):
build_autodeps(bld, t)
for t in tgt_list:
check_dependencies(bld, t)
for t in tgt_list:
check_library_deps(bld, t)
def symbols_syslibcheck(task):
'''check the syslib dependencies'''
bld = task.env.bld
tgt_list = get_tgt_list(bld)
build_syslib_sets(bld, tgt_list)
check_syslib_collisions(bld, tgt_list)
for t in tgt_list:
check_syslib_dependencies(bld, t)
def symbols_whyneeded(task):
"""check why 'target' needs to link to 'subsystem'"""
bld = task.env.bld
tgt_list = get_tgt_list(bld)
why = Options.options.WHYNEEDED.split(":")
if len(why) != 2:
raise Errors.WafError("usage: WHYNEEDED=TARGET:DEPENDENCY")
target = why[0]
subsystem = why[1]
build_symbol_sets(bld, tgt_list)
build_library_names(bld, tgt_list)
build_syslib_sets(bld, tgt_list)
Logs.info("Checking why %s needs to link to %s" % (target, subsystem))
if not target in bld.env.used_symbols:
Logs.warn("unable to find target '%s' in used_symbols dict" % target)
return
if not subsystem in bld.env.public_symbols:
Logs.warn("unable to find subsystem '%s' in public_symbols dict" % subsystem)
return
overlap = bld.env.used_symbols[target].intersection(bld.env.public_symbols[subsystem])
if not overlap:
Logs.info("target '%s' doesn't use any public symbols from '%s'" % (target, subsystem))
else:
Logs.info("target '%s' uses symbols %s from '%s'" % (target, overlap, subsystem))
def report_duplicate(bld, binname, sym, libs, fail_on_error):
'''report duplicated symbols'''
if sym in ['_init', '_fini', '_edata', '_end', '__bss_start']:
return
libnames = []
for lib in libs:
if lib in bld.env.library_dict:
libnames.append(bld.env.library_dict[lib])
else:
libnames.append(lib)
if fail_on_error:
raise Errors.WafError("%s: Symbol %s linked in multiple libraries %s" % (binname, sym, libnames))
else:
print("%s: Symbol %s linked in multiple libraries %s" % (binname, sym, libnames))
def symbols_dupcheck_binary(bld, binname, fail_on_error):
'''check for duplicated symbols in one binary'''
libs = get_libs_recursive(bld, binname, set())
symlist = symbols_extract(bld, libs, dynamic=True)
symmap = {}
for libpath in symlist:
for sym in symlist[libpath]['PUBLIC']:
if sym == '_GLOBAL_OFFSET_TABLE_':
continue
if not sym in symmap:
symmap[sym] = set()
symmap[sym].add(libpath)
for sym in symmap:
if len(symmap[sym]) > 1:
for libpath in symmap[sym]:
if libpath in bld.env.library_dict:
report_duplicate(bld, binname, sym, symmap[sym], fail_on_error)
break
def symbols_dupcheck(task, fail_on_error=False):
'''check for symbols defined in two different subsystems'''
bld = task.env.bld
tgt_list = get_tgt_list(bld)
targets = LOCAL_CACHE(bld, 'TARGET_TYPE')
build_library_dict(bld, tgt_list)
for t in tgt_list:
if t.samba_type == 'BINARY':
binname = os.path.relpath(t.link_task.outputs[0].abspath(bld.env), os.getcwd())
symbols_dupcheck_binary(bld, binname, fail_on_error)
def symbols_dupcheck_fatal(task):
'''check for symbols defined in two different subsystems (and fail if duplicates are found)'''
symbols_dupcheck(task, fail_on_error=True)
def SYMBOL_CHECK(bld):
'''check our dependency lists'''
if Options.options.SYMBOLCHECK:
bld.SET_BUILD_GROUP('symbolcheck')
task = bld(rule=symbols_symbolcheck, always=True, name='symbol checking')
task.env.bld = bld
bld.SET_BUILD_GROUP('syslibcheck')
task = bld(rule=symbols_syslibcheck, always=True, name='syslib checking')
task.env.bld = bld
bld.SET_BUILD_GROUP('syslibcheck')
task = bld(rule=symbols_dupcheck, always=True, name='symbol duplicate checking')
task.env.bld = bld
if Options.options.WHYNEEDED:
bld.SET_BUILD_GROUP('syslibcheck')
task = bld(rule=symbols_whyneeded, always=True, name='check why a dependency is needed')
task.env.bld = bld
Build.BuildContext.SYMBOL_CHECK = SYMBOL_CHECK
def DUP_SYMBOL_CHECK(bld):
if Options.options.DUP_SYMBOLCHECK and bld.env.DEVELOPER:
'''check for duplicate symbols'''
bld.SET_BUILD_GROUP('syslibcheck')
task = bld(rule=symbols_dupcheck_fatal, always=True, name='symbol duplicate checking')
task.env.bld = bld
Build.BuildContext.DUP_SYMBOL_CHECK = DUP_SYMBOL_CHECK