From c78b1b2641305f7f2c50c504917b469d322cf4d2 Mon Sep 17 00:00:00 2001 From: Packit Service Date: Jan 06 2021 20:14:06 +0000 Subject: ksc-1.7 base --- diff --git a/PKG-INFO b/PKG-INFO index 1f45737..c35aeee 100644 --- a/PKG-INFO +++ b/PKG-INFO @@ -1,6 +1,6 @@ -Metadata-Version: 1.6 +Metadata-Version: 1.7 Name: ksc -Version: 1.6 +Version: 1.7 Summary: ksc tool Home-page: http://redhat.com Author: Kushal Das, Samikshan Bairagya, Stanislav Kozina, Martin Lacko, Ziqian Sun diff --git a/README b/README index 7a90e06..16f5789 100644 --- a/README +++ b/README @@ -1,6 +1,6 @@ ksc tool ======== -A tool to check whitelist symbol usage in kernel module source code. +A tool to check stablelist symbol usage in kernel module source code. Options: -h, --help show this help message and exit @@ -15,7 +15,7 @@ Options: -p PREVIOUS, --previous=PREVIOUS path to previous resultset to submit as bug -r RELEASE, --release=RELEASE - RHEL whitelist release to compare against + RHEL stablelist release to compare against -y SYMVERS, --symvers=SYMVERS Path to the Module.symvers file. The current kernel path is used if not specified. @@ -32,6 +32,6 @@ Example command and output: $ ksc -d ../drivers/ipw2200-1.1.4 -Total symbol usage: 71 Total Non white list symbol usage: 3 +Total symbol usage: 71 Total Non stable list symbol usage: 3 Score: 95.77% diff --git a/ksc.1 b/ksc.1 index ff1faf3..5b788d3 100644 --- a/ksc.1 +++ b/ksc.1 @@ -1,4 +1,4 @@ -.TH ksc "1" "November 2018" "ksc - Version 1.6" "User Commands" +.TH ksc "1" "November 2018" "ksc - Version 1.7" "User Commands" .SH NAME ksc \- report symbols used by kernel modules .SH SYNOPSIS @@ -17,6 +17,19 @@ For more on report generation, see -k. For more on report submission, see -s. +.SH ENVIRONMENT + +It is highly recommended that ksc is used on the latest minor version of RHEL, +and to make sure, prior to ksc usage, to update kabi-stablelists as well as +kernel-devel-provided Module.symvers. This can be accomplished, for example, +by issuing: + +yum \fBupdate\fR \fBkernel-abi-stablelists kernel-devel\fR + +It is also possible to use the -y (--symvers) option to explicitly state the +location of Module.symvers file to make sure that accurate and up-to-date +kABI data be provided to ksc. + .SH OPTIONS You may freely mix different option styles. @@ -72,7 +85,7 @@ Specified \fIPREVIOUS\fR file is marked for submission to Red Hat Bugzilla. To submit, you need to use -s. .TP \fB\-r\fR \fIRELEASE\fR, \fB\-\-release\fR=\fIRELEASE\fR -RHEL \fIRELEASE\fR used for symbol comparison with whitelists, e.g. '6.5'. +RHEL \fIRELEASE\fR used for symbol comparison with stablelists, e.g. '6.5'. .TP \fB\-y\fR \fISYMVERS\fR, \fB\-\-symvers\fR=\fISYMVERS\fR Path to Module.symvers \fISYMVERS\fR file used to check symbol presence in the @@ -125,36 +138,36 @@ server field), it will always be treated as though it were undefined/left blank. .SH EXAMPLES .TP ksc -k a.ko -Process a.ko file using system-provided whitelist and Module.symvers files. -Reports symbols used by a.ko that are not in whitelist, requesting justification +Process a.ko file using system-provided stablelist and Module.symvers files. +Reports symbols used by a.ko that are not in stablelist, requesting justification for their use. .TP ksc -k a.ko -y /tmp/Module.symvers -Process a.ko file using system-provided whitelist and user-provided +Process a.ko file using system-provided stablelist and user-provided Module.symvers (/tmp/Module.symvers) files. Reports symbols used by a.ko that -are not in whitelist, requesting justification for their use. +are not in stablelist, requesting justification for their use. .TP ksc -k a.ko -k b.ko -Process a.ko and b.ko files using system-provided whitelist and Module.symvers -files. Reports symbols used by both a.ko and b.ko that are not in whitelist, +Process a.ko and b.ko files using system-provided stablelist and Module.symvers +files. Reports symbols used by both a.ko and b.ko that are not in stablelist, requesting justification for their use. Only one justification per symbol is required. Any mention of an already justified symbol come with an automatically generated references. .TP ksc -k a.ko -k b.ko -K c.ko -K d.ko -Process a.ko and b.ko files using system-provided whitelist and Module.symvers -files. Reports symbols used by both a.ko and b.ko that are not in whitelist and +Process a.ko and b.ko files using system-provided stablelist and Module.symvers +files. Reports symbols used by both a.ko and b.ko that are not in stablelist and not exported by either c.ko or d.ko, requesting justification for their use. Only one justification per symbol is required. Any mention of an already justified symbol come with an automatically generated references. .TP ksc -k a.ko -k b.ko -j previous-report.txt -Process a.ko and b.ko files using system-provided whitelist and Module.symvers -files. Reports symbols used by both a.ko and b.ko that are not in whitelist, +Process a.ko and b.ko files using system-provided stablelist and Module.symvers +files. Reports symbols used by both a.ko and b.ko that are not in stablelist, requesting justification for their use. Only one justification per symbol is required. Any mention of an already justified symbol come with an automatically generated references. Justifications already present in previous-report.txt diff --git a/ksc.py b/ksc.py index 6e58bb9..53dd192 100755 --- a/ksc.py +++ b/ksc.py @@ -18,7 +18,7 @@ from utils import read_total_list, get_release_name from utils import createbug from utils import query_user, query_user_bool -KSCVERSION = "ksc - Version 1.6" +KSCVERSION = "ksc - Version 1.7" class Ksc(object): @@ -30,12 +30,13 @@ class Ksc(object): SECTION_KO_RE = re.compile(r'{(?P.*)}') # RE to match with KO-body section - LISTS_RE = re.compile(r'.*\[WHITELISTUSAGE\](?P.*)\[NONWHITELISTUSAGE\]\s*(?P.*)', re.S) + LISTS_RE = re.compile(r'.*\[STABLELISTUSAGE\](?P.*)\[NONSTABLELISTUSAGE\]\s*(?P.*)', re.S) + LISTS_RE_DEPRECATED = re.compile(r'.*\[WHITELISTUSAGE\](?P.*)\[NONWHITELISTUSAGE\]\s*(?P.*)', re.S) # RE to extract symbol and its justification JUSTIFICATION_RE = re.compile(r'#*\s*\((?P.*)\)\s*(?P[^#]*)') - # Non-whitelisted symbols justification free-form entries + # Non-stablelisted symbols justification free-form entries JUSTIFICATION_PLACEHOLDER = "ENTER JUSTIFICATION TEXT HERE" JUSTIFICATION_REFERENCE = "JUSTIFICATION STATED UNDER `%s' SECTION" JUSTIFICATION_REF_DETECT = re.compile(r"JUSTIFICATION STATED UNDER `(.*)' SECTION") @@ -43,16 +44,18 @@ class Ksc(object): JUSTIFICATION_BODY = '\n(%s)\n\n%s\n\n' # Sections - SECTION_WHITELISTS = "[WHITELISTUSAGE]\n" - SECTION_CO_WHITELISTS = "[NONWHITELISTUSAGE]\n" + # Note: there needn't be old-style declarations here, since this is used + # for generation only. + SECTION_STABLELISTS = "[STABLELISTUSAGE]\n" + SECTION_CO_STABLELISTS = "[NONSTABLELISTUSAGE]\n" def __init__(self, mock=False): """ Init call """ self.all_symbols_used = {} - self.nonwhite_symbols_used = {} - self.white_symbols = {} + self.nonstable_symbols_used = {} + self.stable_symbols = {} self.defined_symbols = {} self.justified_symbols = {} self.justifications = {} @@ -71,8 +74,8 @@ class Ksc(object): def clean(self): self.all_symbols_used = {} - self.nonwhite_symbols_used = {} - self.white_symbols = {} + self.nonstable_symbols_used = {} + self.stable_symbols = {} self.defined_symbols = {} self.justified_symbols = {} self.justifications = {} @@ -102,7 +105,7 @@ class Ksc(object): parser.add_option("-p", "--previous", dest="previous", help="path to previous resultset to submit as bug") parser.add_option("-r", "--release", dest="release", - help="RHEL whitelist release to compare against, " + help="RHEL stablelist release to compare against, " "e.g '6.7'", metavar="RELEASE") parser.add_option("-y", "--symvers", dest="symvers", help="Path to the Module.symvers file. " @@ -173,21 +176,21 @@ class Ksc(object): if options.ko_dependency: for kmod_path in options.ko_dependency: - self.parse_ko(kmod_path, process_whitelists=False) + self.parse_ko(kmod_path, process_stablelists=False) if options.ko: self.find_arch(options.ko) exists = self.read_data(self.arch, self.releasedir, self.symvers) - # Return if there is any issue in reading whitelists + # Return if there is any issue in reading stablelists if not exists: print(("Release %s for arch %s was not found.\n" - "Do you have right kernel-abi-whitelist installed ?" % + "Do you have right kernel-abi-stablelist installed ?" % (self.releasedir, self.arch))) sys.exit(1) for kmod_path in options.ko: - self.parse_ko(kmod_path, process_whitelists=True) + self.parse_ko(kmod_path, process_stablelists=True) self.remove_internal_symbols() @@ -226,9 +229,14 @@ class Ksc(object): if filename_match: filename_section = filename_match.group('ko_file') + # Attempt to read new-style justifications, provided they are + # present. Otherwise assume that old-style justifications were + # given. match = self.LISTS_RE.match(file_contents) if not match: - continue + match = self.LISTS_RE_DEPRECATED.match(file_contents) + if not match: + continue for symbol, justification in \ self.JUSTIFICATION_RE.findall(file_contents): @@ -350,7 +358,7 @@ class Ksc(object): def get_justification(self, filename): """ Get the justification from User - on non-whitelist symbols + on non-stablelist symbols """ bold = "\033[1m" @@ -359,7 +367,7 @@ class Ksc(object): print(bold) print('On the next screen, the result log will be opened to allow') print('you to provide technical justification on why these symbols') - print('need to be included in the KABI whitelist.') + print('need to be included in the KABI stablelist.') print('Please provide sufficient information in the log, marked with ') print('the line below:') @@ -424,27 +432,27 @@ class Ksc(object): print("Processing %s" % kmod_path) - for name in self.nonwhite_symbols_used[kmod_path]: + for name in self.nonstable_symbols_used[kmod_path]: if name not in self.total: print("WARNING: External symbol in %s does not " "exist in current kernel: %s" \ % (os.path.basename(kmod_path),name)) total_len = len(self.all_symbols_used[kmod_path]) - non_white = len(self.nonwhite_symbols_used[kmod_path]) - white_len = float(len(self.white_symbols[kmod_path])) + non_stable = len(self.nonstable_symbols_used[kmod_path]) + stable_len = float(len(self.stable_symbols[kmod_path])) if total_len == 0: # pragma: no cover print("No kernel symbol usage found in %s." % kmod_path) return - score = (white_len / total_len) * 100 + score = (stable_len / total_len) * 100 if not self.mock: print("Checking against architecture %s" % self.arch) print("Total symbol usage: %s\t" - "Total Non white list symbol usage: %s" - % (total_len, non_white)) + "Total Non stable list symbol usage: %s" + % (total_len, non_stable)) print("Score: %0.2f%%\n" % score) def find_arch(self, kmod_list): @@ -492,18 +500,18 @@ class Ksc(object): f.write("[%s]\n" % self.arch) - # Write whitelisted symbols - f.write(self.SECTION_WHITELISTS) - for name in sorted(self.white_symbols[ko_file]): + # Write stablelisted symbols + f.write(self.SECTION_STABLELISTS) + for name in sorted(self.stable_symbols[ko_file]): f.write(name + '\n') - # Write non-whitelisted symbols as well as their justification + # Write non-stablelisted symbols as well as their justification # Justification can be one of: # - free-form entry # - reference to a different kernel module section (if exists) # - justification placeholder later to be specified by hand - f.write(self.SECTION_CO_WHITELISTS) - for name in sorted(self.nonwhite_symbols_used[ko_file]): + f.write(self.SECTION_CO_STABLELISTS) + for name in sorted(self.nonstable_symbols_used[ko_file]): justification="" if name in self.justified_symbols \ @@ -523,7 +531,7 @@ class Ksc(object): f.write(self.JUSTIFICATION_SEPARATOR) f.write(self.JUSTIFICATION_BODY % (name, justification)) - if self.nonwhite_symbols_used[ko_file]: + if self.nonstable_symbols_used[ko_file]: f.write(self.JUSTIFICATION_SEPARATOR) f.write('\n') except Exception as err: @@ -537,15 +545,15 @@ class Ksc(object): self.total = read_total_list(symvers) return exists - def parse_ko(self, path, process_whitelists=True): + def parse_ko(self, path, process_stablelists=True): """ parse a ko file """ - if process_whitelists: - self.nonwhite_symbols_used[path] = set() + if process_stablelists: + self.nonstable_symbols_used[path] = set() self.all_symbols_used[path] = set() - self.nonwhite_symbols_used[path] = set() - self.white_symbols[path] = set() + self.nonstable_symbols_used[path] = set() + self.stable_symbols[path] = set() self.defined_symbols[path] = set() @@ -565,27 +573,27 @@ class Ksc(object): data = line.split(" ") if len(data) < 2: continue - if "U " in line and process_whitelists: + if "U " in line and process_stablelists: self.find_if(path, data[len(data)-1]) else: self.defined_symbols[path].add(data[len(data)-1]) def remove_internal_symbols(self): - for i in self.nonwhite_symbols_used: + for i in self.nonstable_symbols_used: for j in self.defined_symbols: if i == j: continue - self.nonwhite_symbols_used[i] -= self.defined_symbols[j] + self.nonstable_symbols_used[i] -= self.defined_symbols[j] def find_if(self, path, name): """ - Find if the symbol is in whitelist or not + Find if the symbol is in stablelist or not """ self.all_symbols_used[path].add(name) if name in self.matchdata: - self.white_symbols[path].add(name) + self.stable_symbols[path].add(name) else: - self.nonwhite_symbols_used[path].add(name) + self.nonstable_symbols_used[path].add(name) def get_module_name(self, command_line): try: diff --git a/setup.py b/setup.py index f1ae800..7906ccf 100644 --- a/setup.py +++ b/setup.py @@ -36,7 +36,7 @@ for x in os.listdir('bugzilla/'): bugzilla.append('bugzilla/%s' % x) setup(name='ksc', - version='1.6', + version='1.7', description="ksc tool", long_description="Kernel Module Source Checker tool", cmdclass={'test': TestCommand}, diff --git a/tests.py b/tests.py index e1a0553..79f0589 100644 --- a/tests.py +++ b/tests.py @@ -6,7 +6,7 @@ from mock import Mock, patch class ReadListTest(unittest.TestCase): """ - Test reading whitelist + Test reading stablelist """ def runTest(self): data, _ = read_list("x86_64", "kabi-current") @@ -62,8 +62,8 @@ class ParseKOTest(unittest.TestCase): k.read_data('x86_64', 'kabi-current') k.parse_ko('./ksc.py') assert len(k.all_symbols_used) == 2 - assert len(k.nonwhite_symbols_used) == 1 - assert len(k.white_symbols) == 1 + assert len(k.nonstable_symbols_used) == 1 + assert len(k.stable_symbols) == 1 if __name__ == '__main__': diff --git a/utils.py b/utils.py index f2b2b8c..1004e09 100644 --- a/utils.py +++ b/utils.py @@ -23,7 +23,7 @@ import locale from bugzilla import Bugzilla, BugzillaError -# whitelist directory +# stablelist directory WHPATH = '/lib/modules' # Module.symvers directory SRCPATH = '/usr/src/kernels' @@ -94,10 +94,12 @@ def get_release_name(): def read_list(arch, kabipath, verbose=False): """ - Reads a whitelist file and returns the symbols + Reads a stablelist file and returns the symbols """ result = [] - fpath = os.path.join(WHPATH, kabipath, "kabi_whitelist_%s" % arch) + fpath = os.path.join(WHPATH, kabipath, "kabi_stablelist_%s" % arch) + if not os.path.isfile(fpath): + fpath = os.path.join(WHPATH, kabipath, "kabi_whitelist_%s" % arch) if not os.path.isfile(fpath): # pragma: no cover print("File not found:", fpath) return [], False @@ -112,7 +114,7 @@ def read_list(arch, kabipath, verbose=False): fptr.close() except IOError as err: # pragma: no cover print(err) - print("whitelist missing") + print("stablelist missing") return result, True @@ -224,7 +226,6 @@ def createbug(filename, arch, mock=False, path='/etc/ksc.conf', print("Invalid releasename: Bug not created") return bughash["component"] = 'kernel' - bughash["sub_component"] = 'kabi-whitelists' bughash["summary"] = "kABI Symbol Usage" bughash["version"] = releasename bughash["platform"] = arch @@ -271,8 +272,8 @@ def createbug(filename, arch, mock=False, path='/etc/ksc.conf', bughash["Bugzilla_login"] = conf["user"] bughash["Bugzilla_password"] = conf["password"] bughash["cf_partner"] = [conf["partner"], ] + bughash["keywords"] = ["Tracking"] - bugid = 0 try: if 'api_key' in conf and conf['api_key'] != 'api_key': bz = Bugzilla( @@ -285,60 +286,77 @@ def createbug(filename, arch, mock=False, path='/etc/ksc.conf', user=conf["user"], password=conf["password"] ) + except BugzillaError as err: + print("Bug not submitted. %s" % err) + if not mock: + sys.exit(1) - if not mock: # pragma: no cover - print("Creating a new bug") + if not mock: # pragma: no cover + print("Creating a new bug") - try: - ret = bz.build_createbug( - product=bughash['product'], - component=bughash['component'], - sub_component=bughash['sub_component'], - summary=bughash['summary'], - version=bughash['version'], - platform=bughash['platform'], - qa_contact=bughash['qa_contact'], - severity=bughash['severity'], - priority=bughash['priority'], - description=bughash['description'], - groups=bughash['groups'] - ) - ret['cf_partner'] = bughash['cf_partner'] - bug = bz.createbug(ret) - - bugid = bug.id - - if not mock: # pragma: no cover - print("Bug URL %s/show_bug.cgi?id=%s" % \ - (conf['server'][:-11], bugid)) - print("Attaching the report") - - dhash = {} - dhash["filename"] = "ksc-result.txt" - dhash["contenttype"] = "text/plain" - desc = "kABI symbol usage." - - for _ in range(3): - with open(filename, "r") as fptr: - attachment_id = bz.attachfile(bugid, fptr, desc, **dhash) - - if not mock: # pragma: no cover - if not attachment_id: - time.sleep(1) - else: - print("Attached successfully as %s on bug %s" % (attachment_id, bugid)) - break - else: - print("Failed to attach symbol usage result") - sys.exit() + bughash["sub_component"] = 'kabi-stablelists' + # As it is as yet unclear whether the new sub_component will be + # set up at the time of deployment, attemp to file with the old + # sub_component as well. + # If kabi-stablelists does not exist, an attempt to createbug + # will cause an xmlrpc.client.Fault exception. + try: + trycreatebug(filename, mock, bughash, conf, bz) + except Exception as err: # pragma: no cover + try: + bughash["sub_component"] = 'kabi-whitelists' + trycreatebug(filename, mock, bughash, conf, bz) except Exception as err: # pragma: no cover - print("Could not create bug. %s" % err) + print ("Could not create bug. %s" % err) if not mock: sys.exit(1) - except BugzillaError as err: - print("Bug not submitted. %s" % err) - if not mock: - sys.exit(1) + +def trycreatebug(filename, mock, bughash, conf, bz): + + bugid = 0 + + ret = bz.build_createbug( + product=bughash['product'], + component=bughash['component'], + sub_component=bughash['sub_component'], + summary=bughash['summary'], + version=bughash['version'], + platform=bughash['platform'], + qa_contact=bughash['qa_contact'], + severity=bughash['severity'], + priority=bughash['priority'], + description=bughash['description'], + groups=bughash['groups'], + keywords=bughash['keywords'] + ) + ret['cf_partner'] = bughash['cf_partner'] + bug = bz.createbug(ret) + + bugid = bug.id + + if not mock: # pragma: no cover + print("Bug URL %s/show_bug.cgi?id=%s" % \ + (conf['server'][:-11], bugid)) + print("Attaching the report") + + dhash = {} + dhash["filename"] = "ksc-result.txt" + dhash["contenttype"] = "text/plain" + desc = "kABI symbol usage." + + for _ in range(3): + with open(filename, "r") as fptr: + attachment_id = bz.attachfile(bugid, fptr, desc, **dhash) + + if not mock: # pragma: no cover + if not attachment_id: + time.sleep(1) + else: + print("Attached successfully as %s on bug %s" % (attachment_id, bugid)) + break + else: + print("Failed to attach symbol usage result") + sys.exit() return bugid