diff --git a/COPYING b/COPYING new file mode 100644 index 0000000..dec24bd --- /dev/null +++ b/COPYING @@ -0,0 +1,340 @@ + GNU GENERAL PUBLIC LICENSE + Version 2, June 1991 + + Copyright (C) 1989, 1991 Free Software Foundation, Inc. + 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA + Everyone is permitted to copy and distribute verbatim copies + of this license document, but changing it is not allowed. + + Preamble + + The licenses for most software are designed to take away your +freedom to share and change it. By contrast, the GNU General Public +License is intended to guarantee your freedom to share and change free +software--to make sure the software is free for all its users. This +General Public License applies to most of the Free Software +Foundation's software and to any other program whose authors commit to +using it. (Some other Free Software Foundation software is covered by +the GNU Library General Public License instead.) You can apply it to +your programs, too. + + When we speak of free software, we are referring to freedom, not +price. Our General Public Licenses are designed to make sure that you +have the freedom to distribute copies of free software (and charge for +this service if you wish), that you receive source code or can get it +if you want it, that you can change the software or use pieces of it +in new free programs; and that you know you can do these things. + + To protect your rights, we need to make restrictions that forbid +anyone to deny you these rights or to ask you to surrender the rights. +These restrictions translate to certain responsibilities for you if you +distribute copies of the software, or if you modify it. + + For example, if you distribute copies of such a program, whether +gratis or for a fee, you must give the recipients all the rights that +you have. You must make sure that they, too, receive or can get the +source code. And you must show them these terms so they know their +rights. + + We protect your rights with two steps: (1) copyright the software, and +(2) offer you this license which gives you legal permission to copy, +distribute and/or modify the software. + + Also, for each author's protection and ours, we want to make certain +that everyone understands that there is no warranty for this free +software. If the software is modified by someone else and passed on, we +want its recipients to know that what they have is not the original, so +that any problems introduced by others will not reflect on the original +authors' reputations. + + Finally, any free program is threatened constantly by software +patents. We wish to avoid the danger that redistributors of a free +program will individually obtain patent licenses, in effect making the +program proprietary. To prevent this, we have made it clear that any +patent must be licensed for everyone's free use or not licensed at all. + + The precise terms and conditions for copying, distribution and +modification follow. + + GNU GENERAL PUBLIC LICENSE + TERMS AND CONDITIONS FOR COPYING, DISTRIBUTION AND MODIFICATION + + 0. This License applies to any program or other work which contains +a notice placed by the copyright holder saying it may be distributed +under the terms of this General Public License. The "Program", below, +refers to any such program or work, and a "work based on the Program" +means either the Program or any derivative work under copyright law: +that is to say, a work containing the Program or a portion of it, +either verbatim or with modifications and/or translated into another +language. (Hereinafter, translation is included without limitation in +the term "modification".) Each licensee is addressed as "you". + +Activities other than copying, distribution and modification are not +covered by this License; they are outside its scope. The act of +running the Program is not restricted, and the output from the Program +is covered only if its contents constitute a work based on the +Program (independent of having been made by running the Program). +Whether that is true depends on what the Program does. + + 1. You may copy and distribute verbatim copies of the Program's +source code as you receive it, in any medium, provided that you +conspicuously and appropriately publish on each copy an appropriate +copyright notice and disclaimer of warranty; keep intact all the +notices that refer to this License and to the absence of any warranty; +and give any other recipients of the Program a copy of this License +along with the Program. + +You may charge a fee for the physical act of transferring a copy, and +you may at your option offer warranty protection in exchange for a fee. + + 2. You may modify your copy or copies of the Program or any portion +of it, thus forming a work based on the Program, and copy and +distribute such modifications or work under the terms of Section 1 +above, provided that you also meet all of these conditions: + + a) You must cause the modified files to carry prominent notices + stating that you changed the files and the date of any change. + + b) You must cause any work that you distribute or publish, that in + whole or in part contains or is derived from the Program or any + part thereof, to be licensed as a whole at no charge to all third + parties under the terms of this License. + + c) If the modified program normally reads commands interactively + when run, you must cause it, when started running for such + interactive use in the most ordinary way, to print or display an + announcement including an appropriate copyright notice and a + notice that there is no warranty (or else, saying that you provide + a warranty) and that users may redistribute the program under + these conditions, and telling the user how to view a copy of this + License. (Exception: if the Program itself is interactive but + does not normally print such an announcement, your work based on + the Program is not required to print an announcement.) + +These requirements apply to the modified work as a whole. If +identifiable sections of that work are not derived from the Program, +and can be reasonably considered independent and separate works in +themselves, then this License, and its terms, do not apply to those +sections when you distribute them as separate works. But when you +distribute the same sections as part of a whole which is a work based +on the Program, the distribution of the whole must be on the terms of +this License, whose permissions for other licensees extend to the +entire whole, and thus to each and every part regardless of who wrote it. + +Thus, it is not the intent of this section to claim rights or contest +your rights to work written entirely by you; rather, the intent is to +exercise the right to control the distribution of derivative or +collective works based on the Program. + +In addition, mere aggregation of another work not based on the Program +with the Program (or with a work based on the Program) on a volume of +a storage or distribution medium does not bring the other work under +the scope of this License. + + 3. You may copy and distribute the Program (or a work based on it, +under Section 2) in object code or executable form under the terms of +Sections 1 and 2 above provided that you also do one of the following: + + a) Accompany it with the complete corresponding machine-readable + source code, which must be distributed under the terms of Sections + 1 and 2 above on a medium customarily used for software interchange; or, + + b) Accompany it with a written offer, valid for at least three + years, to give any third party, for a charge no more than your + cost of physically performing source distribution, a complete + machine-readable copy of the corresponding source code, to be + distributed under the terms of Sections 1 and 2 above on a medium + customarily used for software interchange; or, + + c) Accompany it with the information you received as to the offer + to distribute corresponding source code. (This alternative is + allowed only for noncommercial distribution and only if you + received the program in object code or executable form with such + an offer, in accord with Subsection b above.) + +The source code for a work means the preferred form of the work for +making modifications to it. For an executable work, complete source +code means all the source code for all modules it contains, plus any +associated interface definition files, plus the scripts used to +control compilation and installation of the executable. However, as a +special exception, the source code distributed need not include +anything that is normally distributed (in either source or binary +form) with the major components (compiler, kernel, and so on) of the +operating system on which the executable runs, unless that component +itself accompanies the executable. + +If distribution of executable or object code is made by offering +access to copy from a designated place, then offering equivalent +access to copy the source code from the same place counts as +distribution of the source code, even though third parties are not +compelled to copy the source along with the object code. + + 4. You may not copy, modify, sublicense, or distribute the Program +except as expressly provided under this License. Any attempt +otherwise to copy, modify, sublicense or distribute the Program is +void, and will automatically terminate your rights under this License. +However, parties who have received copies, or rights, from you under +this License will not have their licenses terminated so long as such +parties remain in full compliance. + + 5. You are not required to accept this License, since you have not +signed it. However, nothing else grants you permission to modify or +distribute the Program or its derivative works. These actions are +prohibited by law if you do not accept this License. Therefore, by +modifying or distributing the Program (or any work based on the +Program), you indicate your acceptance of this License to do so, and +all its terms and conditions for copying, distributing or modifying +the Program or works based on it. + + 6. Each time you redistribute the Program (or any work based on the +Program), the recipient automatically receives a license from the +original licensor to copy, distribute or modify the Program subject to +these terms and conditions. You may not impose any further +restrictions on the recipients' exercise of the rights granted herein. +You are not responsible for enforcing compliance by third parties to +this License. + + 7. If, as a consequence of a court judgment or allegation of patent +infringement or for any other reason (not limited to patent issues), +conditions are imposed on you (whether by court order, agreement or +otherwise) that contradict the conditions of this License, they do not +excuse you from the conditions of this License. If you cannot +distribute so as to satisfy simultaneously your obligations under this +License and any other pertinent obligations, then as a consequence you +may not distribute the Program at all. For example, if a patent +license would not permit royalty-free redistribution of the Program by +all those who receive copies directly or indirectly through you, then +the only way you could satisfy both it and this License would be to +refrain entirely from distribution of the Program. + +If any portion of this section is held invalid or unenforceable under +any particular circumstance, the balance of the section is intended to +apply and the section as a whole is intended to apply in other +circumstances. + +It is not the purpose of this section to induce you to infringe any +patents or other property right claims or to contest validity of any +such claims; this section has the sole purpose of protecting the +integrity of the free software distribution system, which is +implemented by public license practices. Many people have made +generous contributions to the wide range of software distributed +through that system in reliance on consistent application of that +system; it is up to the author/donor to decide if he or she is willing +to distribute software through any other system and a licensee cannot +impose that choice. + +This section is intended to make thoroughly clear what is believed to +be a consequence of the rest of this License. + + 8. If the distribution and/or use of the Program is restricted in +certain countries either by patents or by copyrighted interfaces, the +original copyright holder who places the Program under this License +may add an explicit geographical distribution limitation excluding +those countries, so that distribution is permitted only in or among +countries not thus excluded. In such case, this License incorporates +the limitation as if written in the body of this License. + + 9. The Free Software Foundation may publish revised and/or new versions +of the General Public License from time to time. Such new versions will +be similar in spirit to the present version, but may differ in detail to +address new problems or concerns. + +Each version is given a distinguishing version number. If the Program +specifies a version number of this License which applies to it and "any +later version", you have the option of following the terms and conditions +either of that version or of any later version published by the Free +Software Foundation. If the Program does not specify a version number of +this License, you may choose any version ever published by the Free Software +Foundation. + + 10. If you wish to incorporate parts of the Program into other free +programs whose distribution conditions are different, write to the author +to ask for permission. For software which is copyrighted by the Free +Software Foundation, write to the Free Software Foundation; we sometimes +make exceptions for this. Our decision will be guided by the two goals +of preserving the free status of all derivatives of our free software and +of promoting the sharing and reuse of software generally. + + NO WARRANTY + + 11. BECAUSE THE PROGRAM IS LICENSED FREE OF CHARGE, THERE IS NO WARRANTY +FOR THE PROGRAM, TO THE EXTENT PERMITTED BY APPLICABLE LAW. EXCEPT WHEN +OTHERWISE STATED IN WRITING THE COPYRIGHT HOLDERS AND/OR OTHER PARTIES +PROVIDE THE PROGRAM "AS IS" WITHOUT WARRANTY OF ANY KIND, EITHER EXPRESSED +OR IMPLIED, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF +MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE. THE ENTIRE RISK AS +TO THE QUALITY AND PERFORMANCE OF THE PROGRAM IS WITH YOU. SHOULD THE +PROGRAM PROVE DEFECTIVE, YOU ASSUME THE COST OF ALL NECESSARY SERVICING, +REPAIR OR CORRECTION. + + 12. IN NO EVENT UNLESS REQUIRED BY APPLICABLE LAW OR AGREED TO IN WRITING +WILL ANY COPYRIGHT HOLDER, OR ANY OTHER PARTY WHO MAY MODIFY AND/OR +REDISTRIBUTE THE PROGRAM AS PERMITTED ABOVE, BE LIABLE TO YOU FOR DAMAGES, +INCLUDING ANY GENERAL, SPECIAL, INCIDENTAL OR CONSEQUENTIAL DAMAGES ARISING +OUT OF THE USE OR INABILITY TO USE THE PROGRAM (INCLUDING BUT NOT LIMITED +TO LOSS OF DATA OR DATA BEING RENDERED INACCURATE OR LOSSES SUSTAINED BY +YOU OR THIRD PARTIES OR A FAILURE OF THE PROGRAM TO OPERATE WITH ANY OTHER +PROGRAMS), EVEN IF SUCH HOLDER OR OTHER PARTY HAS BEEN ADVISED OF THE +POSSIBILITY OF SUCH DAMAGES. + + END OF TERMS AND CONDITIONS + + How to Apply These Terms to Your New Programs + + If you develop a new program, and you want it to be of the greatest +possible use to the public, the best way to achieve this is to make it +free software which everyone can redistribute and change under these terms. + + To do so, attach the following notices to the program. It is safest +to attach them to the start of each source file to most effectively +convey the exclusion of warranty; and each file should have at least +the "copyright" line and a pointer to where the full notice is found. + + + Copyright (C) + + This program is free software; you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation; either version 2 of the License, or + (at your option) any later version. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with this program; if not, write to the Free Software + Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA + + +Also add information on how to contact you by electronic and paper mail. + +If the program is interactive, make it output a short notice like this +when it starts in an interactive mode: + + Gnomovision version 69, Copyright (C) year name of author + Gnomovision comes with ABSOLUTELY NO WARRANTY; for details type `show w'. + This is free software, and you are welcome to redistribute it + under certain conditions; type `show c' for details. + +The hypothetical commands `show w' and `show c' should show the appropriate +parts of the General Public License. Of course, the commands you use may +be called something other than `show w' and `show c'; they could even be +mouse-clicks or menu items--whatever suits your program. + +You should also get your employer (if you work as a programmer) or your +school, if any, to sign a "copyright disclaimer" for the program, if +necessary. Here is a sample; alter the names: + + Yoyodyne, Inc., hereby disclaims all copyright interest in the program + `Gnomovision' (which makes passes at compilers) written by James Hacker. + + , 1 April 1989 + Ty Coon, President of Vice + +This General Public License does not permit incorporating your program into +proprietary programs. If your program is a subroutine library, you may +consider it more useful to permit linking proprietary applications with the +library. If this is what you want to do, use the GNU Library General +Public License instead of this License. diff --git a/MANIFEST.in b/MANIFEST.in new file mode 100644 index 0000000..116c2e6 --- /dev/null +++ b/MANIFEST.in @@ -0,0 +1,7 @@ +include bugzilla/* +include data/*.c +include data/ksc.conf +include ksc.1 +include COPYING +include README +include *.py diff --git a/PKG-INFO b/PKG-INFO new file mode 100644 index 0000000..1f45737 --- /dev/null +++ b/PKG-INFO @@ -0,0 +1,10 @@ +Metadata-Version: 1.6 +Name: ksc +Version: 1.6 +Summary: ksc tool +Home-page: http://redhat.com +Author: Kushal Das, Samikshan Bairagya, Stanislav Kozina, Martin Lacko, Ziqian Sun +Author-email: kdas@redhat.com, sbairagy@redhat.com, skozina@redhat.com, mlacko@redhat.com, zsun@redhat.com +License: http://www.gnu.org/copyleft/gpl.html +Description: Kernel Module Source Checker tool +Platform: Linux diff --git a/README b/README new file mode 100644 index 0000000..7a90e06 --- /dev/null +++ b/README @@ -0,0 +1,37 @@ +ksc tool +======== +A tool to check whitelist symbol usage in kernel module source code. + +Options: + -h, --help show this help message and exit + -c CONFIG, --config=CONFIG + path to configuration file + -d DIRECTORY, --directory=DIRECTORY + path to the directory + -i, --internal to create text files to be used internally + -k KO, --ko=KO path to the ko file + -n RELEASENAME, --name=RELEASENAME + Red Hat release against which the bug is to be filed. Default value is 7.0 + -p PREVIOUS, --previous=PREVIOUS + path to previous resultset to submit as bug + -r RELEASE, --release=RELEASE + RHEL whitelist release to compare against + -y SYMVERS, --symvers=SYMVERS + Path to the Module.symvers file. The current kernel + path is used if not specified. + -s, --submit Submit to Red Hat Bugzilla + -v, --version Prints KSC version number + +Valid architectures for binary modules are x86_64, ppc64, s390x. +The total symbol usage number will vary depending on which architecture it is running +against as it depends on the symbols provided by Red Hat kernel on that +architecture. + +Example command and output: + +$ ksc -d ../drivers/ipw2200-1.1.4 + + +Total symbol usage: 71 Total Non white list symbol usage: 3 +Score: 95.77% + diff --git a/bugzilla/__init__.py b/bugzilla/__init__.py new file mode 100644 index 0000000..74f5514 --- /dev/null +++ b/bugzilla/__init__.py @@ -0,0 +1,39 @@ +# python-bugzilla - a Python interface to bugzilla using xmlrpclib. +# +# Copyright (C) 2007, 2008 Red Hat Inc. +# Author: Will Woods +# +# This program is free software; you can redistribute it and/or modify it +# under the terms of the GNU General Public License as published by the +# Free Software Foundation; either version 2 of the License, or (at your +# option) any later version. See http://www.gnu.org/copyleft/gpl.html for +# the full text of the license. + +from .apiversion import version, __version__ +from .base import Bugzilla +from .transport import BugzillaError +from .rhbugzilla import RHBugzilla +from .oldclasses import (Bugzilla3, Bugzilla32, Bugzilla34, Bugzilla36, + Bugzilla4, Bugzilla42, Bugzilla44, + NovellBugzilla, RHBugzilla3, RHBugzilla4) + + +# This is the public API. If you are explicitly instantiating any other +# class, using some function, or poking into internal files, don't complain +# if things break on you. +__all__ = [ + "Bugzilla3", "Bugzilla32", "Bugzilla34", "Bugzilla36", + "Bugzilla4", "Bugzilla42", "Bugzilla44", + "NovellBugzilla", + "RHBugzilla3", "RHBugzilla4", "RHBugzilla", + 'BugzillaError', + 'Bugzilla', "version", +] + + +# Clear all other locals() from the public API +for __sym in locals().copy(): + if __sym.startswith("__") or __sym in __all__: + continue + locals().pop(__sym) +locals().pop("__sym") diff --git a/bugzilla/_cli.py b/bugzilla/_cli.py new file mode 100755 index 0000000..fae2370 --- /dev/null +++ b/bugzilla/_cli.py @@ -0,0 +1,1182 @@ +#!/usr/libexec/platform-python +# +# bugzilla - a commandline frontend for the python bugzilla module +# +# Copyright (C) 2007-2017 Red Hat Inc. +# Author: Will Woods +# Author: Cole Robinson +# +# This program is free software; you can redistribute it and/or modify it +# under the terms of the GNU General Public License as published by the +# Free Software Foundation; either version 2 of the License, or (at your +# option) any later version. See http://www.gnu.org/copyleft/gpl.html for +# the full text of the license. + +from __future__ import print_function + +import locale +from logging import getLogger, DEBUG, INFO, WARN, StreamHandler, Formatter +import argparse +import os +import re +import socket +import sys +import tempfile + +# pylint: disable=import-error +if sys.version_info[0] >= 3: + # pylint: disable=no-name-in-module,redefined-builtin + from xmlrpc.client import Fault, ProtocolError + from urllib.parse import urlparse + basestring = (str, bytes) +else: + from xmlrpclib import Fault, ProtocolError + from urlparse import urlparse +# pylint: enable=import-error + +import requests.exceptions + +import bugzilla + +DEFAULT_BZ = 'https://bugzilla.redhat.com/xmlrpc.cgi' + +format_field_re = re.compile("%{([a-z0-9_]+)(?::([^}]*))?}") + +log = getLogger(bugzilla.__name__) + + +################ +# Util helpers # +################ + +def _is_unittest(): + return bool(os.getenv("__BUGZILLA_UNITTEST")) + + +def _is_unittest_debug(): + return bool(os.getenv("__BUGZILLA_UNITTEST_DEBUG")) + + +def to_encoding(ustring): + string = '' + if isinstance(ustring, basestring): + string = ustring + elif ustring is not None: + string = str(ustring) + + if sys.version_info[0] >= 3: + return string + + preferred = locale.getpreferredencoding() + if _is_unittest(): + preferred = "UTF-8" + return string.encode(preferred, 'replace') + + +def open_without_clobber(name, *args): + """ + Try to open the given file with the given mode; if that filename exists, + try "name.1", "name.2", etc. until we find an unused filename. + """ + fd = None + count = 1 + orig_name = name + while fd is None: + try: + fd = os.open(name, os.O_CREAT | os.O_EXCL, 0o666) + except OSError as err: + if err.errno == os.errno.EEXIST: + name = "%s.%i" % (orig_name, count) + count += 1 + else: + raise IOError(err.errno, err.strerror, err.filename) + fobj = open(name, *args) + if fd != fobj.fileno(): + os.close(fd) + return fobj + + +def get_default_url(): + """ + Grab a default URL from bugzillarc [DEFAULT] url=X + """ + from bugzilla.base import _open_bugzillarc + cfg = _open_bugzillarc() + if cfg: + cfgurl = cfg.defaults().get("url", None) + if cfgurl is not None: + log.debug("bugzillarc: found cli url=%s", cfgurl) + return cfgurl + return DEFAULT_BZ + + +def setup_logging(debug, verbose): + handler = StreamHandler(sys.stderr) + handler.setFormatter(Formatter( + "[%(asctime)s] %(levelname)s (%(module)s:%(lineno)d) %(message)s", + "%H:%M:%S")) + log.addHandler(handler) + + if debug: + log.setLevel(DEBUG) + elif verbose: + log.setLevel(INFO) + else: + log.setLevel(WARN) + + if _is_unittest_debug(): + log.setLevel(DEBUG) + + +################## +# Option parsing # +################## + +def _setup_root_parser(): + epilog = 'Try "bugzilla COMMAND --help" for command-specific help.' + p = argparse.ArgumentParser(epilog=epilog) + + default_url = get_default_url() + + # General bugzilla connection options + p.add_argument('--bugzilla', default=default_url, + help="bugzilla XMLRPC URI. default: %s" % default_url) + p.add_argument("--nosslverify", dest="sslverify", + action="store_false", default=True, + help="Don't error on invalid bugzilla SSL certificate") + p.add_argument('--cert', + help="client side certificate file needed by the webserver") + + p.add_argument('--login', action="store_true", + help='Run interactive "login" before performing the ' + 'specified command.') + p.add_argument('--username', help="Log in with this username") + p.add_argument('--password', help="Log in with this password") + + p.add_argument('--ensure-logged-in', action="store_true", + help="Raise an error if we aren't logged in to bugzilla. " + "Consider using this if you are depending on " + "cached credentials, to ensure that when they expire the " + "tool errors, rather than subtly change output.") + p.add_argument('--no-cache-credentials', + action='store_false', default=True, dest='cache_credentials', + help="Don't save any bugzilla cookies or tokens to disk, and " + "don't use any pre-existing credentials.") + + p.add_argument('--cookiefile', default=None, + help="cookie file to use for bugzilla authentication") + p.add_argument('--tokenfile', default=None, + help="token file to use for bugzilla authentication") + + p.add_argument('--verbose', action='store_true', + help="give more info about what's going on") + p.add_argument('--debug', action='store_true', + help="output bunches of debugging info") + p.add_argument('--version', action='version', + version=bugzilla.__version__) + + # Allow user to specify BZClass to initialize. Kinda weird for the + # CLI, I'd rather people file bugs about this so we can fix our detection. + # So hide it from the help output but keep it for back compat + p.add_argument('--bztype', default='auto', help=argparse.SUPPRESS) + + return p + + +def _parser_add_output_options(p): + outg = p.add_argument_group("Output format options") + outg.add_argument('--full', action='store_const', dest='output', + const='full', default='normal', + help="output detailed bug info") + outg.add_argument('-i', '--ids', action='store_const', dest='output', + const='ids', help="output only bug IDs") + outg.add_argument('-e', '--extra', action='store_const', + dest='output', const='extra', + help="output additional bug information " + "(keywords, Whiteboards, etc.)") + outg.add_argument('--oneline', action='store_const', dest='output', + const='oneline', + help="one line summary of the bug (useful for scripts)") + outg.add_argument('--raw', action='store_const', dest='output', + const='raw', help="raw output of the bugzilla contents") + outg.add_argument('--outputformat', + help="Print output in the form given. " + "You can use RPM-style tags that match bug " + "fields, e.g.: '%%{id}: %%{summary}'. See the man page " + "section 'Output options' for more details.") + + +def _parser_add_bz_fields(rootp, command): + cmd_new = (command == "new") + cmd_query = (command == "query") + cmd_modify = (command == "modify") + if cmd_new: + comment_help = "Set initial bug comment/description" + elif cmd_query: + comment_help = "Search all bug comments" + else: + comment_help = "Add new bug comment" + + p = rootp.add_argument_group("Standard bugzilla options") + + p.add_argument('-p', '--product', help="Product name") + p.add_argument('-v', '--version', help="Product version") + p.add_argument('-c', '--component', help="Component name") + p.add_argument('-t', '--summary', '--short_desc', help="Bug summary") + p.add_argument('-l', '--comment', '--long_desc', help=comment_help) + if not cmd_query: + p.add_argument("--comment-tag", action="append", + help="Comment tag for the new comment") + p.add_argument("--sub-component", action="append", + help="RHBZ sub component field") + p.add_argument('-o', '--os', help="Operating system") + p.add_argument('--arch', help="Arch this bug occurs on") + p.add_argument('-x', '--severity', help="Bug severity") + p.add_argument('-z', '--priority', help="Bug priority") + p.add_argument('--alias', help='Bug alias (name)') + p.add_argument('-s', '--status', '--bug_status', + help='Bug status (NEW, ASSIGNED, etc.)') + p.add_argument('-u', '--url', help="URL field") + p.add_argument('-m', '--target_milestone', help="Target milestone") + p.add_argument('--target_release', help="RHBZ Target release") + + p.add_argument('--blocked', action="append", + help="Bug IDs that this bug blocks") + p.add_argument('--dependson', action="append", + help="Bug IDs that this bug depends on") + p.add_argument('--keywords', action="append", + help="Bug keywords") + p.add_argument('--groups', action="append", + help="Which user groups can view this bug") + + p.add_argument('--cc', action="append", help="CC list") + p.add_argument('-a', '--assigned_to', '--assignee', help="Bug assignee") + p.add_argument('-q', '--qa_contact', help='QA contact') + + if not cmd_new: + p.add_argument('-f', '--flag', action='append', + help="Bug flags state. Ex:\n" + " --flag needinfo?\n" + " --flag dev_ack+ \n" + " clear with --flag needinfoX") + p.add_argument("--tags", action="append", + help="Tags/Personal Tags field.") + + p.add_argument('-w', "--whiteboard", '--status_whiteboard', + action="append", help='Whiteboard field') + p.add_argument("--devel_whiteboard", action="append", + help='RHBZ devel whiteboard field') + p.add_argument("--internal_whiteboard", action="append", + help='RHBZ internal whiteboard field') + p.add_argument("--qa_whiteboard", action="append", + help='RHBZ QA whiteboard field') + p.add_argument('-F', '--fixed_in', + help="RHBZ 'Fixed in version' field") + + # Put this at the end, so it sticks out more + p.add_argument('--field', + metavar="FIELD=VALUE", action="append", dest="fields", + help="Manually specify a bugzilla XMLRPC field. FIELD is " + "the raw name used by the bugzilla instance. For example if your " + "bugzilla instance has a custom field cf_my_field, do:\n" + " --field cf_my_field=VALUE") + + # Used by unit tests, not for end user consumption + p.add_argument('--__test-return-result', action="store_true", + dest="test_return_result", help=argparse.SUPPRESS) + + if not cmd_modify: + _parser_add_output_options(rootp) + + +def _setup_action_new_parser(subparsers): + description = ("Create a new bug report. " + "--product, --component, --version, --summary, and --comment " + "must be specified. " + "Options that take multiple values accept comma separated lists, " + "including --cc, --blocks, --dependson, --groups, and --keywords.") + p = subparsers.add_parser("new", description=description) + + _parser_add_bz_fields(p, "new") + + +def _setup_action_query_parser(subparsers): + description = ("List bug reports that match the given criteria. " + "Certain options can accept a comma separated list to query multiple " + "values, including --status, --component, --product, --version, --id.") + epilog = ("Note: querying via explicit command line options will only " + "get you so far. See the --from-url option for a way to use powerful " + "Web UI queries from the command line.") + p = subparsers.add_parser("query", + description=description, epilog=epilog) + + _parser_add_bz_fields(p, "query") + + g = p.add_argument_group("'query' specific options") + g.add_argument('-b', '--id', '--bug_id', + help="specify individual bugs by IDs, separated with commas") + g.add_argument('-r', '--reporter', + help="Email: search reporter email for given address") + g.add_argument('--quicksearch', + help="Search using bugzilla's quicksearch functionality.") + g.add_argument('--savedsearch', + help="Name of a bugzilla saved search. If you don't own this " + "saved search, you must passed --savedsearch_sharer_id.") + g.add_argument('--savedsearch-sharer-id', + help="Owner ID of the --savedsearch. You can get this ID from " + "the URL bugzilla generates when running the saved search " + "from the web UI.") + + # Keep this at the end so it sticks out more + g.add_argument('--from-url', metavar="WEB_QUERY_URL", + help="Make a working query via bugzilla's 'Advanced search' web UI, " + "grab the url from your browser (the string with query.cgi or " + "buglist.cgi in it), and --from-url will run it via the " + "bugzilla API. Don't forget to quote the string! " + "This only works for Bugzilla 5 and Red Hat bugzilla") + + # Deprecated options + p.add_argument('-E', '--emailtype', help=argparse.SUPPRESS) + p.add_argument('--components_file', help=argparse.SUPPRESS) + p.add_argument('-U', '--url_type', + help=argparse.SUPPRESS) + p.add_argument('-K', '--keywords_type', + help=argparse.SUPPRESS) + p.add_argument('-W', '--status_whiteboard_type', + help=argparse.SUPPRESS) + p.add_argument('-B', '--booleantype', + help=argparse.SUPPRESS) + p.add_argument('--boolean_query', action="append", + help=argparse.SUPPRESS) + p.add_argument('--fixed_in_type', help=argparse.SUPPRESS) + + +def _setup_action_info_parser(subparsers): + description = ("List products or component information about the " + "bugzilla server.") + p = subparsers.add_parser("info", description=description) + + x = p.add_mutually_exclusive_group(required=True) + x.add_argument('-p', '--products', action='store_true', + help='Get a list of products') + x.add_argument('-c', '--components', metavar="PRODUCT", + help='List the components in the given product') + x.add_argument('-o', '--component_owners', metavar="PRODUCT", + help='List components (and their owners)') + x.add_argument('-v', '--versions', metavar="PRODUCT", + help='List the versions for the given product') + p.add_argument('--active-components', action="store_true", + help='Only show active components. Combine with --components*') + + + +def _setup_action_modify_parser(subparsers): + usage = ("bugzilla modify [options] BUGID [BUGID...]\n" + "Fields that take multiple values have a special input format.\n" + "Append: --cc=foo@example.com\n" + "Overwrite: --cc==foo@example.com\n" + "Remove: --cc=-foo@example.com\n" + "Options that accept this format: --cc, --blocked, --dependson,\n" + " --groups, --tags, whiteboard fields.") + p = subparsers.add_parser("modify", usage=usage) + + _parser_add_bz_fields(p, "modify") + + g = p.add_argument_group("'modify' specific options") + g.add_argument("ids", nargs="+", help="Bug IDs to modify") + g.add_argument('-k', '--close', metavar="RESOLUTION", + help='Close with the given resolution (WONTFIX, NOTABUG, etc.)') + g.add_argument('-d', '--dupeid', metavar="ORIGINAL", + help='ID of original bug. Implies --close DUPLICATE') + g.add_argument('--private', action='store_true', default=False, + help='Mark new comment as private') + g.add_argument('--reset-assignee', action="store_true", + help='Reset assignee to component default') + g.add_argument('--reset-qa-contact', action="store_true", + help='Reset QA contact to component default') + + +def _setup_action_attach_parser(subparsers): + usage = """ +bugzilla attach --file=FILE --desc=DESC [--type=TYPE] BUGID [BUGID...] +bugzilla attach --get=ATTACHID --getall=BUGID [...] +bugzilla attach --type=TYPE BUGID [BUGID...]""" + description = "Attach files or download attachments." + p = subparsers.add_parser("attach", description=description, usage=usage) + + p.add_argument("ids", nargs="*", help="BUGID references") + p.add_argument('-f', '--file', metavar="FILENAME", + help='File to attach, or filename for data provided on stdin') + p.add_argument('-d', '--description', '--summary', + metavar="SUMMARY", dest='desc', + help="A short summary of the file being attached") + p.add_argument('-t', '--type', metavar="MIMETYPE", + help="Mime-type for the file being attached") + p.add_argument('-g', '--get', metavar="ATTACHID", action="append", + default=[], help="Download the attachment with the given ID") + p.add_argument("--getall", "--get-all", metavar="BUGID", action="append", + default=[], help="Download all attachments on the given bug") + p.add_argument('-l', '--comment', '--long_desc', + help="Add comment with attachment") + + +def _setup_action_login_parser(subparsers): + usage = 'bugzilla login [username [password]]' + description = "Log into bugzilla and save a login cookie or token." + p = subparsers.add_parser("login", description=description, usage=usage) + p.add_argument("pos_username", nargs="?", help="Optional username", + metavar="username") + p.add_argument("pos_password", nargs="?", help="Optional password", + metavar="password") + + +def setup_parser(): + rootparser = _setup_root_parser() + subparsers = rootparser.add_subparsers(dest="command") + subparsers.required = True + _setup_action_new_parser(subparsers) + _setup_action_query_parser(subparsers) + _setup_action_info_parser(subparsers) + _setup_action_modify_parser(subparsers) + _setup_action_attach_parser(subparsers) + _setup_action_login_parser(subparsers) + return rootparser + + +#################### +# Command routines # +#################### + +def _merge_field_opts(query, opt, parser): + # Add any custom fields if specified + if opt.fields is None: + return + + for f in opt.fields: + try: + f, v = f.split('=', 1) + query[f] = v + except Exception: + parser.error("Invalid field argument provided: %s" % (f)) + + +def _do_query(bz, opt, parser): + q = {} + + # Parse preconstructed queries. + u = opt.from_url + if u: + q = bz.url_to_query(u) + + if opt.components_file: + # Components slurped in from file (one component per line) + # This can be made more robust + clist = [] + f = open(opt.components_file, 'r') + for line in f.readlines(): + line = line.rstrip("\n") + clist.append(line) + opt.component = clist + + if opt.status: + val = opt.status + stat = val + if val == 'ALL': + # leaving this out should return bugs of any status + stat = None + elif val == 'DEV': + # Alias for all development bug statuses + stat = ['NEW', 'ASSIGNED', 'NEEDINFO', 'ON_DEV', + 'MODIFIED', 'POST', 'REOPENED'] + elif val == 'QE': + # Alias for all QE relevant bug statuses + stat = ['ASSIGNED', 'ON_QA', 'FAILS_QA', 'PASSES_QA'] + elif val == 'EOL': + # Alias for EndOfLife bug statuses + stat = ['VERIFIED', 'RELEASE_PENDING', 'CLOSED'] + elif val == 'OPEN': + # non-Closed statuses + stat = ['NEW', 'ASSIGNED', 'MODIFIED', 'ON_DEV', 'ON_QA', + 'VERIFIED', 'RELEASE_PENDING', 'POST'] + opt.status = stat + + # Convert all comma separated list parameters to actual lists, + # which is what bugzilla wants + # According to bugzilla docs, any parameter can be a list, but + # let's only do this for options we explicitly mention can be + # comma separated. + for optname in ["severity", "id", "status", "component", + "priority", "product", "version"]: + val = getattr(opt, optname, None) + if not isinstance(val, str): + continue + setattr(opt, optname, val.split(",")) + + include_fields = None + if opt.output == 'raw': + # 'raw' always does a getbug() call anyways, so just ask for ID back + include_fields = ['id'] + + elif opt.outputformat: + include_fields = [] + for fieldname, rest in format_field_re.findall(opt.outputformat): + if fieldname == "whiteboard" and rest: + fieldname = rest + "_" + fieldname + elif fieldname == "flag": + fieldname = "flags" + elif fieldname == "cve": + fieldname = ["keywords", "blocks"] + elif fieldname == "__unicode__": + # Needs to be in sync with bug.__unicode__ + fieldname = ["id", "status", "assigned_to", "summary"] + + flist = isinstance(fieldname, list) and fieldname or [fieldname] + for f in flist: + if f not in include_fields: + include_fields.append(f) + + if include_fields is not None: + include_fields.sort() + + built_query = bz.build_query( + product=opt.product or None, + component=opt.component or None, + sub_component=opt.sub_component or None, + version=opt.version or None, + reporter=opt.reporter or None, + bug_id=opt.id or None, + short_desc=opt.summary or None, + long_desc=opt.comment or None, + cc=opt.cc or None, + assigned_to=opt.assigned_to or None, + qa_contact=opt.qa_contact or None, + status=opt.status or None, + blocked=opt.blocked or None, + dependson=opt.dependson or None, + keywords=opt.keywords or None, + keywords_type=opt.keywords_type or None, + url=opt.url or None, + url_type=opt.url_type or None, + status_whiteboard=opt.whiteboard or None, + status_whiteboard_type=opt.status_whiteboard_type or None, + fixed_in=opt.fixed_in or None, + fixed_in_type=opt.fixed_in_type or None, + flag=opt.flag or None, + alias=opt.alias or None, + qa_whiteboard=opt.qa_whiteboard or None, + devel_whiteboard=opt.devel_whiteboard or None, + boolean_query=opt.boolean_query or None, + bug_severity=opt.severity or None, + priority=opt.priority or None, + target_release=opt.target_release or None, + target_milestone=opt.target_milestone or None, + emailtype=opt.emailtype or None, + booleantype=opt.booleantype or None, + include_fields=include_fields, + quicksearch=opt.quicksearch or None, + savedsearch=opt.savedsearch or None, + savedsearch_sharer_id=opt.savedsearch_sharer_id or None, + tags=opt.tags or None) + + _merge_field_opts(built_query, opt, parser) + + built_query.update(q) + q = built_query + + if not q: + parser.error("'query' command requires additional arguments") + if opt.test_return_result: + return q + return bz.query(q) + + +def _do_info(bz, opt): + """ + Handle the 'info' subcommand + """ + # All these commands call getproducts internally, so do it up front + # with minimal include_fields for speed + def _filter_components(compdetails): + ret = {} + for k, v in compdetails.items(): + if v.get("is_active", True): + ret[k] = v + return ret + + productname = (opt.components or opt.component_owners or opt.versions) + include_fields = ["name", "id"] + fastcomponents = (opt.components and not opt.active_components) + if opt.versions: + include_fields += ["versions"] + if opt.component_owners: + include_fields += [ + "components.default_assigned_to", + "components.name", + ] + if (opt.active_components and + any(["components" in i for i in include_fields])): + include_fields += ["components.is_active"] + + bz.refresh_products(names=productname and [productname] or None, + include_fields=include_fields) + + if opt.products: + for name in sorted([p["name"] for p in bz.getproducts()]): + print(name) + + elif fastcomponents: + for name in sorted(bz.getcomponents(productname)): + print(name) + + elif opt.components: + details = bz.getcomponentsdetails(productname) + for name in sorted(_filter_components(details)): + print(name) + + elif opt.versions: + proddict = bz.getproducts()[0] + for v in proddict['versions']: + print(to_encoding(v["name"])) + + elif opt.component_owners: + details = bz.getcomponentsdetails(productname) + for c in sorted(_filter_components(details)): + print(to_encoding(u"%s: %s" % (c, + details[c]['default_assigned_to']))) + + +def _convert_to_outputformat(output): + fmt = "" + + if output == "normal": + fmt = "%{__unicode__}" + + elif output == "ids": + fmt = "%{id}" + + elif output == 'full': + fmt += "%{__unicode__}\n" + fmt += "Component: %{component}\n" + fmt += "CC: %{cc}\n" + fmt += "Blocked: %{blocks}\n" + fmt += "Depends: %{depends_on}\n" + fmt += "%{comments}\n" + + elif output == 'extra': + fmt += "%{__unicode__}\n" + fmt += " +Keywords: %{keywords}\n" + fmt += " +QA Whiteboard: %{qa_whiteboard}\n" + fmt += " +Status Whiteboard: %{status_whiteboard}\n" + fmt += " +Devel Whiteboard: %{devel_whiteboard}\n" + + elif output == 'oneline': + fmt += "#%{bug_id} %{status} %{assigned_to} %{component}\t" + fmt += "[%{target_milestone}] %{flags} %{cve}" + + else: + raise RuntimeError("Unknown output type '%s'" % output) + + return fmt + + +def _format_output(bz, opt, buglist): + if opt.output == 'raw': + buglist = bz.getbugs([b.bug_id for b in buglist]) + for b in buglist: + print("Bugzilla %s: " % b.bug_id) + for attrname in sorted(b.__dict__): + print(to_encoding(u"ATTRIBUTE[%s]: %s" % + (attrname, b.__dict__[attrname]))) + print("\n\n") + return + + def bug_field(matchobj): + # whiteboard and flag allow doing + # %{whiteboard:devel} and %{flag:needinfo} + # That's what 'rest' matches + (fieldname, rest) = matchobj.groups() + + if fieldname == "whiteboard" and rest: + fieldname = rest + "_" + fieldname + + if fieldname == "flag" and rest: + val = b.get_flag_status(rest) + + elif fieldname == "flags" or fieldname == "flags_requestee": + tmpstr = [] + for f in getattr(b, "flags", []): + requestee = f.get('requestee', "") + if fieldname == "flags": + requestee = "" + if fieldname == "flags_requestee": + if requestee == "": + continue + tmpstr.append("%s" % requestee) + else: + tmpstr.append("%s%s%s" % + (f['name'], f['status'], requestee)) + + val = ",".join(tmpstr) + + elif fieldname == "cve": + cves = [] + for key in getattr(b, "keywords", []): + # grab CVE from keywords and blockers + if key.find("Security") == -1: + continue + for bl in b.blocks: + cvebug = bz.getbug(bl) + for cb in cvebug.alias: + if cb.find("CVE") == -1: + continue + if cb.strip() not in cves: + cves.append(cb) + val = ",".join(cves) + + elif fieldname == "comments": + val = "" + for c in getattr(b, "comments", []): + val += ("\n* %s - %s:\n%s\n" % (c['time'], + c.get("creator", c.get("author", "")), c['text'])) + + elif fieldname == "external_bugs": + val = "" + for e in getattr(b, "external_bugs", []): + url = e["type"]["full_url"].replace("%id%", e["ext_bz_bug_id"]) + if not val: + val += "\n" + val += "External bug: %s\n" % url + + elif fieldname == "__unicode__": + val = b.__unicode__() + else: + val = getattr(b, fieldname, "") + + vallist = isinstance(val, list) and val or [val] + val = ','.join([to_encoding(v) for v in vallist]) + + return val + + for b in buglist: + print(format_field_re.sub(bug_field, opt.outputformat)) + + +def _parse_triset(vallist, checkplus=True, checkminus=True, checkequal=True, + splitcomma=False): + add_val = [] + rm_val = [] + set_val = None + + def make_list(v): + if not v: + return [] + if splitcomma: + return v.split(",") + return [v] + + for val in isinstance(vallist, list) and vallist or [vallist]: + val = val or "" + + if val.startswith("+") and checkplus: + add_val += make_list(val[1:]) + elif val.startswith("-") and checkminus: + rm_val += make_list(val[1:]) + elif val.startswith("=") and checkequal: + # Intentionally overwrite this + set_val = make_list(val[1:]) + else: + add_val += make_list(val) + + return add_val, rm_val, set_val + + +def _do_new(bz, opt, parser): + # Parse options that accept comma separated list + def parse_multi(val): + return _parse_triset(val, checkplus=False, checkminus=False, + checkequal=False, splitcomma=True)[0] + + ret = bz.build_createbug( + blocks=parse_multi(opt.blocked) or None, + cc=parse_multi(opt.cc) or None, + component=opt.component or None, + depends_on=parse_multi(opt.dependson) or None, + description=opt.comment or None, + groups=parse_multi(opt.groups) or None, + keywords=parse_multi(opt.keywords) or None, + op_sys=opt.os or None, + platform=opt.arch or None, + priority=opt.priority or None, + product=opt.product or None, + severity=opt.severity or None, + summary=opt.summary or None, + url=opt.url or None, + version=opt.version or None, + assigned_to=opt.assigned_to or None, + qa_contact=opt.qa_contact or None, + sub_component=opt.sub_component or None, + alias=opt.alias or None, + comment_tags=opt.comment_tag or None, + ) + + _merge_field_opts(ret, opt, parser) + + if opt.test_return_result: + return ret + + b = bz.createbug(ret) + b.refresh() + return [b] + + +def _do_modify(bz, parser, opt): + bugid_list = [bugid for a in opt.ids for bugid in a.split(',')] + + add_wb, rm_wb, set_wb = _parse_triset(opt.whiteboard) + add_devwb, rm_devwb, set_devwb = _parse_triset(opt.devel_whiteboard) + add_intwb, rm_intwb, set_intwb = _parse_triset(opt.internal_whiteboard) + add_qawb, rm_qawb, set_qawb = _parse_triset(opt.qa_whiteboard) + + add_blk, rm_blk, set_blk = _parse_triset(opt.blocked, splitcomma=True) + add_deps, rm_deps, set_deps = _parse_triset(opt.dependson, splitcomma=True) + add_key, rm_key, set_key = _parse_triset(opt.keywords) + add_cc, rm_cc, ignore = _parse_triset(opt.cc, + checkplus=False, + checkequal=False) + add_groups, rm_groups, ignore = _parse_triset(opt.groups, + checkequal=False, + splitcomma=True) + add_tags, rm_tags, ignore = _parse_triset(opt.tags, checkequal=False) + + status = opt.status or None + if opt.dupeid is not None: + opt.close = "DUPLICATE" + if opt.close: + status = "CLOSED" + + flags = [] + if opt.flag: + # Convert "foo+" to tuple ("foo", "+") + for f in opt.flag: + flags.append({"name": f[:-1], "status": f[-1]}) + + update = bz.build_update( + assigned_to=opt.assigned_to or None, + comment=opt.comment or None, + comment_private=opt.private or None, + component=opt.component or None, + product=opt.product or None, + blocks_add=add_blk or None, + blocks_remove=rm_blk or None, + blocks_set=set_blk, + url=opt.url or None, + cc_add=add_cc or None, + cc_remove=rm_cc or None, + depends_on_add=add_deps or None, + depends_on_remove=rm_deps or None, + depends_on_set=set_deps, + groups_add=add_groups or None, + groups_remove=rm_groups or None, + keywords_add=add_key or None, + keywords_remove=rm_key or None, + keywords_set=set_key, + op_sys=opt.os or None, + platform=opt.arch or None, + priority=opt.priority or None, + qa_contact=opt.qa_contact or None, + severity=opt.severity or None, + status=status, + summary=opt.summary or None, + version=opt.version or None, + reset_assigned_to=opt.reset_assignee or None, + reset_qa_contact=opt.reset_qa_contact or None, + resolution=opt.close or None, + target_release=opt.target_release or None, + target_milestone=opt.target_milestone or None, + dupe_of=opt.dupeid or None, + fixed_in=opt.fixed_in or None, + whiteboard=set_wb and set_wb[0] or None, + devel_whiteboard=set_devwb and set_devwb[0] or None, + internal_whiteboard=set_intwb and set_intwb[0] or None, + qa_whiteboard=set_qawb and set_qawb[0] or None, + sub_component=opt.sub_component or None, + alias=opt.alias or None, + flags=flags or None, + comment_tags=opt.comment_tag or None, + ) + + # We make this a little convoluted to facilitate unit testing + wbmap = { + "whiteboard": (add_wb, rm_wb), + "internal_whiteboard": (add_intwb, rm_intwb), + "qa_whiteboard": (add_qawb, rm_qawb), + "devel_whiteboard": (add_devwb, rm_devwb), + } + + for k, v in wbmap.copy().items(): + if not v[0] and not v[1]: + del(wbmap[k]) + + _merge_field_opts(update, opt, parser) + + log.debug("update bug dict=%s", update) + log.debug("update whiteboard dict=%s", wbmap) + + if not any([update, wbmap, add_tags, rm_tags]): + parser.error("'modify' command requires additional arguments") + + if opt.test_return_result: + return (update, wbmap, add_tags, rm_tags) + + if add_tags or rm_tags: + ret = bz.update_tags(bugid_list, + tags_add=add_tags, tags_remove=rm_tags) + log.debug("bz.update_tags returned=%s", ret) + if update: + ret = bz.update_bugs(bugid_list, update) + log.debug("bz.update_bugs returned=%s", ret) + + if not wbmap: + return + + # Now for the things we can't blindly batch. + # Being able to prepend/append to whiteboards, which are just + # plain string values, is an old rhbz semantic that we try to maintain + # here. This is a bit weird for traditional bugzilla XMLRPC + log.debug("Adjusting whiteboard fields one by one") + for bug in bz.getbugs(bugid_list): + for wb, (add_list, rm_list) in wbmap.items(): + for tag in add_list: + newval = getattr(bug, wb) or "" + if newval: + newval += " " + newval += tag + bz.update_bugs([bug.id], + bz.build_update(**{wb: newval})) + + for tag in rm_list: + newval = (getattr(bug, wb) or "").split() + for t in newval[:]: + if t == tag: + newval.remove(t) + bz.update_bugs([bug.id], + bz.build_update(**{wb: " ".join(newval)})) + + +def _do_get_attach(bz, opt): + for bug in bz.getbugs(opt.getall): + opt.get += bug.get_attachment_ids() + + for attid in set(opt.get): + att = bz.openattachment(attid) + outfile = open_without_clobber(att.name, "wb") + data = att.read(4096) + while data: + outfile.write(data) + data = att.read(4096) + print("Wrote %s" % outfile.name) + + return + + +def _do_set_attach(bz, opt, parser): + if not opt.ids: + parser.error("Bug ID must be specified for setting attachments") + + if sys.stdin.isatty(): + if not opt.file: + parser.error("--file must be specified") + fileobj = open(opt.file, "rb") + else: + # piped input on stdin + if not opt.desc: + parser.error("--description must be specified if passing " + "file on stdin") + + fileobj = tempfile.NamedTemporaryFile(prefix="bugzilla-attach.") + data = sys.stdin.read(4096) + + while data: + fileobj.write(data.encode(locale.getpreferredencoding())) + data = sys.stdin.read(4096) + fileobj.seek(0) + + kwargs = {} + if opt.file: + kwargs["filename"] = os.path.basename(opt.file) + if opt.type: + kwargs["contenttype"] = opt.type + if opt.type in ["text/x-patch"]: + kwargs["ispatch"] = True + if opt.comment: + kwargs["comment"] = opt.comment + desc = opt.desc or os.path.basename(fileobj.name) + + # Upload attachments + for bugid in opt.ids: + attid = bz.attachfile(bugid, fileobj, desc, **kwargs) + print("Created attachment %i on bug %s" % (attid, bugid)) + + +################# +# Main handling # +################# + +def _make_bz_instance(opt): + """ + Build the Bugzilla instance we will use + """ + if opt.bztype != 'auto': + log.info("Explicit --bztype is no longer supported, ignoring") + + cookiefile = None + tokenfile = None + if opt.cache_credentials: + cookiefile = opt.cookiefile or -1 + tokenfile = opt.tokenfile or -1 + + bz = bugzilla.Bugzilla( + url=opt.bugzilla, + cookiefile=cookiefile, + tokenfile=tokenfile, + sslverify=opt.sslverify, + cert=opt.cert) + return bz + + +def _handle_login(opt, action, bz): + """ + Handle all login related bits + """ + is_login_command = (action == 'login') + + do_interactive_login = (is_login_command or + opt.login or opt.username or opt.password) + username = getattr(opt, "pos_username", None) or opt.username + password = getattr(opt, "pos_password", None) or opt.password + + try: + if do_interactive_login: + if bz.url: + print("Logging into %s" % urlparse(bz.url)[1]) + bz.interactive_login(username, password) + except bugzilla.BugzillaError as e: + print(str(e)) + sys.exit(1) + + if opt.ensure_logged_in and not bz.logged_in: + print("--ensure-logged-in passed but you aren't logged in to %s" % + bz.url) + sys.exit(1) + + if is_login_command: + msg = "Login successful." + if bz.cookiefile or bz.tokenfile: + msg = "Login successful, token cache updated." + + print(msg) + sys.exit(0) + + +def _main(unittest_bz_instance): + parser = setup_parser() + opt = parser.parse_args() + action = opt.command + setup_logging(opt.debug, opt.verbose) + + log.debug("Launched with command line: %s", " ".join(sys.argv)) + log.debug("Bugzilla module: %s", bugzilla) + + # Connect to bugzilla + log.info('Connecting to %s', opt.bugzilla) + + if unittest_bz_instance: + bz = unittest_bz_instance + else: + bz = _make_bz_instance(opt) + + # Handle login options + _handle_login(opt, action, bz) + + + ########################### + # Run the actual commands # + ########################### + + if hasattr(opt, "outputformat"): + if not opt.outputformat and opt.output not in ['raw', None]: + opt.outputformat = _convert_to_outputformat(opt.output) + + buglist = [] + if action == 'info': + if not (opt.products or + opt.components or + opt.component_owners or + opt.versions): + parser.error("'info' command requires additional arguments") + + _do_info(bz, opt) + + elif action == 'query': + buglist = _do_query(bz, opt, parser) + if opt.test_return_result: + return buglist + + elif action == 'new': + buglist = _do_new(bz, opt, parser) + if opt.test_return_result: + return buglist + + elif action == 'attach': + if opt.get or opt.getall: + if opt.ids: + parser.error("Bug IDs '%s' not used for " + "getting attachments" % opt.ids) + _do_get_attach(bz, opt) + else: + _do_set_attach(bz, opt, parser) + + elif action == 'modify': + modout = _do_modify(bz, parser, opt) + if opt.test_return_result: + return modout + else: + raise RuntimeError("Unexpected action '%s'" % action) + + # If we're doing new/query/modify, output our results + if action in ['new', 'query']: + _format_output(bz, opt, buglist) + + +def main(unittest_bz_instance=None): + try: + try: + return _main(unittest_bz_instance) + except (Exception, KeyboardInterrupt): + log.debug("", exc_info=True) + raise + except (Fault, bugzilla.BugzillaError) as e: + print("\nServer error: %s" % str(e)) + sys.exit(3) + except requests.exceptions.SSLError as e: + # Give SSL recommendations + print("SSL error: %s" % e) + print("\nIf you trust the remote server, you can work " + "around this error with:\n" + " bugzilla --nosslverify ...") + sys.exit(4) + except (socket.error, + requests.exceptions.HTTPError, + requests.exceptions.ConnectionError, + ProtocolError) as e: + print("\nConnection lost/failed: %s" % str(e)) + sys.exit(2) + + +def cli(): + try: + main() + except KeyboardInterrupt: + log.debug("", exc_info=True) + print("\nExited at user request.") + sys.exit(1) diff --git a/bugzilla/apiversion.py b/bugzilla/apiversion.py new file mode 100644 index 0000000..9762d4a --- /dev/null +++ b/bugzilla/apiversion.py @@ -0,0 +1,11 @@ +# +# Copyright (C) 2014 Red Hat Inc. +# +# This program is free software; you can redistribute it and/or modify it +# under the terms of the GNU General Public License as published by the +# Free Software Foundation; either version 2 of the License, or (at your +# option) any later version. See http://www.gnu.org/copyleft/gpl.html for +# the full text of the license. + +version = "2.2.0" +__version__ = version diff --git a/bugzilla/base.py b/bugzilla/base.py new file mode 100644 index 0000000..20ebe79 --- /dev/null +++ b/bugzilla/base.py @@ -0,0 +1,1837 @@ +# base.py - the base classes etc. for a Python interface to bugzilla +# +# Copyright (C) 2007, 2008, 2009, 2010 Red Hat Inc. +# Author: Will Woods +# +# This program is free software; you can redistribute it and/or modify it +# under the terms of the GNU General Public License as published by the +# Free Software Foundation; either version 2 of the License, or (at your +# option) any later version. See http://www.gnu.org/copyleft/gpl.html for +# the full text of the license. + +import collections +import getpass +import locale +from logging import getLogger +import mimetypes +import os +import sys + +from io import BytesIO + +# pylint: disable=import-error +if sys.version_info[0] >= 3: + # pylint: disable=no-name-in-module + from configparser import ConfigParser + from http.cookiejar import LoadError, MozillaCookieJar + from urllib.parse import urlparse, parse_qsl + from xmlrpc.client import Binary, Fault +else: + from ConfigParser import SafeConfigParser as ConfigParser + from cookielib import LoadError, MozillaCookieJar + from urlparse import urlparse, parse_qsl + from xmlrpclib import Binary, Fault +# pylint: enable=import-error + + +from .apiversion import __version__ +from .bug import Bug, User +from .transport import BugzillaError, _BugzillaServerProxy, _RequestsTransport + + +log = getLogger(__name__) + + +def _nested_update(d, u): + # Helper for nested dict update() + # https://stackoverflow.com/questions/3232943/update-value-of-a-nested-dictionary-of-varying-depth + for k, v in list(u.items()): + if isinstance(v, collections.Mapping): + d[k] = _nested_update(d.get(k, {}), v) + else: + d[k] = v + return d + + +def _default_auth_location(filename): + """ + Determine auth location for filename, like 'bugzillacookies'. If + old style ~/.bugzillacookies exists, we use that, otherwise we + use ~/.cache/python-bugzilla/bugzillacookies. Same for bugzillatoken + """ + homepath = os.path.expanduser("~/.%s" % filename) + xdgpath = os.path.expanduser("~/.cache/python-bugzilla/%s" % filename) + if os.path.exists(xdgpath): + return xdgpath + if os.path.exists(homepath): + return homepath + + if not os.path.exists(os.path.dirname(xdgpath)): + os.makedirs(os.path.dirname(xdgpath), 0o700) + return xdgpath + + +def _build_cookiejar(cookiefile): + cj = MozillaCookieJar(cookiefile) + if cookiefile is None: + return cj + if not os.path.exists(cookiefile): + # Make sure a new file has correct permissions + open(cookiefile, 'a').close() + os.chmod(cookiefile, 0o600) + cj.save() + return cj + + try: + cj.load() + return cj + except LoadError: + raise BugzillaError("cookiefile=%s not in Mozilla format" % + cookiefile) + + +_default_configpaths = [ + '/etc/bugzillarc', + '~/.bugzillarc', + '~/.config/python-bugzilla/bugzillarc', +] + + +def _open_bugzillarc(configpaths=-1): + if configpaths == -1: + configpaths = _default_configpaths[:] + + # pylint: disable=protected-access + configpaths = [os.path.expanduser(p) for p in + Bugzilla._listify(configpaths)] + # pylint: enable=protected-access + cfg = ConfigParser() + read_files = cfg.read(configpaths) + if not read_files: + return + + log.info("Found bugzillarc files: %s", read_files) + return cfg + + +class _FieldAlias(object): + """ + Track API attribute names that differ from what we expose in users. + + For example, originally 'short_desc' was the name of the property that + maps to 'summary' on modern bugzilla. We want pre-existing API users + to be able to continue to use Bug.short_desc, and + query({"short_desc": "foo"}). This class tracks that mapping. + + @oldname: The old attribute name + @newname: The modern attribute name + @is_api: If True, use this mapping for values sent to the xmlrpc API + (like the query example) + @is_bug: If True, use this mapping for Bug attribute names. + """ + def __init__(self, newname, oldname, is_api=True, is_bug=True): + self.newname = newname + self.oldname = oldname + self.is_api = is_api + self.is_bug = is_bug + + +class _BugzillaAPICache(object): + """ + Helper class that holds cached API results for things like products, + components, etc. + """ + def __init__(self): + self.products = [] + self.component_names = {} + self.bugfields = [] + + +class Bugzilla(object): + """ + The main API object. Connects to a bugzilla instance over XMLRPC, and + provides wrapper functions to simplify dealing with API calls. + + The most common invocation here will just be with just a URL: + + bzapi = Bugzilla("http://bugzilla.example.com") + + If you have previously logged into that URL, and have cached login + cookies/tokens, you will automatically be logged in. Otherwise to + log in, you can either pass auth options to __init__, or call a login + helper like interactive_login(). + + If you are not logged in, you won be able to access restricted data like + user email, or perform write actions like bug create/update. But simple + querys will work correctly. + + If you are unsure if you are logged in, you can check the .logged_in + property. + + Another way to specify auth credentials is via a 'bugzillarc' file. + See readconfig() documentation for details. + """ + + # bugzilla version that the class is targeting. filled in by + # subclasses + bz_ver_major = 0 + bz_ver_minor = 0 + + @staticmethod + def url_to_query(url): + """ + Given a big huge bugzilla query URL, returns a query dict that can + be passed along to the Bugzilla.query() method. + """ + q = {} + + # pylint: disable=unpacking-non-sequence + (ignore, ignore, path, + ignore, query, ignore) = urlparse(url) + + base = os.path.basename(path) + if base not in ('buglist.cgi', 'query.cgi'): + return {} + + for (k, v) in parse_qsl(query): + if k not in q: + q[k] = v + elif isinstance(q[k], list): + q[k].append(v) + else: + oldv = q[k] + q[k] = [oldv, v] + + # Handle saved searches + if base == "buglist.cgi" and "namedcmd" in q and "sharer_id" in q: + q = { + "sharer_id": q["sharer_id"], + "savedsearch": q["namedcmd"], + } + + return q + + @staticmethod + def fix_url(url): + """ + Turn passed url into a bugzilla XMLRPC web url + """ + if '://' not in url: + log.debug('No scheme given for url, assuming https') + url = 'https://' + url + if url.count('/') < 3: + log.debug('No path given for url, assuming /xmlrpc.cgi') + url = url + '/xmlrpc.cgi' + return url + + @staticmethod + def _listify(val): + if val is None: + return val + if isinstance(val, list): + return val + return [val] + + + def __init__(self, url=-1, user=None, password=None, cookiefile=-1, + sslverify=True, tokenfile=-1, use_creds=True, api_key=None, + cert=None): + """ + :param url: The bugzilla instance URL, which we will connect + to immediately. Most users will want to specify this at + __init__ time, but you can defer connecting by passing + url=None and calling connect(URL) manually + :param user: optional username to connect with + :param password: optional password for the connecting user + :param cert: optional certificate file for client side certificate + authentication + :param cookiefile: Location to cache the login session cookies so you + don't have to keep specifying username/password. Bugzilla 5+ will + use tokens instead of cookies. + If -1, use the default path. If None, don't use or save + any cookiefile. + :param sslverify: Set this to False to skip SSL hostname and CA + validation checks, like out of date certificate + :param tokenfile: Location to cache the API login token so youi + don't have to keep specifying username/password. + If -1, use the default path. If None, don't use + or save any tokenfile. + :param use_creds: If False, this disables cookiefile, tokenfile, + and any bugzillarc reading. This overwrites any tokenfile + or cookiefile settings + :param sslverify: Maps to 'requests' sslverify parameter. Set to + False to disable SSL verification, but it can also be a path + to file or directory for custom certs. + :param api_key: A bugzilla + """ + if url == -1: + raise TypeError("Specify a valid bugzilla url, or pass url=None") + + # Settings the user might want to tweak + self.user = user or '' + self.password = password or '' + self.api_key = api_key + self.cert = cert or '' + self.url = '' + + self._proxy = None + self._transport = None + self._cookiejar = None + self._sslverify = sslverify + self._cache = _BugzillaAPICache() + self._bug_autorefresh = False + + self._field_aliases = [] + self._init_field_aliases() + + self.configpath = _default_configpaths[:] + if not use_creds: + cookiefile = None + tokenfile = None + self.configpath = [] + + if cookiefile == -1: + cookiefile = _default_auth_location("bugzillacookies") + if tokenfile == -1: + tokenfile = _default_auth_location("bugzillatoken") + log.debug("Using tokenfile=%s", tokenfile) + self.cookiefile = cookiefile + self.tokenfile = tokenfile + + if url: + self.connect(url) + self._init_class_from_url() + self._init_class_state() + + def _init_class_from_url(self): + """ + Detect if we should use RHBugzilla class, and if so, set it + """ + from bugzilla import RHBugzilla + if isinstance(self, RHBugzilla): + return + + c = None + if "bugzilla.redhat.com" in self.url: + log.info("Using RHBugzilla for URL containing bugzilla.redhat.com") + c = RHBugzilla + else: + try: + extensions = self._proxy.Bugzilla.extensions() + if "RedHat" in extensions.get('extensions', {}): + log.info("Found RedHat bugzilla extension, " + "using RHBugzilla") + c = RHBugzilla + except Fault: + log.debug("Failed to fetch bugzilla extensions", exc_info=True) + + if not c: + return + + self.__class__ = c + + def _init_class_state(self): + """ + Hook for subclasses to do any __init__ time setup + """ + pass + + def _init_field_aliases(self): + # List of field aliases. Maps old style RHBZ parameter + # names to actual upstream values. Used for createbug() and + # query include_fields at least. + self._add_field_alias('summary', 'short_desc') + self._add_field_alias('description', 'comment') + self._add_field_alias('platform', 'rep_platform') + self._add_field_alias('severity', 'bug_severity') + self._add_field_alias('status', 'bug_status') + self._add_field_alias('id', 'bug_id') + self._add_field_alias('blocks', 'blockedby') + self._add_field_alias('blocks', 'blocked') + self._add_field_alias('depends_on', 'dependson') + self._add_field_alias('creator', 'reporter') + self._add_field_alias('url', 'bug_file_loc') + self._add_field_alias('dupe_of', 'dupe_id') + self._add_field_alias('dupe_of', 'dup_id') + self._add_field_alias('comments', 'longdescs') + self._add_field_alias('creation_time', 'opendate') + self._add_field_alias('creation_time', 'creation_ts') + self._add_field_alias('whiteboard', 'status_whiteboard') + self._add_field_alias('last_change_time', 'delta_ts') + + def _get_user_agent(self): + return 'python-bugzilla/%s' % __version__ + user_agent = property(_get_user_agent) + + + ################### + # Private helpers # + ################### + + def _check_version(self, major, minor): + """ + Check if the detected bugzilla version is >= passed major/minor pair. + """ + if major < self.bz_ver_major: + return True + if (major == self.bz_ver_major and minor <= self.bz_ver_minor): + return True + return False + + def _add_field_alias(self, *args, **kwargs): + self._field_aliases.append(_FieldAlias(*args, **kwargs)) + + def _get_bug_aliases(self): + return [(f.newname, f.oldname) + for f in self._field_aliases if f.is_bug] + + def _get_api_aliases(self): + return [(f.newname, f.oldname) + for f in self._field_aliases if f.is_api] + + + ################### + # Cookie handling # + ################### + + def _getcookiefile(self): + """ + cookiefile is the file that bugzilla session cookies are loaded + and saved from. + """ + return self._cookiejar.filename + + def _delcookiefile(self): + self._cookiejar = None + + def _setcookiefile(self, cookiefile): + if (self._cookiejar and cookiefile == self._cookiejar.filename): + return + + if self._proxy is not None: + raise RuntimeError("Can't set cookies with an open connection, " + "disconnect() first.") + + log.debug("Using cookiefile=%s", cookiefile) + self._cookiejar = _build_cookiejar(cookiefile) + + cookiefile = property(_getcookiefile, _setcookiefile, _delcookiefile) + + + ############################# + # Login/connection handling # + ############################# + + def readconfig(self, configpath=None): + """ + :param configpath: Optional bugzillarc path to read, instead of + the default list. + + This function is called automatically from Bugzilla connect(), which + is called at __init__ if a URL is passed. Calling it manually is + just for passing in a non-standard configpath. + + The locations for the bugzillarc file are preferred in this order: + + ~/.config/python-bugzilla/bugzillarc + ~/.bugzillarc + /etc/bugzillarc + + It has content like: + [bugzilla.yoursite.com] + user = username + password = password + Or + [bugzilla.yoursite.com] + api_key = key + + The file can have multiple sections for different bugzilla instances. + A 'url' field in the [DEFAULT] section can be used to set a default + URL for the bugzilla command line tool. + + Be sure to set appropriate permissions on bugzillarc if you choose to + store your password in it! + """ + cfg = _open_bugzillarc(configpath or self.configpath) + if not cfg: + return + + section = "" + log.debug("bugzillarc: Searching for config section matching %s", + self.url) + for s in sorted(cfg.sections()): + # Substring match - prefer the longest match found + if s in self.url: + log.debug("bugzillarc: Found matching section: %s", s) + section = s + + if not section: + log.debug("bugzillarc: No section found") + return + + for key, val in cfg.items(section): + if key == "api_key": + log.debug("bugzillarc: setting api_key") + self.api_key = val + elif key == "user": + log.debug("bugzillarc: setting user=%s", val) + self.user = val + elif key == "password": + log.debug("bugzillarc: setting password") + self.password = val + elif key == "cert": + log.debug("bugzillarc: setting cert") + self.cert = val + else: + log.debug("bugzillarc: unknown key=%s", key) + + def _set_bz_version(self, version): + try: + self.bz_ver_major, self.bz_ver_minor = [ + int(i) for i in version.split(".")[0:2]] + except Exception: + log.debug("version doesn't match expected format X.Y.Z, " + "assuming 5.0", exc_info=True) + self.bz_ver_major = 5 + self.bz_ver_minor = 0 + + def connect(self, url=None): + """ + Connect to the bugzilla instance with the given url. This is + called by __init__ if a URL is passed. Or it can be called manually + at any time with a passed URL. + + This will also read any available config files (see readconfig()), + which may set 'user' and 'password', and others. + + If 'user' and 'password' are both set, we'll run login(). Otherwise + you'll have to login() yourself before some methods will work. + """ + if self._transport: + self.disconnect() + + if url is None and self.url: + url = self.url + url = self.fix_url(url) + + self._transport = _RequestsTransport( + url, self._cookiejar, sslverify=self._sslverify, cert=self.cert) + self._transport.user_agent = self.user_agent + self._proxy = _BugzillaServerProxy(url, self.tokenfile, + self._transport) + + self.url = url + # we've changed URLs - reload config + self.readconfig() + + if (self.user and self.password): + log.info("user and password present - doing login()") + self.login() + + if self.api_key: + log.debug("using API key") + self._proxy.use_api_key(self.api_key) + + version = self._proxy.Bugzilla.version()["version"] + log.debug("Bugzilla version string: %s", version) + self._set_bz_version(version) + + def disconnect(self): + """ + Disconnect from the given bugzilla instance. + """ + self._proxy = None + self._transport = None + self._cache = _BugzillaAPICache() + + + def _login(self, user, password): + """ + Backend login method for Bugzilla3 + """ + return self._proxy.User.login({'login': user, 'password': password}) + + def _logout(self): + """ + Backend login method for Bugzilla3 + """ + return self._proxy.User.logout() + + def login(self, user=None, password=None): + """ + Attempt to log in using the given username and password. Subsequent + method calls will use this username and password. Returns False if + login fails, otherwise returns some kind of login info - typically + either a numeric userid, or a dict of user info. + + If user is not set, the value of Bugzilla.user will be used. If *that* + is not set, ValueError will be raised. If login fails, BugzillaError + will be raised. + + This method will be called implicitly at the end of connect() if user + and password are both set. So under most circumstances you won't need + to call this yourself. + """ + if self.api_key: + raise ValueError("cannot login when using an API key") + + if user: + self.user = user + if password: + self.password = password + + if not self.user: + raise ValueError("missing username") + if not self.password: + raise ValueError("missing password") + + try: + ret = self._login(self.user, self.password) + self.password = '' + log.info("login successful for user=%s", self.user) + return ret + except Fault as e: + raise BugzillaError("Login failed: %s" % str(e.faultString)) + + def interactive_login(self, user=None, password=None, force=False): + """ + Helper method to handle login for this bugzilla instance. + + :param user: bugzilla username. If not specified, prompt for it. + :param password: bugzilla password. If not specified, prompt for it. + :param force: Unused + """ + ignore = force + log.debug('Calling interactive_login') + + if not user: + sys.stdout.write('Bugzilla Username: ') + sys.stdout.flush() + user = sys.stdin.readline().strip() + if not password: + password = getpass.getpass('Bugzilla Password: ') + + log.info('Logging in... ') + self.login(user, password) + log.info('Authorization cookie received.') + + def logout(self): + """ + Log out of bugzilla. Drops server connection and user info, and + destroys authentication cookies. + """ + self._logout() + self.disconnect() + self.user = '' + self.password = '' + + @property + def logged_in(self): + """ + This is True if this instance is logged in else False. + + We test if this session is authenticated by calling the User.get() + XMLRPC method with ids set. Logged-out users cannot pass the 'ids' + parameter and will result in a 505 error. If we tried to login with a + token, but the token was incorrect or expired, the server returns a + 32000 error. + + For Bugzilla 5 and later, a new method, User.valid_login is available + to test the validity of the token. However, this will require that the + username be cached along with the token in order to work effectively in + all scenarios and is not currently used. For more information, refer to + the following url. + + http://bugzilla.readthedocs.org/en/latest/api/core/v1/user.html#valid-login + """ + try: + self._proxy.User.get({'ids': []}) + return True + except Fault as e: + if e.faultCode == 505 or e.faultCode == 32000: + return False + raise e + + + ###################### + # Bugfields querying # + ###################### + + def _getbugfields(self): + """ + Get the list of valid fields for Bug objects + """ + r = self._proxy.Bug.fields({'include_fields': ['name']}) + return [f['name'] for f in r['fields']] + + def getbugfields(self, force_refresh=False): + """ + Calls getBugFields, which returns a list of fields in each bug + for this bugzilla instance. This can be used to set the list of attrs + on the Bug object. + """ + if force_refresh or not self._cache.bugfields: + log.debug("Refreshing bugfields") + self._cache.bugfields = self._getbugfields() + self._cache.bugfields.sort() + log.debug("bugfields = %s", self._cache.bugfields) + + return self._cache.bugfields + bugfields = property(fget=lambda self: self.getbugfields(), + fdel=lambda self: setattr(self, '_bugfields', None)) + + + #################### + # Product querying # + #################### + + def product_get(self, ids=None, names=None, + include_fields=None, exclude_fields=None, + ptype=None): + """ + Raw wrapper around Product.get + https://bugzilla.readthedocs.io/en/latest/api/core/v1/product.html#get-product + + This does not perform any caching like other product API calls. + If ids, names, or ptype is not specified, we default to + ptype=accessible for historical reasons + + @ids: List of product IDs to lookup + @names: List of product names to lookup + @ptype: Either 'accessible', 'selectable', or 'enterable'. If + specified, we return data for all those + @include_fields: Only include these fields in the output + @exclude_fields: Do not include these fields in the output + """ + if ids is None and names is None and ptype is None: + ptype = "accessible" + + if ptype: + raw = None + if ptype == "accessible": + raw = self._proxy.Product.get_accessible_products() + elif ptype == "selectable": + raw = self._proxy.Product.get_selectable_products() + elif ptype == "enterable": + raw = self._proxy.Product.get_enterable_products() + + if raw is None: + raise RuntimeError("Unknown ptype=%s" % ptype) + ids = raw['ids'] + log.debug("For ptype=%s found ids=%s", ptype, ids) + + kwargs = {} + if ids: + kwargs["ids"] = self._listify(ids) + if names: + kwargs["names"] = self._listify(names) + if include_fields: + kwargs["include_fields"] = include_fields + if exclude_fields: + kwargs["exclude_fields"] = exclude_fields + + ret = self._proxy.Product.get(kwargs) + return ret['products'] + + def refresh_products(self, **kwargs): + """ + Refresh a product's cached info. Basically calls product_get + with the passed arguments, and tries to intelligently update + our product cache. + + For example, if we already have cached info for product=foo, + and you pass in names=["bar", "baz"], the new cache will have + info for products foo, bar, baz. Individual product fields are + also updated. + """ + for product in self.product_get(**kwargs): + updated = False + for current in self._cache.products[:]: + if (current.get("id", -1) != product.get("id", -2) and + current.get("name", -1) != product.get("name", -2)): + continue + + _nested_update(current, product) + updated = True + break + if not updated: + self._cache.products.append(product) + + def getproducts(self, force_refresh=False, **kwargs): + """ + Query all products and return the raw dict info. Takes all the + same arguments as product_get. + + On first invocation this will contact bugzilla and internally + cache the results. Subsequent getproducts calls or accesses to + self.products will return this cached data only. + + :param force_refresh: force refreshing via refresh_products() + """ + if force_refresh or not self._cache.products: + self.refresh_products(**kwargs) + return self._cache.products + + products = property( + fget=lambda self: self.getproducts(), + fdel=lambda self: setattr(self, '_products', None), + doc="Helper for accessing the products cache. If nothing " + "has been cached yet, this calls getproducts()") + + + ####################### + # components querying # + ####################### + + def _lookup_product_in_cache(self, productname): + prodstr = isinstance(productname, str) and productname or None + prodint = isinstance(productname, int) and productname or None + for proddict in self._cache.products: + if prodstr == proddict.get("name", -1): + return proddict + if prodint == proddict.get("id", "nope"): + return proddict + return {} + + def getcomponentsdetails(self, product, force_refresh=False): + """ + Wrapper around Product.get(include_fields=["components"]), + returning only the "components" data for the requested product, + slightly reworked to a dict mapping of components.name: components, + for historical reasons. + + This uses the product cache, but will update it if the product + isn't found or "components" isn't cached for the product. + + In cases like bugzilla.redhat.com where there are tons of + components for some products, this API will time out. You + should use product_get instead. + """ + proddict = self._lookup_product_in_cache(product) + + if (force_refresh or not proddict or "components" not in proddict): + self.refresh_products(names=[product], + include_fields=["name", "id", "components"]) + proddict = self._lookup_product_in_cache(product) + + ret = {} + for compdict in proddict["components"]: + ret[compdict["name"]] = compdict + return ret + + def getcomponentdetails(self, product, component, force_refresh=False): + """ + Helper for accessing a single component's info. This is a wrapper + around getcomponentsdetails, see that for explanation + """ + d = self.getcomponentsdetails(product, force_refresh) + return d[component] + + def getcomponents(self, product, force_refresh=False): + """ + Return a list of component names for the passed product. + + This can be implemented with Product.get, but behind the + scenes it uses Bug.legal_values. Reason being that on bugzilla + instances with tons of components, like bugzilla.redhat.com + Product=Fedora for example, there's a 10x speed difference + even with properly limited Product.get calls. + + On first invocation the value is cached, and subsequent calls + will return the cached data. + + :param force_refresh: Force refreshing the cache, and return + the new data + """ + proddict = self._lookup_product_in_cache(product) + product_id = proddict.get("id", None) + + if (force_refresh or + product_id is None or + product_id not in self._cache.component_names): + self.refresh_products(names=[product], + include_fields=["names", "id"]) + proddict = self._lookup_product_in_cache(product) + product_id = proddict["id"] + + opts = {'product_id': product_id, 'field': 'component'} + names = self._proxy.Bug.legal_values(opts)["values"] + self._cache.component_names[product_id] = names + + return self._cache.component_names[product_id] + + + ############################ + # component adding/editing # + ############################ + + def _component_data_convert(self, data, update=False): + # Back compat for the old RH interface + convert_fields = [ + ("initialowner", "default_assignee"), + ("initialqacontact", "default_qa_contact"), + ("initialcclist", "default_cc"), + ] + for old, new in convert_fields: + if old in data: + data[new] = data.pop(old) + + if update: + names = {"product": data.pop("product"), + "component": data.pop("component")} + updates = {} + for k in list(data.keys()): + updates[k] = data.pop(k) + + data["names"] = [names] + data["updates"] = updates + + + def addcomponent(self, data): + """ + A method to create a component in Bugzilla. Takes a dict, with the + following elements: + + product: The product to create the component in + component: The name of the component to create + desription: A one sentence summary of the component + default_assignee: The bugzilla login (email address) of the initial + owner of the component + default_qa_contact (optional): The bugzilla login of the + initial QA contact + default_cc: (optional) The initial list of users to be CC'ed on + new bugs for the component. + is_active: (optional) If False, the component is hidden from + the component list when filing new bugs. + """ + data = data.copy() + self._component_data_convert(data) + return self._proxy.Component.create(data) + + def editcomponent(self, data): + """ + A method to edit a component in Bugzilla. Takes a dict, with + mandatory elements of product. component, and initialowner. + All other elements are optional and use the same names as the + addcomponent() method. + """ + data = data.copy() + self._component_data_convert(data, update=True) + return self._proxy.Component.update(data) + + + ################### + # getbug* methods # + ################### + + def _process_include_fields(self, include_fields, exclude_fields, + extra_fields): + """ + Internal helper to process include_fields lists + """ + def _convert_fields(_in): + if not _in: + return _in + + for newname, oldname in self._get_api_aliases(): + if oldname in _in: + _in.remove(oldname) + if newname not in _in: + _in.append(newname) + return _in + + ret = {} + if self._check_version(4, 0): + if include_fields: + include_fields = _convert_fields(include_fields) + if "id" not in include_fields: + include_fields.append("id") + ret["include_fields"] = include_fields + if exclude_fields: + exclude_fields = _convert_fields(exclude_fields) + ret["exclude_fields"] = exclude_fields + if self._supports_getbug_extra_fields: + if extra_fields: + ret["extra_fields"] = _convert_fields(extra_fields) + return ret + + def _get_bug_autorefresh(self): + """ + This value is passed to Bug.autorefresh for all fetched bugs. + If True, and an uncached attribute is requested from a Bug, + the Bug will update its contents and try again. + """ + return self._bug_autorefresh + + def _set_bug_autorefresh(self, val): + self._bug_autorefresh = bool(val) + bug_autorefresh = property(_get_bug_autorefresh, _set_bug_autorefresh) + + + # getbug_extra_fields: Extra fields that need to be explicitly + # requested from Bug.get in order for the data to be returned. + # + # As of Dec 2012 it seems like only RH bugzilla actually has behavior + # like this, for upstream bz it returns all info for every Bug.get() + _getbug_extra_fields = [] + _supports_getbug_extra_fields = False + + def _getbugs(self, idlist, permissive, + include_fields=None, exclude_fields=None, extra_fields=None): + """ + Return a list of dicts of full bug info for each given bug id. + bug ids that couldn't be found will return None instead of a dict. + """ + oldidlist = idlist + idlist = [] + for i in oldidlist: + try: + idlist.append(int(i)) + except ValueError: + # String aliases can be passed as well + idlist.append(i) + + extra_fields = self._listify(extra_fields or []) + extra_fields += self._getbug_extra_fields + + getbugdata = {"ids": idlist} + if permissive: + getbugdata["permissive"] = 1 + + getbugdata.update(self._process_include_fields( + include_fields, exclude_fields, extra_fields)) + + r = self._proxy.Bug.get(getbugdata) + + if self._check_version(4, 0): + bugdict = dict([(b['id'], b) for b in r['bugs']]) + else: + bugdict = dict([(b['id'], b['internals']) for b in r['bugs']]) + + ret = [] + for i in idlist: + found = None + if i in bugdict: + found = bugdict[i] + else: + # Need to map an alias + for valdict in bugdict.values(): + if i in self._listify(valdict.get("alias", None)): + found = valdict + break + + ret.append(found) + + return ret + + def _getbug(self, objid, **kwargs): + """ + Thin wrapper around _getbugs to handle the slight argument tweaks + for fetching a single bug. The main bit is permissive=False, which + will tell bugzilla to raise an explicit error if we can't fetch + that bug. + + This logic is called from Bug() too + """ + return self._getbugs([objid], permissive=False, **kwargs)[0] + + def getbug(self, objid, + include_fields=None, exclude_fields=None, extra_fields=None): + """ + Return a Bug object with the full complement of bug data + already loaded. + """ + data = self._getbug(objid, + include_fields=include_fields, exclude_fields=exclude_fields, + extra_fields=extra_fields) + return Bug(self, dict=data, autorefresh=self.bug_autorefresh) + + def getbugs(self, idlist, + include_fields=None, exclude_fields=None, extra_fields=None, + permissive=True): + """ + Return a list of Bug objects with the full complement of bug data + already loaded. If there's a problem getting the data for a given id, + the corresponding item in the returned list will be None. + """ + data = self._getbugs(idlist, include_fields=include_fields, + exclude_fields=exclude_fields, extra_fields=extra_fields, + permissive=permissive) + return [(b and Bug(self, dict=b, + autorefresh=self.bug_autorefresh)) or None + for b in data] + + def get_comments(self, idlist): + """ + Returns a dictionary of bugs and comments. The comments key will + be empty. See bugzilla docs for details + """ + return self._proxy.Bug.comments({'ids': idlist}) + + + ################# + # query methods # + ################# + + def build_query(self, + product=None, + component=None, + version=None, + long_desc=None, + bug_id=None, + short_desc=None, + cc=None, + assigned_to=None, + reporter=None, + qa_contact=None, + status=None, + blocked=None, + dependson=None, + keywords=None, + keywords_type=None, + url=None, + url_type=None, + status_whiteboard=None, + status_whiteboard_type=None, + fixed_in=None, + fixed_in_type=None, + flag=None, + alias=None, + qa_whiteboard=None, + devel_whiteboard=None, + boolean_query=None, + bug_severity=None, + priority=None, + target_release=None, + target_milestone=None, + emailtype=None, + booleantype=None, + include_fields=None, + quicksearch=None, + savedsearch=None, + savedsearch_sharer_id=None, + sub_component=None, + tags=None, + exclude_fields=None, + extra_fields=None): + """ + Build a query string from passed arguments. Will handle + query parameter differences between various bugzilla versions. + + Most of the parameters should be self explanatory. However + if you want to perform a complex query, and easy way is to + create it with the bugzilla web UI, copy the entire URL it + generates, and pass it to the static method + + Bugzilla.url_to_query + + Then pass the output to Bugzilla.query() + + For details about the specific argument formats, see the bugzilla docs: + https://bugzilla.readthedocs.io/en/latest/api/core/v1/bug.html#search-bugs + """ + if boolean_query or booleantype: + raise RuntimeError("boolean_query format is no longer supported. " + "If you need complicated URL queries, look into " + "query --from-url/url_to_query().") + + query = { + "alias": alias, + "product": self._listify(product), + "component": self._listify(component), + "version": version, + "id": bug_id, + "short_desc": short_desc, + "bug_status": status, + "bug_severity": bug_severity, + "priority": priority, + "target_release": target_release, + "target_milestone": target_milestone, + "tag": self._listify(tags), + "quicksearch": quicksearch, + "savedsearch": savedsearch, + "sharer_id": savedsearch_sharer_id, + + # RH extensions... don't add any more. See comment below + "sub_components": self._listify(sub_component), + } + + def add_bool(bzkey, value, bool_id, booltype=None): + value = self._listify(value) + if value is None: + return bool_id + + query["query_format"] = "advanced" + for boolval in value: + def make_bool_str(prefix): + # pylint: disable=cell-var-from-loop + return "%s%i-0-0" % (prefix, bool_id) + + query[make_bool_str("field")] = bzkey + query[make_bool_str("value")] = boolval + query[make_bool_str("type")] = booltype or "substring" + + bool_id += 1 + return bool_id + + # RH extensions that we have to maintain here for back compat, + # but all future custom fields should be specified via + # cli --field option, or via extending the query dict() manually. + # No more supporting custom fields in this API + bool_id = 0 + bool_id = add_bool("keywords", keywords, bool_id, keywords_type) + bool_id = add_bool("blocked", blocked, bool_id) + bool_id = add_bool("dependson", dependson, bool_id) + bool_id = add_bool("bug_file_loc", url, bool_id, url_type) + bool_id = add_bool("cf_fixed_in", fixed_in, bool_id, fixed_in_type) + bool_id = add_bool("flagtypes.name", flag, bool_id) + bool_id = add_bool("status_whiteboard", + status_whiteboard, bool_id, status_whiteboard_type) + bool_id = add_bool("cf_qa_whiteboard", qa_whiteboard, bool_id) + bool_id = add_bool("cf_devel_whiteboard", devel_whiteboard, bool_id) + + def add_email(key, value, count): + if value is None: + return count + if not emailtype: + query[key] = value + return count + + query["query_format"] = "advanced" + query['email%i' % count] = value + query['email%s%i' % (key, count)] = True + query['emailtype%i' % count] = emailtype + return count + 1 + + email_count = 1 + email_count = add_email("cc", cc, email_count) + email_count = add_email("assigned_to", assigned_to, email_count) + email_count = add_email("reporter", reporter, email_count) + email_count = add_email("qa_contact", qa_contact, email_count) + + if long_desc is not None: + query["query_format"] = "advanced" + query["longdesc"] = long_desc + query["longdesc_type"] = "allwordssubstr" + + # 'include_fields' only available for Bugzilla4+ + # 'extra_fields' is an RHBZ extension + query.update(self._process_include_fields( + include_fields, exclude_fields, extra_fields)) + + # Strip out None elements in the dict + for k, v in query.copy().items(): + if v is None: + del(query[k]) + + self.pre_translation(query) + return query + + def query(self, query): + """ + Query bugzilla and return a list of matching bugs. + query must be a dict with fields like those in in querydata['fields']. + Returns a list of Bug objects. + Also see the _query() method for details about the underlying + implementation. + """ + try: + r = self._proxy.Bug.search(query) + except Fault as e: + + # Try to give a hint in the error message if url_to_query + # isn't supported by this bugzilla instance + if ("query_format" not in str(e) or + "RHBugzilla" in str(e.__class__) or + self._check_version(5, 0)): + raise + raise BugzillaError("%s\nYour bugzilla instance does not " + "appear to support API queries derived from bugzilla " + "web URL queries." % e) + + log.debug("Query returned %s bugs", len(r['bugs'])) + return [Bug(self, dict=b, + autorefresh=self.bug_autorefresh) for b in r['bugs']] + + def pre_translation(self, query): + """ + In order to keep the API the same, Bugzilla4 needs to process the + query and the result. This also applies to the refresh() function + """ + pass + + def post_translation(self, query, bug): + """ + In order to keep the API the same, Bugzilla4 needs to process the + query and the result. This also applies to the refresh() function + """ + pass + + def bugs_history_raw(self, bug_ids): + """ + Experimental. Gets the history of changes for + particular bugs in the database. + """ + return self._proxy.Bug.history({'ids': bug_ids}) + + + ####################################### + # Methods for modifying existing bugs # + ####################################### + + # Bug() also has individual methods for many ops, like setassignee() + + def update_bugs(self, ids, updates): + """ + A thin wrapper around bugzilla Bug.update(). Used to update all + values of an existing bug report, as well as add comments. + + The dictionary passed to this function should be generated with + build_update(), otherwise we cannot guarantee back compatibility. + """ + tmp = updates.copy() + tmp["ids"] = self._listify(ids) + + return self._proxy.Bug.update(tmp) + + def update_tags(self, idlist, tags_add=None, tags_remove=None): + """ + Updates the 'tags' field for a bug. + """ + tags = {} + if tags_add: + tags["add"] = self._listify(tags_add) + if tags_remove: + tags["remove"] = self._listify(tags_remove) + + d = { + "ids": self._listify(idlist), + "tags": tags, + } + + return self._proxy.Bug.update_tags(d) + + def update_flags(self, idlist, flags): + """ + A thin back compat wrapper around build_update(flags=X) + """ + return self.update_bugs(idlist, self.build_update(flags=flags)) + + + def build_update(self, + alias=None, + assigned_to=None, + blocks_add=None, + blocks_remove=None, + blocks_set=None, + depends_on_add=None, + depends_on_remove=None, + depends_on_set=None, + cc_add=None, + cc_remove=None, + is_cc_accessible=None, + comment=None, + comment_private=None, + component=None, + deadline=None, + dupe_of=None, + estimated_time=None, + groups_add=None, + groups_remove=None, + keywords_add=None, + keywords_remove=None, + keywords_set=None, + op_sys=None, + platform=None, + priority=None, + product=None, + qa_contact=None, + is_creator_accessible=None, + remaining_time=None, + reset_assigned_to=None, + reset_qa_contact=None, + resolution=None, + see_also_add=None, + see_also_remove=None, + severity=None, + status=None, + summary=None, + target_milestone=None, + target_release=None, + url=None, + version=None, + whiteboard=None, + work_time=None, + fixed_in=None, + qa_whiteboard=None, + devel_whiteboard=None, + internal_whiteboard=None, + sub_component=None, + flags=None, + comment_tags=None): + """ + Returns a python dict() with properly formatted parameters to + pass to update_bugs(). See bugzilla documentation for the format + of the individual fields: + + https://bugzilla.readthedocs.io/en/latest/api/core/v1/bug.html#create-bug + """ + ret = {} + + # These are only supported for rhbugzilla + for key, val in [ + ("fixed_in", fixed_in), + ("devel_whiteboard", devel_whiteboard), + ("qa_whiteboard", qa_whiteboard), + ("internal_whiteboard", internal_whiteboard), + ("sub_component", sub_component), + ]: + if val is not None: + raise ValueError("bugzilla instance does not support " + "updating '%s'" % key) + + def s(key, val, convert=None): + if val is None: + return + if convert: + val = convert(val) + ret[key] = val + + def add_dict(key, add, remove, _set=None, convert=None): + if add is remove is _set is None: + return + + def c(val): + val = self._listify(val) + if convert: + val = [convert(v) for v in val] + return val + + newdict = {} + if add is not None: + newdict["add"] = c(add) + if remove is not None: + newdict["remove"] = c(remove) + if _set is not None: + newdict["set"] = c(_set) + ret[key] = newdict + + + s("alias", alias) + s("assigned_to", assigned_to) + s("is_cc_accessible", is_cc_accessible, bool) + s("component", component) + s("deadline", deadline) + s("dupe_of", dupe_of, int) + s("estimated_time", estimated_time, int) + s("op_sys", op_sys) + s("platform", platform) + s("priority", priority) + s("product", product) + s("qa_contact", qa_contact) + s("is_creator_accessible", is_creator_accessible, bool) + s("remaining_time", remaining_time, float) + s("reset_assigned_to", reset_assigned_to, bool) + s("reset_qa_contact", reset_qa_contact, bool) + s("resolution", resolution) + s("severity", severity) + s("status", status) + s("summary", summary) + s("target_milestone", target_milestone) + s("target_release", target_release) + s("url", url) + s("version", version) + s("whiteboard", whiteboard) + s("work_time", work_time, float) + s("flags", flags) + s("comment_tags", comment_tags, self._listify) + + add_dict("blocks", blocks_add, blocks_remove, blocks_set, + convert=int) + add_dict("depends_on", depends_on_add, depends_on_remove, + depends_on_set, convert=int) + add_dict("cc", cc_add, cc_remove) + add_dict("groups", groups_add, groups_remove) + add_dict("keywords", keywords_add, keywords_remove, keywords_set) + add_dict("see_also", see_also_add, see_also_remove) + + if comment is not None: + ret["comment"] = {"comment": comment} + if comment_private: + ret["comment"]["is_private"] = comment_private + + return ret + + + ######################################## + # Methods for working with attachments # + ######################################## + + def _attachment_uri(self, attachid): + """ + Returns the URI for the given attachment ID. + """ + att_uri = self.url.replace('xmlrpc.cgi', 'attachment.cgi') + att_uri = att_uri + '?id=%s' % attachid + return att_uri + + def attachfile(self, idlist, attachfile, description, **kwargs): + """ + Attach a file to the given bug IDs. Returns the ID of the attachment + or raises XMLRPC Fault if something goes wrong. + + attachfile may be a filename (which will be opened) or a file-like + object, which must provide a 'read' method. If it's not one of these, + this method will raise a TypeError. + description is the short description of this attachment. + + Optional keyword args are as follows: + file_name: this will be used as the filename for the attachment. + REQUIRED if attachfile is a file-like object with no + 'name' attribute, otherwise the filename or .name + attribute will be used. + comment: An optional comment about this attachment. + is_private: Set to True if the attachment should be marked private. + is_patch: Set to True if the attachment is a patch. + content_type: The mime-type of the attached file. Defaults to + application/octet-stream if not set. NOTE that text + files will *not* be viewable in bugzilla unless you + remember to set this to text/plain. So remember that! + + Returns the list of attachment ids that were added. If only one + attachment was added, we return the single int ID for back compat + """ + if isinstance(attachfile, str): + f = open(attachfile, "rb") + elif hasattr(attachfile, 'read'): + f = attachfile + else: + raise TypeError("attachfile must be filename or file-like object") + + # Back compat + if "contenttype" in kwargs: + kwargs["content_type"] = kwargs.pop("contenttype") + if "ispatch" in kwargs: + kwargs["is_patch"] = kwargs.pop("ispatch") + if "isprivate" in kwargs: + kwargs["is_private"] = kwargs.pop("isprivate") + if "filename" in kwargs: + kwargs["file_name"] = kwargs.pop("filename") + + kwargs['summary'] = description + + data = f.read() + if not isinstance(data, bytes): + data = data.encode(locale.getpreferredencoding()) + kwargs['data'] = Binary(data) + + kwargs['ids'] = self._listify(idlist) + + if 'file_name' not in kwargs and hasattr(f, "name"): + kwargs['file_name'] = os.path.basename(f.name) + if 'content_type' not in kwargs: + ctype = None + if kwargs['file_name']: + ctype = mimetypes.guess_type( + kwargs['file_name'], strict=False)[0] + kwargs['content_type'] = ctype or 'application/octet-stream' + + ret = self._proxy.Bug.add_attachment(kwargs) + + if "attachments" in ret: + # Up to BZ 4.2 + ret = [int(k) for k in ret["attachments"].keys()] + elif "ids" in ret: + # BZ 4.4+ + ret = ret["ids"] + + if isinstance(ret, list) and len(ret) == 1: + ret = ret[0] + return ret + + + def openattachment(self, attachid): + """ + Get the contents of the attachment with the given attachment ID. + Returns a file-like object. + """ + attachments = self.get_attachments(None, attachid) + data = attachments["attachments"][str(attachid)] + xmlrpcbinary = data["data"] + + ret = BytesIO() + ret.write(xmlrpcbinary.data) + ret.name = data["file_name"] + ret.seek(0) + return ret + + def updateattachmentflags(self, bugid, attachid, flagname, **kwargs): + """ + Updates a flag for the given attachment ID. + Optional keyword args are: + status: new status for the flag ('-', '+', '?', 'X') + requestee: new requestee for the flag + """ + # Bug ID was used for the original custom redhat API, no longer + # needed though + ignore = bugid + + flags = {"name": flagname} + flags.update(kwargs) + update = {'ids': [int(attachid)], 'flags': [flags]} + + return self._proxy.Bug.update_attachment(update) + + def get_attachments(self, ids, attachment_ids, + include_fields=None, exclude_fields=None): + """ + Wrapper for Bug.attachments. One of ids or attachment_ids is required + + :param ids: Get attachments for this bug ID + :param attachment_ids: Specific attachment ID to get + + https://bugzilla.readthedocs.io/en/latest/api/core/v1/attachment.html#get-attachment + """ + params = { + "ids": self._listify(ids) or [], + "attachment_ids": self._listify(attachment_ids) or [], + } + if include_fields: + params["include_fields"] = self._listify(include_fields) + if exclude_fields: + params["exclude_fields"] = self._listify(exclude_fields) + + return self._proxy.Bug.attachments(params) + + + ##################### + # createbug methods # + ##################### + + createbug_required = ('product', 'component', 'summary', 'version', + 'description') + + def build_createbug(self, + product=None, + component=None, + version=None, + summary=None, + description=None, + comment_private=None, + blocks=None, + cc=None, + assigned_to=None, + keywords=None, + depends_on=None, + groups=None, + op_sys=None, + platform=None, + priority=None, + qa_contact=None, + resolution=None, + severity=None, + status=None, + target_milestone=None, + target_release=None, + url=None, + sub_component=None, + alias=None, + comment_tags=None): + """ + Returns a python dict() with properly formatted parameters to + pass to createbug(). See bugzilla documentation for the format + of the individual fields: + + https://bugzilla.readthedocs.io/en/latest/api/core/v1/bug.html#update-bug + """ + + localdict = {} + if blocks: + localdict["blocks"] = self._listify(blocks) + if cc: + localdict["cc"] = self._listify(cc) + if depends_on: + localdict["depends_on"] = self._listify(depends_on) + if groups: + localdict["groups"] = self._listify(groups) + if keywords: + localdict["keywords"] = self._listify(keywords) + if description: + localdict["description"] = description + if comment_private: + localdict["comment_is_private"] = True + + # Most of the machinery and formatting here is the same as + # build_update, so reuse that as much as possible + ret = self.build_update(product=product, component=component, + version=version, summary=summary, op_sys=op_sys, + platform=platform, priority=priority, qa_contact=qa_contact, + resolution=resolution, severity=severity, status=status, + target_milestone=target_milestone, + target_release=target_release, url=url, + assigned_to=assigned_to, sub_component=sub_component, + alias=alias, comment_tags=comment_tags) + + ret.update(localdict) + return ret + + def _validate_createbug(self, *args, **kwargs): + # Previous API required users specifying keyword args that mapped + # to the XMLRPC arg names. Maintain that bad compat, but also allow + # receiving a single dictionary like query() does + if kwargs and args: + raise BugzillaError("createbug: cannot specify positional " + "args=%s with kwargs=%s, must be one or the " + "other." % (args, kwargs)) + if args: + if len(args) > 1 or not isinstance(args[0], dict): + raise BugzillaError("createbug: positional arguments only " + "accept a single dictionary.") + data = args[0] + else: + data = kwargs + + # If we're getting a call that uses an old fieldname, convert it to the + # new fieldname instead. + for newname, oldname in self._get_api_aliases(): + if (newname in self.createbug_required and + newname not in data and + oldname in data): + data[newname] = data.pop(oldname) + + # Back compat handling for check_args + if "check_args" in data: + del(data["check_args"]) + + return data + + def createbug(self, *args, **kwargs): + """ + Create a bug with the given info. Returns a new Bug object. + Check bugzilla API documentation for valid values, at least + product, component, summary, version, and description need to + be passed. + """ + data = self._validate_createbug(*args, **kwargs) + rawbug = self._proxy.Bug.create(data) + return Bug(self, bug_id=rawbug["id"], + autorefresh=self.bug_autorefresh) + + + ############################## + # Methods for handling Users # + ############################## + + def _getusers(self, ids=None, names=None, match=None): + """ + Return a list of users that match criteria. + + :kwarg ids: list of user ids to return data on + :kwarg names: list of user names to return data on + :kwarg match: list of patterns. Returns users whose real name or + login name match the pattern. + :raises XMLRPC Fault: Code 51: if a Bad Login Name was sent to the + names array. + Code 304: if the user was not authorized to see user they + requested. + Code 505: user is logged out and can't use the match or ids + parameter. + + Available in Bugzilla-3.4+ + """ + params = {} + if ids: + params['ids'] = self._listify(ids) + if names: + params['names'] = self._listify(names) + if match: + params['match'] = self._listify(match) + if not params: + raise BugzillaError('_get() needs one of ids, ' + ' names, or match kwarg.') + + return self._proxy.User.get(params) + + def getuser(self, username): + """ + Return a bugzilla User for the given username + + :arg username: The username used in bugzilla. + :raises XMLRPC Fault: Code 51 if the username does not exist + :returns: User record for the username + """ + ret = self.getusers(username) + return ret and ret[0] + + def getusers(self, userlist): + """ + Return a list of Users from . + + :userlist: List of usernames to lookup + :returns: List of User records + """ + userobjs = [User(self, **rawuser) for rawuser in + self._getusers(names=userlist).get('users', [])] + + # Return users in same order they were passed in + ret = [] + for u in userlist: + for uobj in userobjs[:]: + if uobj.email == u: + userobjs.remove(uobj) + ret.append(uobj) + break + ret += userobjs + return ret + + + def searchusers(self, pattern): + """ + Return a bugzilla User for the given list of patterns + + :arg pattern: List of patterns to match against. + :returns: List of User records + """ + return [User(self, **rawuser) for rawuser in + self._getusers(match=pattern).get('users', [])] + + def createuser(self, email, name='', password=''): + """ + Return a bugzilla User for the given username + + :arg email: The email address to use in bugzilla + :kwarg name: Real name to associate with the account + :kwarg password: Password to set for the bugzilla account + :raises XMLRPC Fault: Code 501 if the username already exists + Code 500 if the email address isn't valid + Code 502 if the password is too short + Code 503 if the password is too long + :return: User record for the username + """ + self._proxy.User.create(email, name, password) + return self.getuser(email) + + def updateperms(self, user, action, groups): + """ + A method to update the permissions (group membership) of a bugzilla + user. + + :arg user: The e-mail address of the user to be acted upon. Can + also be a list of emails. + :arg action: add, remove, or set + :arg groups: list of groups to be added to (i.e. ['fedora_contrib']) + """ + groups = self._listify(groups) + if action == "rem": + action = "remove" + if action not in ["add", "remove", "set"]: + raise BugzillaError("Unknown user permission action '%s'" % action) + + update = { + "names": self._listify(user), + "groups": { + action: groups, + } + } + + return self._proxy.User.update(update) diff --git a/bugzilla/bug.py b/bugzilla/bug.py new file mode 100644 index 0000000..d5b581d --- /dev/null +++ b/bugzilla/bug.py @@ -0,0 +1,456 @@ +# base.py - the base classes etc. for a Python interface to bugzilla +# +# Copyright (C) 2007, 2008, 2009, 2010 Red Hat Inc. +# Author: Will Woods +# +# This program is free software; you can redistribute it and/or modify it +# under the terms of the GNU General Public License as published by the +# Free Software Foundation; either version 2 of the License, or (at your +# option) any later version. See http://www.gnu.org/copyleft/gpl.html for +# the full text of the license. + +from __future__ import unicode_literals +import locale +from logging import getLogger +import sys + +log = getLogger(__name__) + + +class Bug(object): + """ + A container object for a bug report. Requires a Bugzilla instance - + every Bug is on a Bugzilla, obviously. + Optional keyword args: + dict=DICT - populate attributes with the result of a getBug() call + bug_id=ID - if dict does not contain bug_id, this is required before + you can read any attributes or make modifications to this + bug. + """ + def __init__(self, bugzilla, bug_id=None, dict=None, autorefresh=False): + # pylint: disable=redefined-builtin + # API had pre-existing issue that we can't change ('dict' usage) + + self.bugzilla = bugzilla + self._bug_fields = [] + self.autorefresh = autorefresh + + if not dict: + dict = {} + if bug_id: + dict["id"] = bug_id + + log.debug("Bug(%s)", sorted(dict.keys())) + self._update_dict(dict) + + self.weburl = bugzilla.url.replace('xmlrpc.cgi', + 'show_bug.cgi?id=%i' % self.bug_id) + + def __str__(self): + """ + Return a simple string representation of this bug + + This is available only for compatibility. Using 'str(bug)' and + 'print(bug)' is not recommended because of potential encoding issues. + Please use unicode(bug) where possible. + """ + if sys.version_info[0] >= 3: + return self.__unicode__() + else: + return self.__unicode__().encode( + locale.getpreferredencoding(), 'replace') + + def __unicode__(self): + """ + Return a simple unicode string representation of this bug + """ + return "#%-6s %-10s - %s - %s" % (self.bug_id, self.bug_status, + self.assigned_to, self.summary) + + def __repr__(self): + return '' % (self.bug_id, self.bugzilla.url, + id(self)) + + def __getattr__(self, name): + refreshed = False + while True: + if refreshed and name in self.__dict__: + # If name was in __dict__ to begin with, __getattr__ would + # have never been called. + return self.__dict__[name] + + # pylint: disable=protected-access + aliases = self.bugzilla._get_bug_aliases() + # pylint: enable=protected-access + + for newname, oldname in aliases: + if name == oldname and newname in self.__dict__: + return self.__dict__[newname] + + # Doing dir(bugobj) does getattr __members__/__methods__, + # don't refresh for those + if name.startswith("__") and name.endswith("__"): + break + + if refreshed or not self.autorefresh: + break + + log.info("Bug %i missing attribute '%s' - doing implicit " + "refresh(). This will be slow, if you want to avoid " + "this, properly use query/getbug include_fields, and " + "set bugzilla.bug_autorefresh = False to force failure.", + self.bug_id, name) + + # We pass the attribute name to getbug, since for something like + # 'attachments' which downloads lots of data we really want the + # user to opt in. + self.refresh(extra_fields=[name]) + refreshed = True + + msg = ("Bug object has no attribute '%s'." % name) + if not self.autorefresh: + msg += ("\nIf '%s' is a bugzilla attribute, it may not have " + "been cached when the bug was fetched. You may want " + "to adjust your include_fields for getbug/query." % name) + raise AttributeError(msg) + + def refresh(self, include_fields=None, exclude_fields=None, + extra_fields=None): + """ + Refresh the bug with the latest data from bugzilla + """ + # pylint: disable=protected-access + r = self.bugzilla._getbug(self.bug_id, + include_fields=include_fields, exclude_fields=exclude_fields, + extra_fields=self._bug_fields + (extra_fields or [])) + # pylint: enable=protected-access + self._update_dict(r) + reload = refresh + + def _update_dict(self, newdict): + """ + Update internal dictionary, in a way that ensures no duplicate + entries are stored WRT field aliases + """ + if self.bugzilla: + self.bugzilla.post_translation({}, newdict) + + # pylint: disable=protected-access + aliases = self.bugzilla._get_bug_aliases() + # pylint: enable=protected-access + + for newname, oldname in aliases: + if oldname not in newdict: + continue + + if newname not in newdict: + newdict[newname] = newdict[oldname] + elif newdict[newname] != newdict[oldname]: + log.debug("Update dict contained differing alias values " + "d[%s]=%s and d[%s]=%s , dropping the value " + "d[%s]", newname, newdict[newname], oldname, + newdict[oldname], oldname) + del(newdict[oldname]) + + for key in newdict.keys(): + if key not in self._bug_fields: + self._bug_fields.append(key) + self.__dict__.update(newdict) + + if 'id' not in self.__dict__ and 'bug_id' not in self.__dict__: + raise TypeError("Bug object needs a bug_id") + + + ################## + # pickle helpers # + ################## + + def __getstate__(self): + ret = {} + for key in self._bug_fields: + ret[key] = self.__dict__[key] + return ret + + def __setstate__(self, vals): + self._bug_fields = [] + self.bugzilla = None + self._update_dict(vals) + + + ##################### + # Modify bug status # + ##################### + + def setstatus(self, status, comment=None, private=False): + """ + Update the status for this bug report. + Commonly-used values are ASSIGNED, MODIFIED, and NEEDINFO. + + To change bugs to CLOSED, use .close() instead. + """ + # Note: fedora bodhi uses this function + vals = self.bugzilla.build_update(status=status, + comment=comment, + comment_private=private) + log.debug("setstatus: update=%s", vals) + + return self.bugzilla.update_bugs(self.bug_id, vals) + + def close(self, resolution, dupeid=None, fixedin=None, + comment=None, isprivate=False): + """ + Close this bug. + Valid values for resolution are in bz.querydefaults['resolution_list'] + For bugzilla.redhat.com that's: + ['NOTABUG', 'WONTFIX', 'DEFERRED', 'WORKSFORME', 'CURRENTRELEASE', + 'RAWHIDE', 'ERRATA', 'DUPLICATE', 'UPSTREAM', 'NEXTRELEASE', + 'CANTFIX', 'INSUFFICIENT_DATA'] + If using DUPLICATE, you need to set dupeid to the ID of the other bug. + If using WORKSFORME/CURRENTRELEASE/RAWHIDE/ERRATA/UPSTREAM/NEXTRELEASE + you can (and should) set 'new_fixed_in' to a string representing the + version that fixes the bug. + You can optionally add a comment while closing the bug. Set 'isprivate' + to True if you want that comment to be private. + """ + # Note: fedora bodhi uses this function + vals = self.bugzilla.build_update(comment=comment, + comment_private=isprivate, + resolution=resolution, + dupe_of=dupeid, + fixed_in=fixedin, + status="CLOSED") + log.debug("close: update=%s", vals) + + return self.bugzilla.update_bugs(self.bug_id, vals) + + + ##################### + # Modify bug emails # + ##################### + + def setassignee(self, assigned_to=None, + qa_contact=None, comment=None): + """ + Set any of the assigned_to or qa_contact fields to a new + bugzilla account, with an optional comment, e.g. + setassignee(assigned_to='wwoods@redhat.com') + setassignee(qa_contact='wwoods@redhat.com', comment='wwoods QA ftw') + + You must set at least one of the two assignee fields, or this method + will throw a ValueError. + + Returns [bug_id, mailresults]. + """ + if not (assigned_to or qa_contact): + raise ValueError("You must set one of assigned_to " + " or qa_contact") + + vals = self.bugzilla.build_update(assigned_to=assigned_to, + qa_contact=qa_contact, + comment=comment) + log.debug("setassignee: update=%s", vals) + + return self.bugzilla.update_bugs(self.bug_id, vals) + + def addcc(self, cclist, comment=None): + """ + Adds the given email addresses to the CC list for this bug. + cclist: list of email addresses (strings) + comment: optional comment to add to the bug + """ + vals = self.bugzilla.build_update(comment=comment, + cc_add=cclist) + log.debug("addcc: update=%s", vals) + + return self.bugzilla.update_bugs(self.bug_id, vals) + + def deletecc(self, cclist, comment=None): + """ + Removes the given email addresses from the CC list for this bug. + """ + vals = self.bugzilla.build_update(comment=comment, + cc_remove=cclist) + log.debug("deletecc: update=%s", vals) + + return self.bugzilla.update_bugs(self.bug_id, vals) + + + #################### + # comment handling # + #################### + + def addcomment(self, comment, private=False): + """ + Add the given comment to this bug. Set private to True to mark this + comment as private. + """ + # Note: fedora bodhi uses this function + vals = self.bugzilla.build_update(comment=comment, + comment_private=private) + log.debug("addcomment: update=%s", vals) + + return self.bugzilla.update_bugs(self.bug_id, vals) + + def getcomments(self): + """ + Returns an array of comment dictionaries for this bug + """ + comment_list = self.bugzilla.get_comments([self.bug_id]) + return comment_list['bugs'][str(self.bug_id)]['comments'] + + + ##################### + # Get/Set bug flags # + ##################### + + def get_flag_type(self, name): + """ + Return flag_type information for a specific flag + + Older RHBugzilla returned a lot more info here, but it was + non-upstream and is now gone. + """ + for t in self.flags: + if t['name'] == name: + return t + return None + + def get_flags(self, name): + """ + Return flag value information for a specific flag + """ + ft = self.get_flag_type(name) + if not ft: + return None + + return [ft] + + def get_flag_status(self, name): + """ + Return a flag 'status' field + + This method works only for simple flags that have only a 'status' field + with no "requestee" info, and no multiple values. For more complex + flags, use get_flags() to get extended flag value information. + """ + f = self.get_flags(name) + if not f: + return None + + # This method works only for simple flags that have only one + # value set. + assert len(f) <= 1 + + return f[0]['status'] + + def updateflags(self, flags): + """ + Thin wrapper around build_update(flags=X). This only handles simple + status changes, anything like needinfo requestee needs to call + build_update + update_bugs directly + + :param flags: Dictionary of the form {"flagname": "status"}, example + {"needinfo": "?", "devel_ack": "+"} + """ + flaglist = [] + for key, value in flags.items(): + flaglist.append({"name": key, "status": value}) + return self.bugzilla.update_bugs([self.bug_id], + self.bugzilla.build_update(flags=flaglist)) + + + ######################## + # Experimental methods # + ######################## + + def get_attachments(self, include_fields=None, exclude_fields=None): + """ + Helper call to Bugzilla.get_attachments. If you want to fetch + specific attachment IDs, use that function instead + """ + if "attachments" in self.__dict__: + return self.attachments + + data = self.bugzilla.get_attachments([self.bug_id], None, + include_fields, exclude_fields) + return data["bugs"][str(self.bug_id)] + + def get_attachment_ids(self): + """ + Helper function to return only the attachment IDs for this bug + """ + return [a["id"] for a in self.get_attachments(exclude_fields=["data"])] + + def get_history_raw(self): + """ + Experimental. Get the history of changes for this bug. + """ + return self.bugzilla.bugs_history_raw([self.bug_id]) + + +class User(object): + """ + Container object for a bugzilla User. + + :arg bugzilla: Bugzilla instance that this User belongs to. + Rest of the params come straight from User.get() + """ + def __init__(self, bugzilla, **kwargs): + self.bugzilla = bugzilla + self.__userid = kwargs.get('id') + self.__name = kwargs.get('name') + + self.__email = kwargs.get('email', self.__name) + self.__can_login = kwargs.get('can_login', False) + + self.real_name = kwargs.get('real_name', None) + self.password = None + + self.groups = kwargs.get('groups', {}) + self.groupnames = [] + for g in self.groups: + if "name" in g: + self.groupnames.append(g["name"]) + self.groupnames.sort() + + + ######################## + # Read-only attributes # + ######################## + + # We make these properties so that the user cannot set them. They are + # unaffected by the update() method so it would be misleading to let them + # be changed. + @property + def userid(self): + return self.__userid + + @property + def email(self): + return self.__email + + @property + def can_login(self): + return self.__can_login + + # name is a key in some methods. Mark it dirty when we change it # + @property + def name(self): + return self.__name + + def refresh(self): + """ + Update User object with latest info from bugzilla + """ + newuser = self.bugzilla.getuser(self.email) + self.__dict__.update(newuser.__dict__) + + def updateperms(self, action, groups): + """ + A method to update the permissions (group membership) of a bugzilla + user. + + :arg action: add, remove, or set + :arg groups: list of groups to be added to (i.e. ['fedora_contrib']) + """ + self.bugzilla.updateperms(self.name, action, groups) diff --git a/bugzilla/bugzilla3.py b/bugzilla/bugzilla3.py new file mode 100644 index 0000000..efacdea --- /dev/null +++ b/bugzilla/bugzilla3.py @@ -0,0 +1,34 @@ +# bugzilla3.py - a Python interface to Bugzilla 3.x using xmlrpclib. +# +# Copyright (C) 2008, 2009 Red Hat Inc. +# Author: Will Woods +# +# This program is free software; you can redistribute it and/or modify it +# under the terms of the GNU General Public License as published by the +# Free Software Foundation; either version 2 of the License, or (at your +# option) any later version. See http://www.gnu.org/copyleft/gpl.html for +# the full text of the license. + +from bugzilla.base import BugzillaBase + + +class Bugzilla3(BugzillaBase): + bz_ver_major = 3 + bz_ver_minor = 0 + + +class Bugzilla32(Bugzilla3): + bz_ver_minor = 2 + + +class Bugzilla34(Bugzilla32): + bz_ver_minor = 4 + + +class Bugzilla36(Bugzilla34): + bz_ver_minor = 6 + + def _getbugfields(self): + '''Get the list of valid fields for Bug objects''' + r = self._proxy.Bug.fields({'include_fields': ['name']}) + return [f['name'] for f in r['fields']] diff --git a/bugzilla/bugzilla4.py b/bugzilla/bugzilla4.py new file mode 100644 index 0000000..7f5e127 --- /dev/null +++ b/bugzilla/bugzilla4.py @@ -0,0 +1,47 @@ +# +# Copyright (C) 2008-2012 Red Hat Inc. +# Author: Michal Novotny +# +# This program is free software; you can redistribute it and/or modify it +# under the terms of the GNU General Public License as published by the +# Free Software Foundation; either version 2 of the License, or (at your +# option) any later version. See http://www.gnu.org/copyleft/gpl.html for +# the full text of the license. + +from bugzilla.bugzilla3 import Bugzilla36 + + +class Bugzilla4(Bugzilla36): + bz_ver_major = 4 + bz_ver_minor = 0 + + + ################# + # Query Methods # + ################# + + def build_query(self, **kwargs): + query = Bugzilla36.build_query(self, **kwargs) + + # 'include_fields' only available for Bugzilla4+ + include_fields = self._convert_include_field_list( + kwargs.pop('include_fields', None)) + if include_fields: + if 'id' not in include_fields: + include_fields.append('id') + query["include_fields"] = include_fields + + exclude_fields = self._convert_include_field_list( + kwargs.pop('exclude_fields', None)) + if exclude_fields: + query["exclude_fields"] = exclude_fields + + return query + + +class Bugzilla42(Bugzilla4): + bz_ver_minor = 2 + + +class Bugzilla44(Bugzilla42): + bz_ver_minor = 4 diff --git a/bugzilla/oldclasses.py b/bugzilla/oldclasses.py new file mode 100644 index 0000000..18169e7 --- /dev/null +++ b/bugzilla/oldclasses.py @@ -0,0 +1,23 @@ +# This program is free software; you can redistribute it and/or modify it +# under the terms of the GNU General Public License as published by the +# Free Software Foundation; either version 2 of the License, or (at your +# option) any later version. See http://www.gnu.org/copyleft/gpl.html for +# the full text of the license. + +from .base import Bugzilla +from .rhbugzilla import RHBugzilla + + +# These are old compat classes. Nothing new should be added here, +# and these should not be altered + +class Bugzilla3(Bugzilla): pass +class Bugzilla32(Bugzilla): pass +class Bugzilla34(Bugzilla): pass +class Bugzilla36(Bugzilla): pass +class Bugzilla4(Bugzilla): pass +class Bugzilla42(Bugzilla): pass +class Bugzilla44(Bugzilla): pass +class NovellBugzilla(Bugzilla): pass +class RHBugzilla3(RHBugzilla): pass +class RHBugzilla4(RHBugzilla): pass diff --git a/bugzilla/rhbugzilla.py b/bugzilla/rhbugzilla.py new file mode 100644 index 0000000..b82365e --- /dev/null +++ b/bugzilla/rhbugzilla.py @@ -0,0 +1,354 @@ +# rhbugzilla.py - a Python interface to Red Hat Bugzilla using xmlrpclib. +# +# Copyright (C) 2008-2012 Red Hat Inc. +# Author: Will Woods +# +# This program is free software; you can redistribute it and/or modify it +# under the terms of the GNU General Public License as published by the +# Free Software Foundation; either version 2 of the License, or (at your +# option) any later version. See http://www.gnu.org/copyleft/gpl.html for +# the full text of the license. + +from logging import getLogger + +from .base import Bugzilla + +log = getLogger(__name__) + + +class RHBugzilla(Bugzilla): + """ + Bugzilla class for connecting Red Hat's forked bugzilla instance, + bugzilla.redhat.com + + Historically this class used many more non-upstream methods, but + in 2012 RH started dropping most of its custom bits. By that time, + upstream BZ had most of the important functionality. + + Much of the remaining code here is just trying to keep things operating + in python-bugzilla back compatible manner. + + This class was written using bugzilla.redhat.com's API docs: + https://bugzilla.redhat.com/docs/en/html/api/ + """ + def _init_class_state(self): + def _add_both_alias(newname, origname): + self._add_field_alias(newname, origname, is_api=False) + self._add_field_alias(origname, newname, is_bug=False) + + _add_both_alias('fixed_in', 'cf_fixed_in') + _add_both_alias('qa_whiteboard', 'cf_qa_whiteboard') + _add_both_alias('devel_whiteboard', 'cf_devel_whiteboard') + _add_both_alias('internal_whiteboard', 'cf_internal_whiteboard') + + self._add_field_alias('component', 'components', is_bug=False) + self._add_field_alias('version', 'versions', is_bug=False) + # Yes, sub_components is the field name the API expects + self._add_field_alias('sub_components', 'sub_component', is_bug=False) + + # flags format isn't exactly the same but it's the closest approx + self._add_field_alias('flags', 'flag_types') + + self._getbug_extra_fields = self._getbug_extra_fields + [ + "comments", "description", + "external_bugs", "flags", "sub_components", + "tags", + ] + self._supports_getbug_extra_fields = True + + + ###################### + # Bug update methods # + ###################### + + def build_update(self, **kwargs): + # pylint: disable=arguments-differ + adddict = {} + + def pop(key, destkey): + val = kwargs.pop(key, None) + if val is None: + return + adddict[destkey] = val + + def get_sub_component(): + val = kwargs.pop("sub_component", None) + if val is None: + return + + if not isinstance(val, dict): + component = self._listify(kwargs.get("component")) + if not component: + raise ValueError("component must be specified if " + "specifying sub_component") + val = {component[0]: val} + adddict["sub_components"] = val + + def get_alias(): + # RHBZ has a custom extension to allow a bug to have multiple + # aliases, so the format of aliases is + # {"add": [...], "remove": [...]} + # But that means in order to approximate upstream, behavior + # which just overwrites the existing alias, we need to read + # the bug's state first to know what string to remove. Which + # we can't do, since we don't know the bug numbers at this point. + # So fail for now. + # + # The API should provide {"set": [...]} + # https://bugzilla.redhat.com/show_bug.cgi?id=1173114 + # + # Implementation will go here when it's available + pass + + pop("fixed_in", "cf_fixed_in") + pop("qa_whiteboard", "cf_qa_whiteboard") + pop("devel_whiteboard", "cf_devel_whiteboard") + pop("internal_whiteboard", "cf_internal_whiteboard") + + get_sub_component() + get_alias() + + vals = Bugzilla.build_update(self, **kwargs) + vals.update(adddict) + + return vals + + def add_external_tracker(self, bug_ids, ext_bz_bug_id, ext_type_id=None, + ext_type_description=None, ext_type_url=None, + ext_status=None, ext_description=None, + ext_priority=None): + """ + Wrapper method to allow adding of external tracking bugs using the + ExternalBugs::WebService::add_external_bug method. + + This is documented at + https://bugzilla.redhat.com/docs/en/html/api/extensions/ExternalBugs/lib/WebService.html#add_external_bug + + bug_ids: A single bug id or list of bug ids to have external trackers + added. + ext_bz_bug_id: The external bug id (ie: the bug number in the + external tracker). + ext_type_id: The external tracker id as used by Bugzilla. + ext_type_description: The external tracker description as used by + Bugzilla. + ext_type_url: The external tracker url as used by Bugzilla. + ext_status: The status of the external bug. + ext_description: The description of the external bug. + ext_priority: The priority of the external bug. + """ + param_dict = {'ext_bz_bug_id': ext_bz_bug_id} + if ext_type_id is not None: + param_dict['ext_type_id'] = ext_type_id + if ext_type_description is not None: + param_dict['ext_type_description'] = ext_type_description + if ext_type_url is not None: + param_dict['ext_type_url'] = ext_type_url + if ext_status is not None: + param_dict['ext_status'] = ext_status + if ext_description is not None: + param_dict['ext_description'] = ext_description + if ext_priority is not None: + param_dict['ext_priority'] = ext_priority + params = { + 'bug_ids': self._listify(bug_ids), + 'external_bugs': [param_dict], + } + + log.debug("Calling ExternalBugs.add_external_bug(%s)", params) + return self._proxy.ExternalBugs.add_external_bug(params) + + def update_external_tracker(self, ids=None, ext_type_id=None, + ext_type_description=None, ext_type_url=None, + ext_bz_bug_id=None, bug_ids=None, + ext_status=None, ext_description=None, + ext_priority=None): + """ + Wrapper method to allow adding of external tracking bugs using the + ExternalBugs::WebService::update_external_bug method. + + This is documented at + https://bugzilla.redhat.com/docs/en/html/api/extensions/ExternalBugs/lib/WebService.html#update_external_bug + + ids: A single external tracker bug id or list of external tracker bug + ids. + ext_type_id: The external tracker id as used by Bugzilla. + ext_type_description: The external tracker description as used by + Bugzilla. + ext_type_url: The external tracker url as used by Bugzilla. + ext_bz_bug_id: A single external bug id or list of external bug ids + (ie: the bug number in the external tracker). + bug_ids: A single bug id or list of bug ids to have external tracker + info updated. + ext_status: The status of the external bug. + ext_description: The description of the external bug. + ext_priority: The priority of the external bug. + """ + params = {} + if ids is not None: + params['ids'] = self._listify(ids) + if ext_type_id is not None: + params['ext_type_id'] = ext_type_id + if ext_type_description is not None: + params['ext_type_description'] = ext_type_description + if ext_type_url is not None: + params['ext_type_url'] = ext_type_url + if ext_bz_bug_id is not None: + params['ext_bz_bug_id'] = self._listify(ext_bz_bug_id) + if bug_ids is not None: + params['bug_ids'] = self._listify(bug_ids) + if ext_status is not None: + params['ext_status'] = ext_status + if ext_description is not None: + params['ext_description'] = ext_description + if ext_priority is not None: + params['ext_priority'] = ext_priority + + log.debug("Calling ExternalBugs.update_external_bug(%s)", params) + return self._proxy.ExternalBugs.update_external_bug(params) + + def remove_external_tracker(self, ids=None, ext_type_id=None, + ext_type_description=None, ext_type_url=None, + ext_bz_bug_id=None, bug_ids=None): + """ + Wrapper method to allow removal of external tracking bugs using the + ExternalBugs::WebService::remove_external_bug method. + + This is documented at + https://bugzilla.redhat.com/docs/en/html/api/extensions/ExternalBugs/lib/WebService.html#remove_external_bug + + ids: A single external tracker bug id or list of external tracker bug + ids. + ext_type_id: The external tracker id as used by Bugzilla. + ext_type_description: The external tracker description as used by + Bugzilla. + ext_type_url: The external tracker url as used by Bugzilla. + ext_bz_bug_id: A single external bug id or list of external bug ids + (ie: the bug number in the external tracker). + bug_ids: A single bug id or list of bug ids to have external tracker + info updated. + """ + params = {} + if ids is not None: + params['ids'] = self._listify(ids) + if ext_type_id is not None: + params['ext_type_id'] = ext_type_id + if ext_type_description is not None: + params['ext_type_description'] = ext_type_description + if ext_type_url is not None: + params['ext_type_url'] = ext_type_url + if ext_bz_bug_id is not None: + params['ext_bz_bug_id'] = self._listify(ext_bz_bug_id) + if bug_ids is not None: + params['bug_ids'] = self._listify(bug_ids) + + log.debug("Calling ExternalBugs.remove_external_bug(%s)", params) + return self._proxy.ExternalBugs.remove_external_bug(params) + + + ################# + # Query methods # + ################# + + def pre_translation(self, query): + """ + Translates the query for possible aliases + """ + old = query.copy() + + if 'bug_id' in query: + if not isinstance(query['bug_id'], list): + query['id'] = query['bug_id'].split(',') + else: + query['id'] = query['bug_id'] + del query['bug_id'] + + if 'component' in query: + if not isinstance(query['component'], list): + query['component'] = query['component'].split(',') + + if 'include_fields' not in query and 'column_list' not in query: + return + + if 'include_fields' not in query: + query['include_fields'] = [] + if 'column_list' in query: + query['include_fields'] = query['column_list'] + del query['column_list'] + + # We need to do this for users here for users that + # don't call build_query + query.update(self._process_include_fields(query["include_fields"], + None, None)) + + if old != query: + log.debug("RHBugzilla pretranslated query to: %s", query) + + def post_translation(self, query, bug): + """ + Convert the results of getbug back to the ancient RHBZ value + formats + """ + ignore = query + + # RHBZ _still_ returns component and version as lists, which + # deviates from upstream. Copy the list values to components + # and versions respectively. + if 'component' in bug and "components" not in bug: + val = bug['component'] + bug['components'] = isinstance(val, list) and val or [val] + bug['component'] = bug['components'][0] + + if 'version' in bug and "versions" not in bug: + val = bug['version'] + bug['versions'] = isinstance(val, list) and val or [val] + bug['version'] = bug['versions'][0] + + # sub_components isn't too friendly of a format, add a simpler + # sub_component value + if 'sub_components' in bug and 'sub_component' not in bug: + val = bug['sub_components'] + bug['sub_component'] = "" + if isinstance(val, dict): + values = [] + for vallist in val.values(): + values += vallist + bug['sub_component'] = " ".join(values) + + def build_external_tracker_boolean_query(self, *args, **kwargs): + ignore1 = args + ignore2 = kwargs + raise RuntimeError("Building external boolean queries is " + "no longer supported. Please build a URL query " + "via the bugzilla web UI and pass it to 'query --from-url' " + "or url_to_query()") + + + def build_query(self, **kwargs): + # pylint: disable=arguments-differ + + # We previously accepted a text format to approximate boolean + # queries, and only for RHBugzilla. Upstream bz has --from-url + # support now, so point people to that instead so we don't have + # to document and maintain this logic anymore + def _warn_bool(kwkey): + vallist = self._listify(kwargs.get(kwkey, None)) + for value in vallist or []: + for s in value.split(" "): + if s not in ["|", "&", "!"]: + continue + log.warning("%s value '%s' appears to use the now " + "unsupported boolean formatting, your query may " + "be incorrect. If you need complicated URL queries, " + "look into bugzilla --from-url/url_to_query().", + kwkey, value) + return + + _warn_bool("fixed_in") + _warn_bool("blocked") + _warn_bool("dependson") + _warn_bool("flag") + _warn_bool("qa_whiteboard") + _warn_bool("devel_whiteboard") + _warn_bool("alias") + + return Bugzilla.build_query(self, **kwargs) diff --git a/bugzilla/transport.py b/bugzilla/transport.py new file mode 100644 index 0000000..3dbb425 --- /dev/null +++ b/bugzilla/transport.py @@ -0,0 +1,201 @@ +# This program is free software; you can redistribute it and/or modify it +# under the terms of the GNU General Public License as published by the +# Free Software Foundation; either version 2 of the License, or (at your +# option) any later version. See http://www.gnu.org/copyleft/gpl.html for +# the full text of the license. + +from logging import getLogger +import sys + +# pylint: disable=import-error +if sys.version_info[0] >= 3: + from configparser import ConfigParser + from urllib.parse import urlparse # pylint: disable=no-name-in-module + from xmlrpc.client import Fault, ProtocolError, ServerProxy, Transport +else: + from ConfigParser import SafeConfigParser as ConfigParser + from urlparse import urlparse + from xmlrpclib import Fault, ProtocolError, ServerProxy, Transport +# pylint: enable=import-error + +import requests + + +log = getLogger(__name__) + + +class BugzillaError(Exception): + """ + Error raised in the Bugzilla client code. + """ + pass + + +class _BugzillaTokenCache(object): + """ + Cache for tokens, including, with apologies for the duplicative + terminology, both Bugzilla Tokens and API Keys. + """ + + def __init__(self, uri, tokenfilename): + self.tokenfilename = tokenfilename + self.tokenfile = ConfigParser() + self.domain = urlparse(uri)[1] + + if self.tokenfilename: + self.tokenfile.read(self.tokenfilename) + + if self.domain not in self.tokenfile.sections(): + self.tokenfile.add_section(self.domain) + + @property + def value(self): + if self.tokenfile.has_option(self.domain, 'token'): + return self.tokenfile.get(self.domain, 'token') + else: + return None + + @value.setter + def value(self, value): + if self.value == value: + return + + if value is None: + self.tokenfile.remove_option(self.domain, 'token') + else: + self.tokenfile.set(self.domain, 'token', value) + + if self.tokenfilename: + with open(self.tokenfilename, 'w') as tokenfile: + log.debug("Saving to tokenfile") + self.tokenfile.write(tokenfile) + + def __repr__(self): + return '' % self.value + + +class _BugzillaServerProxy(ServerProxy, object): + def __init__(self, uri, tokenfile, *args, **kwargs): + super(_BugzillaServerProxy, self).__init__(uri, *args, **kwargs) + self.token_cache = _BugzillaTokenCache(uri, tokenfile) + self.api_key = None + + def use_api_key(self, api_key): + self.api_key = api_key + + def clear_token(self): + self.token_cache.value = None + + def _ServerProxy__request(self, methodname, params): + if len(params) == 0: + params = ({}, ) + + log.debug("XMLRPC call: %s(%s)", methodname, params[0]) + + if self.api_key is not None: + if 'Bugzilla_api_key' not in params[0]: + params[0]['Bugzilla_api_key'] = self.api_key + elif self.token_cache.value is not None: + if 'Bugzilla_token' not in params[0]: + params[0]['Bugzilla_token'] = self.token_cache.value + + # pylint: disable=no-member + ret = super(_BugzillaServerProxy, + self)._ServerProxy__request(methodname, params) + # pylint: enable=no-member + + if isinstance(ret, dict) and 'token' in ret.keys(): + self.token_cache.value = ret.get('token') + return ret + + +class _RequestsTransport(Transport): + user_agent = 'Python/Bugzilla' + + def __init__(self, url, cookiejar=None, + sslverify=True, sslcafile=None, debug=True, cert=None): + if hasattr(Transport, "__init__"): + Transport.__init__(self, use_datetime=False) + + self.verbose = debug + self._cookiejar = cookiejar + + # transport constructor needs full url too, as xmlrpc does not pass + # scheme to request + self.scheme = urlparse(url)[0] + if self.scheme not in ["http", "https"]: + raise Exception("Invalid URL scheme: %s (%s)" % (self.scheme, url)) + + self.use_https = self.scheme == 'https' + + self.request_defaults = { + 'cert': sslcafile if self.use_https else None, + 'cookies': cookiejar, + 'verify': sslverify, + 'headers': { + 'Content-Type': 'text/xml', + 'User-Agent': self.user_agent, + } + } + + # Using an explicit Session, rather than requests.get, will use + # HTTP KeepAlive if the server supports it. + self.session = requests.Session() + if cert: + self.session.cert = cert + + def parse_response(self, response): + """ + Parse XMLRPC response + """ + parser, unmarshaller = self.getparser() + parser.feed(response.text.encode('utf-8')) + parser.close() + return unmarshaller.close() + + def _request_helper(self, url, request_body): + """ + A helper method to assist in making a request and provide a parsed + response. + """ + response = None + try: + response = self.session.post( + url, data=request_body, **self.request_defaults) + + # We expect utf-8 from the server + response.encoding = 'UTF-8' + + # update/set any cookies + if self._cookiejar is not None: + for cookie in response.cookies: + self._cookiejar.set_cookie(cookie) + + if self._cookiejar.filename is not None: + # Save is required only if we have a filename + self._cookiejar.save() + + response.raise_for_status() + return self.parse_response(response) + except requests.RequestException as e: + if not response: + raise + raise ProtocolError( + url, response.status_code, str(e), response.headers) + except Fault: + raise + except Exception: + e = BugzillaError(str(sys.exc_info()[1])) + # pylint: disable=attribute-defined-outside-init + e.__traceback__ = sys.exc_info()[2] + # pylint: enable=attribute-defined-outside-init + raise e + + def request(self, host, handler, request_body, verbose=0): + self.verbose = verbose + url = "%s://%s%s" % (self.scheme, host, handler) + + # xmlrpclib fails to escape \r + request_body = request_body.replace(b'\r', b' ') + + return self._request_helper(url, request_body) diff --git a/data/ksc.conf b/data/ksc.conf new file mode 100644 index 0000000..8ee4551 --- /dev/null +++ b/data/ksc.conf @@ -0,0 +1,6 @@ +[bugzilla] +user=user@redhat.com +partner=partner-name +partnergroup=partner-group +server=https://bugzilla.redhat.com/xmlrpc.cgi +api_key=api_key diff --git a/ksc b/ksc new file mode 100755 index 0000000..8b3dde4 --- /dev/null +++ b/ksc @@ -0,0 +1,4 @@ +#!/bin/sh +export PYTHONPATH=/usr/share/ksc +exec /usr/libexec/platform-python -tt /usr/share/ksc/ksc.py "$@" + diff --git a/ksc.1 b/ksc.1 new file mode 100644 index 0000000..ff1faf3 --- /dev/null +++ b/ksc.1 @@ -0,0 +1,166 @@ +.TH ksc "1" "November 2018" "ksc - Version 1.6" "User Commands" +.SH NAME +ksc \- report symbols used by kernel modules +.SH SYNOPSIS +.B ksc -k \fIFILE\fR ... [OPTIONS] + +.B ksc -k \fIFILE\fR ... -s [OPTIONS] + +.B ksc -s -p \fIKSC_RESULT\fR [OPTIONS] + +.SH DESCRIPTION +.B ksc +produces and submits reports of symbols used by kernel object \fIFILE\fRs +(specified by -k). Resulting report is always saved to \fI~/ksc-result.txt\fR. + +For more on report generation, see -k. + +For more on report submission, see -s. + +.SH OPTIONS +You may freely mix different option styles. + +.TP +\fB\-h\fR, \fB\-\-help\fR +Show help message and exit. +.TP +\fB\-c\fR \fICONFIG\fR, \fB\-\-\fRconfig=\fICONFIG\fR +Read config options from \fICONFIG\fR file. If not supplied, \fBksc\fR will use +~/ksc.conf. If not found, /etc/ksc.conf will be used instead. +.TP +\fB\-j\fR \fIKSC_RESULT\fR, \fB\-\-\fRjustification-from=\fIKSC_RESULT\fR +Read symbol justifications from \fIKSC_RESULT\fR file and use them in the new +report. It is important to note that placeholder justifications and reference +justifications are ignored by this option and re-generated every time. + +This option may be specified multiple times. \fIKSC_RESULT\fR file order is +determined by the order of arguments passed to ksc. If there are two conflicting +non-placeholder/non-reference justifications for a kernel module A.ko symbol S +in multiple report files at the same time, it will be carried over from the +last one specified on the command line. + +Symbol justifications can be carried over within the same kernel module only. +Justifications do not propagate through references (i.e., if you justify +usage of S for A.ko, B.ko references justification in A.ko, and you run ksc +without A.ko, justification is not carried over and you will be asked to +specify). +.TP +\fB\-k\fR \fIFILE\fR, \fB\-\-ko\fR=\fIFILE\fR +Collect used symbols in kernel object \fIFILE\fR. +.br +Multiple -k arguments may be supplied, in which case different \fIFILE\fRs are +placed in distinct sections of the \fI~/ksc-result.txt\fR file. + +All kernel object \fIFILE\fRs must be build against the same architecture. + +If multiple \fIFILE\fRs are supplied, ksc report won't include symbols exported +by any one of the \fIFILE\fRs. This allows for bulk collection of symbols of +a functionally dependent set of kernel modules without having to filter +symbols coming from other modules. +.TP +\fB\-K\fR \fIFILE\fR, \fB\-\-ko-dependency\fR=\fIFILE\fR +Omit any symbols exported by \fIFILE\fR from the report. Use this option if +other modules (which are part of the report) depend on symbols exported by +\fIFILE\fR, but \fIFILE\fR itself is not to be reported at this time. +.TP +\fB\-n\fR \fIRELEASE\fR, \fB\-\-name\fR=\fIRELEASE\fR +RHEL \fIRELEASE\fR against which the bug is to be filed, e.g '6.5'. +.TP +\fB\-p\fR \fIPREVIOUS\fR, \fB\-\-previous\fR=\fIPREVIOUS\fR +Specified \fIPREVIOUS\fR file is marked for submission to Red Hat Bugzilla. +.br +To submit, you need to use -s. +.TP +\fB\-r\fR \fIRELEASE\fR, \fB\-\-release\fR=\fIRELEASE\fR +RHEL \fIRELEASE\fR used for symbol comparison with whitelists, e.g. '6.5'. +.TP +\fB\-y\fR \fISYMVERS\fR, \fB\-\-symvers\fR=\fISYMVERS\fR +Path to Module.symvers \fISYMVERS\fR file used to check symbol presence in the +the kernel. If not specified, current kernel version is used to determine the +path and Module.symvers of the currently running kernel is used. + +You are notified whenever symbol is not present in the Module.symvers file. +.TP +\fB\-s\fR, \fB\-\-submit\fR +Submit the report to the Red Hat Bugzilla (https://bugzilla.redhat.com). +Credentials used in submission need to be set a config file (see FILES section +below). If API key is not set, ksc will prompt for Bugzilla password. + +This option needs to be supplemented with -k or -p. + +If used with -k, report gets generated, opened in an editor for you to make +changes, saved in \fI~/ksc-result.txt\fR, and submitted. You may interrupt +submission by pressing ^C. +.TP +\fB\-v\fR, \fB\-\-version\fR +Prints ksc version. + +.SH FILES + +.TP +~/ksc-result.txt +Report produced by ksc. + +.TP +~/ksc.conf (/etc/ksc.conf) +Local (global) ksc config file. Global config file is used only when local +config file does not exist. Syntax is as follows: +.IP +.br +[bugzilla] +.br +user=user@redhat.com +.br +partner=partner\-name +.br +partnergroup=partner\-group +.br +server=https://bugzilla.redhat.com/xmlrpc.cgi +.br +api_key=api_key + +These are default values. Whenever a field has such a value (except the +server field), it will always be treated as though it were undefined/left blank. + +.SH EXAMPLES +.TP +ksc -k a.ko +Process a.ko file using system-provided whitelist and Module.symvers files. +Reports symbols used by a.ko that are not in whitelist, requesting justification +for their use. + +.TP +ksc -k a.ko -y /tmp/Module.symvers +Process a.ko file using system-provided whitelist and user-provided +Module.symvers (/tmp/Module.symvers) files. Reports symbols used by a.ko that +are not in whitelist, requesting justification for their use. + +.TP +ksc -k a.ko -k b.ko +Process a.ko and b.ko files using system-provided whitelist and Module.symvers +files. Reports symbols used by both a.ko and b.ko that are not in whitelist, +requesting justification for their use. Only one justification per symbol is +required. Any mention of an already justified symbol come with an automatically +generated references. + +.TP +ksc -k a.ko -k b.ko -K c.ko -K d.ko +Process a.ko and b.ko files using system-provided whitelist and Module.symvers +files. Reports symbols used by both a.ko and b.ko that are not in whitelist and +not exported by either c.ko or d.ko, requesting justification for their use. +Only one justification per symbol is required. Any mention of an already +justified symbol come with an automatically generated references. + +.TP +ksc -k a.ko -k b.ko -j previous-report.txt +Process a.ko and b.ko files using system-provided whitelist and Module.symvers +files. Reports symbols used by both a.ko and b.ko that are not in whitelist, +requesting justification for their use. Only one justification per symbol is +required. Any mention of an already justified symbol come with an automatically +generated references. Justifications already present in previous-report.txt +are carried over to the newly generated one. + +.TP +ksc -s -p report-to-submit.txt +Submit a report to bugzilla. + diff --git a/ksc.py b/ksc.py new file mode 100755 index 0000000..6e58bb9 --- /dev/null +++ b/ksc.py @@ -0,0 +1,613 @@ +#!/usr/libexec/platform-python +# Copyright 2012,2018 Red Hat Inc. +# Author: Kushal Das + +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation; either version 2 of the License, or +# (at your option) any later version. See +# http://www.gnu.org/copyleft/gpl.html for the full text of the +# license. +# +import os +import re +import sys +from optparse import OptionParser, SUPPRESS_HELP +from utils import run, read_list +from utils import read_total_list, get_release_name +from utils import createbug +from utils import query_user, query_user_bool + +KSCVERSION = "ksc - Version 1.6" + + +class Ksc(object): + + # RE to detect ksc cli execution + HEADER_RE = re.compile(r"\[command: (?P.*)\]") + + # RE to extract KO basename + SECTION_KO_RE = re.compile(r'{(?P.*)}') + + # RE to match with KO-body section + LISTS_RE = re.compile(r'.*\[WHITELISTUSAGE\](?P.*)\[NONWHITELISTUSAGE\]\s*(?P.*)', re.S) + + # RE to extract symbol and its justification + JUSTIFICATION_RE = re.compile(r'#*\s*\((?P.*)\)\s*(?P[^#]*)') + + # Non-whitelisted symbols justification free-form entries + JUSTIFICATION_PLACEHOLDER = "ENTER JUSTIFICATION TEXT HERE" + JUSTIFICATION_REFERENCE = "JUSTIFICATION STATED UNDER `%s' SECTION" + JUSTIFICATION_REF_DETECT = re.compile(r"JUSTIFICATION STATED UNDER `(.*)' SECTION") + JUSTIFICATION_SEPARATOR = '#' * 10 + JUSTIFICATION_BODY = '\n(%s)\n\n%s\n\n' + + # Sections + SECTION_WHITELISTS = "[WHITELISTUSAGE]\n" + SECTION_CO_WHITELISTS = "[NONWHITELISTUSAGE]\n" + + def __init__(self, mock=False): + """ + Init call + """ + self.all_symbols_used = {} + self.nonwhite_symbols_used = {} + self.white_symbols = {} + self.defined_symbols = {} + self.justified_symbols = {} + self.justifications = {} + self.matchdata = None + self.total = None + self.verbose = False + self.mock = mock + self.releasedir = None + self.symvers = None + self.arch = None + self.vermagic = {} + if mock: + self.releasename = '7.0' + else: + self.releasename = None + + def clean(self): + self.all_symbols_used = {} + self.nonwhite_symbols_used = {} + self.white_symbols = {} + self.defined_symbols = {} + self.justified_symbols = {} + self.justifications = {} + self.matchdata = None + self.total = None + self.vermagic = {} + + def main(self, mock_options=None): + """ + Main function for the logic + """ + filename = os.path.join(os.path.expanduser("~"), "ksc-result.txt") + # default architecture + self.arch = "x86_64" + + parser = OptionParser() + parser.add_option("-c", "--config", dest="config", + help="path to configuration file", metavar="CONFIG") + parser.add_option("-k", "--ko", action="append", dest="ko", + help="path to the ko file", metavar="KO") + parser.add_option("-K", "--ko-dependency", action="append", + dest="ko_dependency", help="path to a dependency", + metavar="DEPENDENCY") + parser.add_option("-n", "--name", dest="releasename", + help="Red Hat release to file the bug against, " + "e.g '6.7'", metavar="RELEASENAME") + parser.add_option("-p", "--previous", dest="previous", + help="path to previous resultset to submit as bug") + parser.add_option("-r", "--release", dest="release", + help="RHEL whitelist release to compare against, " + "e.g '6.7'", metavar="RELEASE") + parser.add_option("-y", "--symvers", dest="symvers", + help="Path to the Module.symvers file. " + "The current kernel path is used if " + "not specified.", + metavar="SYMVERS") + parser.add_option("-s", "--submit", + action="store_true", dest="submit", default=False, + help="Submit to Red Hat Bugzilla") + parser.add_option("-v", "--version", + action="store_true", dest="VERSION", default=False, + help="Prints KSC version number") + parser.add_option("-j", "--justification-from", action="append", + dest="justification_from", metavar="JUSTIFICATION", + help="read justification from previous ksc-result") + + if not self.mock: # pragma: no cover + (options, args) = parser.parse_args(sys.argv[1:]) + else: # pragma: no cover + (options, args) = parser.parse_args(mock_options) + + if options.VERSION: + print(KSCVERSION) + sys.exit(0) + + # Create the ksc.conf config path + if options.config: + path = os.path.abspath(os.path.expanduser(options.config)) + else: + path = os.path.expanduser("~/ksc.conf") + + if options.releasename: + self.releasename = options.releasename + if not self.valid_release_version(self.releasename): + sys.exit(1) + + if options.release: + if not self.valid_release_version(options.release): + sys.exit(1) + + if options.releasename and options.release and \ + options.release != options.releasename: + print("Release and release name do not match.") + sys.exit(1) + + if options.previous: # Submit the result of previous run + filename = os.path.abspath(os.path.expanduser(options.previous)) + if os.path.basename(filename) != 'ksc-result.txt': + print("Please point to the ksc-result.txt file in -p option.") + sys.exit(1) + + self.submit(filename, path) + return + + self.releasedir = 'kabi-current' + if options.release: + if not self.valid_release_version(options.release): + sys.exit(1) + + self.releasedir = 'kabi-rhel' + options.release.replace('.', '') + + if options.symvers: + self.symvers = options.symvers + + if options.justification_from: + for file in options.justification_from: + self.read_justifications(file) + + if options.ko_dependency: + for kmod_path in options.ko_dependency: + self.parse_ko(kmod_path, process_whitelists=False) + + if options.ko: + self.find_arch(options.ko) + + exists = self.read_data(self.arch, self.releasedir, self.symvers) + # Return if there is any issue in reading whitelists + if not exists: + print(("Release %s for arch %s was not found.\n" + "Do you have right kernel-abi-whitelist installed ?" % + (self.releasedir, self.arch))) + sys.exit(1) + + for kmod_path in options.ko: + self.parse_ko(kmod_path, process_whitelists=True) + + self.remove_internal_symbols() + + for kmod_path in options.ko: + self.print_result(kmod_path) + + self.save_result() + + else: # pragma: no cover + print("You need to provide a path to at least one .ko file.") + sys.exit(1) + + # Now save the result + + if not options.submit: + return + + if not self.mock: # pragma: no cover + self.get_justification(filename) + self.submit(filename, path) + + def read_justifications(self, filepath): + filepath = os.path.abspath(os.path.expanduser(filepath)) + + if not os.path.isfile(filepath): + print("Filename `%s' does not exist!" % filepath) + return + + with open(filepath, "r") as fd: + + filename_section = "" + + for file_contents in re.split("({[^}]*})", fd.read()): + + filename_match = self.SECTION_KO_RE.match(file_contents) + if filename_match: + filename_section = filename_match.group('ko_file') + + match = self.LISTS_RE.match(file_contents) + if not match: + continue + + for symbol, justification in \ + self.JUSTIFICATION_RE.findall(file_contents): + symbol = symbol.strip() + justification = justification.strip() + + if justification == self.JUSTIFICATION_PLACEHOLDER: + continue + + if self.JUSTIFICATION_REF_DETECT.match(justification): + continue + + if filename_section not in self.justifications: + self.justifications[filename_section] = {} + + self.justifications[filename_section][symbol] = \ + justification + + if symbol not in self.justified_symbols: + self.justified_symbols[symbol] = \ + os.path.basename(filename_section) + + def valid_release_version(self, release): + rels = release.split(".") + if len(rels) != 2: + print("Invalid release: %s" % release) + return False + if not rels[0].isdigit() or int(rels[0]) <= 5: + print("Invalid release: %s" % release) + return False + return True + + def submit_get_consent(self): + + """ + Part of the submission process. User gets queried for Red Hat's + receipt and internal use. Consent is mandatory. + """ + + consent_string = "By using ksc to upload your data to Red Hat, " \ + "you consent to Red Hat's receipt use and analysis of this " \ + "data. Do you agree? [y/N] " + + consent = query_user_bool(consent_string) + if consent.lower() != 'y': + print("Cannot proceed without consent. Qutting.") + sys.exit(1) + + def submit_get_release(self): + + """ + Part of the submission process. User gets queried for release if + not explicitly provided via argv. + """ + + # Release has been specified as argv, no need to query user at this time + if self.releasename is not None: + return + + print("RHEL release not specified with -n flag. Defaulting to your " + "system's RHEL release.") + + self.releasename = get_release_name() + use_sys_release = "" + if not self.releasename: + print("Unable to determine system's release name. Please specify.") + + else: + query = "File bug against RHEL release %s?" % self.releasename + query += "\n" + query += "y/N: " + + use_sys_release = query_user_bool(query) + + if not use_sys_release: + print("Unable to determine user option. Qutting.") + sys.exit(1) + + # Either system-determine RHEL release is not what user wishes to file + # against, or ksc couldn't determine the release; query user to specify + if use_sys_release.lower() == 'n' or not self.releasename: + release_name = query_user( + "Please enter valid RHEL release to file bug against: ", + is_valid=self.valid_release_version + ) + + if not release_name: + print("Unable to determine a valid RHEL release. Qutting.") + sys.exit(1) + + else: + print("Using RHEL %s release." % release_name) + + def submit(self, filename, path): + """ + Submits the resultset into Red Hat bugzilla. + Asks user for Red Hat internal processing consent. + If not set, determines and asks which RHEL release to use. + + :arg filename: Full path the ksc-result.txt file. + :arg path: Path to the config file. + """ + try: + with open(filename, "r") as fptr: + line = fptr.readline().strip() + module_name = self.get_module_name(line) + + except IOError as err: + print("Unable to read previous result: {}".format(err)) + sys.exit(1) + + if not self.mock: # Ask for user permission + self.submit_get_consent() + self.submit_get_release() + + createbug(filename, self.arch, mock=self.mock, path=path, + releasename=self.releasename, module=module_name) + + def get_justification(self, filename): + """ + Get the justification from User + on non-whitelist symbols + + """ + bold = "\033[1m" + reset = "\033[0;0m" + + print(bold) + print('On the next screen, the result log will be opened to allow') + print('you to provide technical justification on why these symbols') + print('need to be included in the KABI whitelist.') + print('Please provide sufficient information in the log, marked with ') + print('the line below:') + + print(("\n%s\n" % self.JUSTIFICATION_PLACEHOLDER) + reset) + print(bold + 'Press ENTER for next screen.' + reset) + try: + input() + except EOFError: + print("Warning. Running in a non-interactive mode.") + + editor = os.getenv('EDITOR') + if editor: + os.system(editor + ' ' + filename) + else: + os.system('vi ' + filename) + return True + + def save_result(self): + """ + Save the result in a text file + """ + output_filename = os.path.expanduser("~/ksc-result.txt") + if os.path.isfile(output_filename): + + overwrite_result_query = "ksc-result.txt already exists. " \ + "Overwrite? [y/N]: " + overwrite = query_user_bool(overwrite_result_query) + + if overwrite.lower() != 'y': + print("Unable to get an explicit overwrite acknowledgement. " + "Quitting.") + sys.exit(1) + + if os.path.isfile(output_filename): + with open(output_filename, 'w+') as f: + f.truncate() + + try: + with open(output_filename, "a") as f: + command = "[command: %s]\n" % " ".join(sys.argv) + + f.write(command) + for ko_file in self.all_symbols_used: + f.write("\n{%s@%s}\n\n" % ( + os.path.basename(ko_file), + self.vermagic[ko_file].strip() + )) + self.write_result(f, ko_file) + + if not self.mock: + print("A copy of the report is saved in %s" % output_filename) + + except Exception as e: + print("Error in saving the result file at %s" % output_filename) + print(e) + sys.exit(1) + + def print_result(self, kmod_path): + """ + Print the result (score) + """ + + print("Processing %s" % kmod_path) + + for name in self.nonwhite_symbols_used[kmod_path]: + if name not in self.total: + print("WARNING: External symbol in %s does not " + "exist in current kernel: %s" \ + % (os.path.basename(kmod_path),name)) + + total_len = len(self.all_symbols_used[kmod_path]) + non_white = len(self.nonwhite_symbols_used[kmod_path]) + white_len = float(len(self.white_symbols[kmod_path])) + + if total_len == 0: # pragma: no cover + print("No kernel symbol usage found in %s." % kmod_path) + return + + score = (white_len / total_len) * 100 + + if not self.mock: + print("Checking against architecture %s" % self.arch) + print("Total symbol usage: %s\t" + "Total Non white list symbol usage: %s" + % (total_len, non_white)) + print("Score: %0.2f%%\n" % score) + + def find_arch(self, kmod_list): + """ + Finds the architecture of the file in given path + """ + rset = {'littleendianIntel80386': 'i686', + 'bigendianPowerPC64': 'ppc64', + 'littleendianPowerPC64': 'ppc64le', + 'littleendianAdvancedMicroDevicesX86-64': 'x86_64', + 'bigendianIBMS/390': 's390x', + 'littleendianAArch64': 'aarch64'} + arch = [] + for kmod_path in kmod_list: + try: + data = run("readelf -h %s | grep -e Data -e Machine | awk -F " + "':' '{print $2}' | paste -d ' ' - - | awk -F ',' " + "'{print $2}' | sed 's/[ \t]*//g'" % kmod_path) + arch.append(rset[data.strip()]) + except IOError as e: + print(e, end=' ') + print(("(Only kernel object files are supported)") + if "No such file" not in str(e) + else "") + sys.exit(1) + except KeyError: + print("%s: Invalid architecture. (only %s are supported)" + % (kmod_path, ', '.join(sorted(rset.values())))) + sys.exit(1) + + arch = list(set(arch)) + if len(arch) > 1: + print("Object files for multiple architectures were provided (%s)." + % ', '.join(sorted(arch))) + sys.exit(1) + + self.arch = arch[0] + + def write_result(self, f, ko_file): + """ + Save the result set in the given file + """ + try: + ko_basename = os.path.basename(ko_file) + + f.write("[%s]\n" % self.arch) + + # Write whitelisted symbols + f.write(self.SECTION_WHITELISTS) + for name in sorted(self.white_symbols[ko_file]): + f.write(name + '\n') + + # Write non-whitelisted symbols as well as their justification + # Justification can be one of: + # - free-form entry + # - reference to a different kernel module section (if exists) + # - justification placeholder later to be specified by hand + f.write(self.SECTION_CO_WHITELISTS) + for name in sorted(self.nonwhite_symbols_used[ko_file]): + + justification="" + if name in self.justified_symbols \ + and ko_basename != self.justified_symbols[name]: + justification=self.JUSTIFICATION_REFERENCE % \ + self.justified_symbols[name] + elif ko_basename in self.justifications and \ + name in self.justifications[ko_basename]: + justification=self.justifications[ko_basename][name] + elif "" in self.justifications and \ + name in self.justifications[ko_basename]: + justification=self.justifications[ko_basename][name] + else: + justification=self.JUSTIFICATION_PLACEHOLDER + self.justified_symbols[name] = os.path.basename(ko_file) + + f.write(self.JUSTIFICATION_SEPARATOR) + f.write(self.JUSTIFICATION_BODY % (name, justification)) + + if self.nonwhite_symbols_used[ko_file]: + f.write(self.JUSTIFICATION_SEPARATOR) + f.write('\n') + except Exception as err: + print(err) + + def read_data(self, arch, releasedir, symvers): + """ + Read both data files + """ + self.matchdata, exists = read_list(arch, releasedir, self.verbose) + self.total = read_total_list(symvers) + return exists + + def parse_ko(self, path, process_whitelists=True): + """ + parse a ko file + """ + if process_whitelists: + self.nonwhite_symbols_used[path] = set() + self.all_symbols_used[path] = set() + self.nonwhite_symbols_used[path] = set() + self.white_symbols[path] = set() + + self.defined_symbols[path] = set() + + try: + self.vermagic[path] = run("modinfo -F vermagic '%s'" % path) + except Exception as e: + print(e) + sys.exit(1) + + try: + out = run("nm '%s'" % path) + except Exception as e: + print(e) + sys.exit(1) + + for line in out.split("\n"): + data = line.split(" ") + if len(data) < 2: + continue + if "U " in line and process_whitelists: + self.find_if(path, data[len(data)-1]) + else: + self.defined_symbols[path].add(data[len(data)-1]) + + def remove_internal_symbols(self): + for i in self.nonwhite_symbols_used: + for j in self.defined_symbols: + if i == j: + continue + self.nonwhite_symbols_used[i] -= self.defined_symbols[j] + + def find_if(self, path, name): + """ + Find if the symbol is in whitelist or not + """ + self.all_symbols_used[path].add(name) + if name in self.matchdata: + self.white_symbols[path].add(name) + else: + self.nonwhite_symbols_used[path].add(name) + + def get_module_name(self, command_line): + try: + match = self.HEADER_RE.match(command_line) + if not match: + return None + commands = match.group("cmd").split() + + # Ignore undefined options in parser instead of throwing error + class IOptParse(OptionParser): + def error(self, msg): + pass + + parser = IOptParse() + parser.add_option("-k", "--ko") + opts, _ = parser.parse_args(commands[0:]) + return opts.ko + except Exception: + return None + + +if __name__ == '__main__': + k = Ksc() + k.main() + sys.exit(0) diff --git a/patch b/patch new file mode 100644 index 0000000..423de5c --- /dev/null +++ b/patch @@ -0,0 +1,66 @@ +RHBZ: #1729039 +Upstream status: RHEL >= 8.0.0 only + +Add modinfo -F vermagic KMOD to the ksc-report, namely the section +identifier. + +Before: {kmod.ko} +Now: {kmod.ko@4.18.0-48.el8.x86_64 SMP mod_unload modversions} + +This will help to determine against which kernel were modules +compiled against. This is a non-intrusive change that does not +alter the existing structure of ksc report and requires only minor +changes to existing ksc parsers/lexers. + +Signed-off-by: Čestmír Kalina +--- + ksc.py | 13 ++++++++++++- + 1 file changed, 12 insertions(+), 1 deletion(-) + +diff --git a/ksc.py b/ksc.py +index 2664d26..57df894 100755 +--- a/ksc.py ++++ b/ksc.py +@@ -63,6 +63,7 @@ class Ksc(object): + self.releasedir = None + self.symvers = None + self.arch = None ++ self.vermagic = {} + if mock: + self.releasename = '7.0' + else: +@@ -77,6 +78,7 @@ class Ksc(object): + self.justifications = {} + self.matchdata = None + self.total = None ++ self.vermagic = {} + + def main(self, mock_options=None): + """ +@@ -401,7 +403,10 @@ class Ksc(object): + + f.write(command) + for ko_file in self.all_symbols_used: +- f.write("\n{%s}\n\n" % os.path.basename(ko_file)) ++ f.write("\n{%s@%s}\n\n" % ( ++ os.path.basename(ko_file), ++ self.vermagic[ko_file].strip() ++ )) + self.write_result(f, ko_file) + + if not self.mock: +@@ -544,6 +549,12 @@ class Ksc(object): + + self.defined_symbols[path] = set() + ++ try: ++ self.vermagic[path] = run("modinfo -F vermagic '%s'" % path) ++ except Exception as e: ++ print(e) ++ sys.exit(1) ++ + try: + out = run("nm '%s'" % path) + except Exception as e: +-- +2.21.0 diff --git a/setup.py b/setup.py new file mode 100644 index 0000000..f1ae800 --- /dev/null +++ b/setup.py @@ -0,0 +1,52 @@ +#!/usr/libexec/platform-python +"""ksc""" +from distutils.core import setup +from distutils.core import Command +from unittest import TextTestRunner, TestLoader +import os + + +class TestCommand(Command): + user_options = [] + + def initialize_options(self): + self._dir = os.getcwd() + + def finalize_options(self): + pass + + def run(self): + ''' + Finds all the tests modules in tests/, and runs them. + ''' + testfiles = ['tests'] + # for t in glob(pjoin(self._dir, 'tests', '*.py')): + # if not t.endswith('__init__.py'): + # testfiles.append('.'.join( + # ['tests', splitext(basename(t))[0]]) + # ) + + tests = TestLoader().loadTestsFromNames(testfiles) + t = TextTestRunner(verbosity=1) + t.run(tests) + + +bugzilla = [] +for x in os.listdir('bugzilla/'): + bugzilla.append('bugzilla/%s' % x) + +setup(name='ksc', + version='1.6', + description="ksc tool", + long_description="Kernel Module Source Checker tool", + cmdclass={'test': TestCommand}, + platforms=["Linux"], + author="Kushal Das, Samikshan Bairagya, Stanislav Kozina, Martin Lacko, Ziqian Sun", + author_email="kdas@redhat.com, sbairagy@redhat.com, skozina@redhat.com, mlacko@redhat.com, zsun@redhat.com", + url="http://redhat.com", + license="http://www.gnu.org/copyleft/gpl.html", + data_files=[("/usr/bin", ['ksc']), + ('/etc', ['data/ksc.conf']), + ('/usr/share/ksc', ['ksc.py', 'utils.py']), + ('/usr/share/ksc/data', ['data/ksc.conf']), + ('/usr/share/ksc/bugzilla', bugzilla)]) diff --git a/tests.py b/tests.py new file mode 100644 index 0000000..e1a0553 --- /dev/null +++ b/tests.py @@ -0,0 +1,70 @@ +import unittest +from utils import * +from ksc import Ksc +from mock import Mock, patch + + +class ReadListTest(unittest.TestCase): + """ + Test reading whitelist + """ + def runTest(self): + data, _ = read_list("x86_64", "kabi-current") + assert len(data) != 0 + + +class ReadTotalListTest(unittest.TestCase): + """ + Test reading all symbol names + """ + def runTest(self): + data = read_total_list() + assert len(data) != 0 + + +class RunCommandTest(unittest.TestCase): + """ + To test our own set function + """ + def runTest(self): + data = run('uname -a') + self.assertTrue(data.startswith('Linux')) + + +class GetConfigTest(unittest.TestCase): + """ + To test our own set function + """ + def runTest(self): + data = getconfig('./data/ksc.conf', True) + assert 'user' in data + assert 'partner' in data + assert 'group' in data + assert 'server' in data + + +class CreateBugTest(unittest.TestCase): + """ + Code to test createbug function + """ + def runTest(self): + bugid = createbug('./data/ksc.conf', 'x86_64', True) # This is mock + + +class ParseKOTest(unittest.TestCase): + """ + Code to test parse_ko + """ + @patch('ksc.run') + def runTest(self, mock_run): + mock_run.return_value = 'U add_disk\nU add_drv\nU call_rcu_bh' + k = Ksc(mock=True) + k.read_data('x86_64', 'kabi-current') + k.parse_ko('./ksc.py') + assert len(k.all_symbols_used) == 2 + assert len(k.nonwhite_symbols_used) == 1 + assert len(k.white_symbols) == 1 + + +if __name__ == '__main__': + unittest.main() diff --git a/utils.py b/utils.py new file mode 100644 index 0000000..f2b2b8c --- /dev/null +++ b/utils.py @@ -0,0 +1,344 @@ +# Copyright 2012,2018 Red Hat Inc. +# Author: Kushal Das + +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation; either version 2 of the License, or +# (at your option) any later version. See +# http://www.gnu.org/copyleft/gpl.html for the full text of the +# license. +# + +""" +Helper functions for ksc. +""" + +import os +import re +import sys +import time +import getpass +import subprocess +import locale + +from bugzilla import Bugzilla, BugzillaError + +# whitelist directory +WHPATH = '/lib/modules' +# Module.symvers directory +SRCPATH = '/usr/src/kernels' + +def query_user(query, max_tries=10, is_valid=lambda x: len(x) > 0): + """ + Queries user for a value. + + :arg query: query string + :arg max_tries: maximal number of times user will be prompted to give a + valid reply (avoid cycling) + :arg is_valid: lambda function that determines if and when user supplied + input is valid + + :return response if valid + :return "" if received max_tries invalid reponses + :return "" if we couldn't read data from stdin + """ + tries_left = max_tries + response = "" + while not is_valid(response): + if tries_left < max_tries: + if response == "": + print("Empty response received. Please try again.") + else: + print("Option `%s' is invalid. Please try again." % response) + + if tries_left == 0: + print("Reached maximum number of invalid responses.") + return "" + + try: + tries_left = tries_left - 1 + response = input(query) + except EOFError: + print("Reached early EOF.") + return "" + + return response + +def query_user_bool(query): + """ + Queries user for a Y/N value + + :arg query: query string + :return response if valid + :return "" if received max_tries invalid reponses + :return "" if we couldn't read data from stdin + """ + check_fx = lambda x: x.lower() in ['y', 'n'] + return query_user(query, is_valid=check_fx) + +def get_release_name(): + if not os.path.isfile('/etc/redhat-release'): + print('This tool needs to run on Red Hat Enterprise Linux') + return None + + with open('/etc/redhat-release', 'r') as fptr: + release = fptr.read().split(' ') + if len(release) <= 6: + print('This tool needs to run on Red Hat Enterprise Linux') + return None + for rel in release: + if re.match("\d.\d+",rel): + return rel + print('This tool needs to run on Red Hat Enterprise Linux') + return None + +def read_list(arch, kabipath, verbose=False): + """ + Reads a whitelist file and returns the symbols + """ + result = [] + fpath = os.path.join(WHPATH, kabipath, "kabi_whitelist_%s" % arch) + if not os.path.isfile(fpath): # pragma: no cover + print("File not found:", fpath) + return [], False + try: + if verbose: # pragma: no cover + print("Reading %s" % fpath) + fptr = open(fpath) + for line in fptr.readlines(): + if line.startswith("["): + continue + result.append(line.strip("\n\t")) + fptr.close() + except IOError as err: # pragma: no cover + print(err) + print("whitelist missing") + + return result, True + + +def read_total_list(symvers=None): + """ + Reads total symbol list and returns the list + """ + if not symvers: + release = os.uname()[2] + symvers = os.path.join(SRCPATH, release, "Module.symvers") + if not os.path.isfile(symvers): # pragma: no cover + print("File not found:", symvers) + print("Do you have current kernel-devel package installed?") + sys.exit(1) + result = [] + try: + with open(symvers, "r") as fptr: + for line in fptr.readlines(): + if line.startswith("["): + continue # pragma: no cover + result.append(line.split()[1]) + except IOError as err: # pragma: no cover + print(err) + print("Missing all symbol list") + return result + + +def run(command): + """ + runs the given command + """ + ret = subprocess.Popen(command, shell=True, stdin=subprocess.PIPE, + stdout=subprocess.PIPE, stderr=subprocess.PIPE, + close_fds=True) + out, err = ret.communicate() + if err: + errs = err.decode(locale.getpreferredencoding()).split(':', 1) + raise IOError(errs[1].strip() if len(errs) > 1 else err) + return out.decode(locale.getpreferredencoding()) + + +def getconfig(path='/etc/ksc.conf', mock=False): + """ + Returns the bugzilla config + """ + result = {} + result['partner'] = '' + + if not os.path.isfile(path): + path = '/etc/ksc.conf' + try: + fptr = open(path) + lines = fptr.readlines() + fptr.close() + for line in lines: + if line.startswith("user="): + result["user"] = line[5:-1] + elif line.startswith("partner="): + result["partner"] = line[8:-1] + elif line.startswith("server="): + result["server"] = line[7:-1] + elif line.startswith("partnergroup="): + result["group"] = line[13:-1] + elif line.startswith("api_key="): + result["api_key"] = line[8:-1] + if 'user' not in result and 'api_key' not in result: + print("Either user name or api_key must be specified in configuration.") + return False + if ('server' not in result or + not result['server'].endswith('xmlrpc.cgi')): + print("Servername is not valid in configuration.") + return False + if not mock: # pragma: no cover + if not result['partner'] or result['partner'] == 'partner-name': + result["partner"] = input("Partner name: ") + if not result['group'] or result['group'] == 'partner-group': + result['group'] = input("Partner group: ") + if "api_key" not in result or not result['api_key'] or result['api_key'] == 'api_key': + print('Current Bugzilla user: %s' % result['user']) + result['password'] = getpass.getpass('Please enter password: ') + else: + print('Using API Key for authentication') + else: + result['password'] = 'mockpassword' + if not result['user']: + print("Error: missing values in configuration file.") + print("Bug not submitted") + sys.exit(1) + except Exception as err: + print("Error reading %s" % path) + sys.exit(1) + return result + + +def createbug(filename, arch, mock=False, path='/etc/ksc.conf', + releasename='7.0', module=None): + """ + Opens a bug in the Bugzilla + """ + + if releasename.startswith('6.'): + bughash = {'product': 'Red Hat Enterprise Linux 6'} + elif releasename.startswith('7.'): + bughash = {'product': 'Red Hat Enterprise Linux 7'} + elif releasename.startswith('8.'): + bughash = {'product': 'Red Hat Enterprise Linux 8'} + else: + print("Invalid releasename: Bug not created") + return + bughash["component"] = 'kernel' + bughash["sub_component"] = 'kabi-whitelists' + bughash["summary"] = "kABI Symbol Usage" + bughash["version"] = releasename + bughash["platform"] = arch + bughash["severity"] = "medium" + bughash["priority"] = "medium" + bughash["description"] = "Creating the bug to attach the symbol " + \ + "usage details." + bughash["qa_contact"] = "kernel-qe@redhat.com" + groups = [] + + if module: + bughash["summary"] += " ({})".format(str(module)) + + # We change the path if only it is mock + if mock: + print("Using local config file data/ksc.conf") + path = './data/ksc.conf' + + try: + conf = getconfig(path, mock=mock) + except Exception as err: + print("Problem in parsing the configuration.") + print(err) + return + + if not conf: + return + if 'group' in conf: + if conf['group'] != 'partner-group': + groups.append(conf['group']) + + groups = list(filter(lambda x: len(x) > 0, groups)) + if not groups: + print("Error: Please specify a non-empty partner-group config " +\ + "option in your ksc.conf config file or in the prompt above. " +\ + "Bug was not filed!") + return + + bughash["groups"] = groups + + if 'api_key' in conf and conf['api_key'] != 'api_key': + bughash["Bugzilla_api_key"] = conf["api_key"] + else: + bughash["Bugzilla_login"] = conf["user"] + bughash["Bugzilla_password"] = conf["password"] + bughash["cf_partner"] = [conf["partner"], ] + + bugid = 0 + try: + if 'api_key' in conf and conf['api_key'] != 'api_key': + bz = Bugzilla( + url=conf['server'], + api_key=conf["api_key"] + ) + else: + bz = Bugzilla( + url=conf['server'], + user=conf["user"], + password=conf["password"] + ) + + if not mock: # pragma: no cover + print("Creating a new bug") + + try: + ret = bz.build_createbug( + product=bughash['product'], + component=bughash['component'], + sub_component=bughash['sub_component'], + summary=bughash['summary'], + version=bughash['version'], + platform=bughash['platform'], + qa_contact=bughash['qa_contact'], + severity=bughash['severity'], + priority=bughash['priority'], + description=bughash['description'], + groups=bughash['groups'] + ) + ret['cf_partner'] = bughash['cf_partner'] + bug = bz.createbug(ret) + + bugid = bug.id + + if not mock: # pragma: no cover + print("Bug URL %s/show_bug.cgi?id=%s" % \ + (conf['server'][:-11], bugid)) + print("Attaching the report") + + dhash = {} + dhash["filename"] = "ksc-result.txt" + dhash["contenttype"] = "text/plain" + desc = "kABI symbol usage." + + for _ in range(3): + with open(filename, "r") as fptr: + attachment_id = bz.attachfile(bugid, fptr, desc, **dhash) + + if not mock: # pragma: no cover + if not attachment_id: + time.sleep(1) + else: + print("Attached successfully as %s on bug %s" % (attachment_id, bugid)) + break + else: + print("Failed to attach symbol usage result") + sys.exit() + + except Exception as err: # pragma: no cover + print("Could not create bug. %s" % err) + if not mock: + sys.exit(1) + except BugzillaError as err: + print("Bug not submitted. %s" % err) + if not mock: + sys.exit(1) + + return bugid