Blame SPECS/gen_updates2.py

Packit Service 10856b
#! /usr/bin/python
Packit Service 10856b
# SPDX-License-Identifier: CC0-1.0
Packit Service 10856b
Packit Service 10856b
import argparse
Packit Service 10856b
import errno
Packit Service 10856b
import io
Packit Service 10856b
import itertools
Packit Service 10856b
import os
Packit Service 10856b
import re
Packit Service 10856b
import shutil
Packit Service 10856b
import struct
Packit Service 10856b
import sys
Packit Service 10856b
import tempfile
Packit Service 10856b
from subprocess import PIPE, Popen, STDOUT
Packit Service 10856b
Packit Service 10856b
# Python 3 shims
Packit Service 10856b
try:
Packit Service 10856b
    from functools import reduce
Packit Service 10856b
except:
Packit Service 10856b
    pass
Packit Service 10856b
try:
Packit Service 10856b
    from itertools import zip_longest as izip_longest
Packit Service 10856b
except:
Packit Service 10856b
    from itertools import izip_longest
Packit Service 10856b
Packit Service 10856b
# revs:
Packit Service 10856b
# [ { "path", "cpuid", "pf", "rev", "date" } ]
Packit Service 10856b
Packit Service 10856b
# artifacts:
Packit Service 10856b
#  * content summary (per-file)
Packit Service 10856b
#   * overlay summary (per-fms/pf)
Packit Service 10856b
#  * changelog (per-file?)
Packit Service 10856b
#  * discrepancies (per-fms/pf)
Packit Service 10856b
Packit Service 10856b
log_level = 0
Packit Service 10856b
print_date = False
Packit Service 10856b
Packit Service 10856b
Packit Service 10856b
def log_status(msg, level=0):
Packit Service 10856b
    global log_level
Packit Service 10856b
Packit Service 10856b
    if log_level >= level:
Packit Service 10856b
        sys.stderr.write(msg + "\n")
Packit Service 10856b
Packit Service 10856b
Packit Service 10856b
def log_info(msg, level=2):
Packit Service 10856b
    global log_level
Packit Service 10856b
Packit Service 10856b
    if log_level >= level:
Packit Service 10856b
        sys.stderr.write("INFO: " + msg + "\n")
Packit Service 10856b
Packit Service 10856b
Packit Service 10856b
def log_warn(msg, level=1):
Packit Service 10856b
    global log_level
Packit Service 10856b
Packit Service 10856b
    if log_level >= level:
Packit Service 10856b
        sys.stderr.write("WARNING: " + msg + "\n")
Packit Service 10856b
Packit Service 10856b
Packit Service 10856b
def log_error(msg, level=-1):
Packit Service 10856b
    global log_level
Packit Service 10856b
Packit Service 10856b
    if log_level >= level:
Packit Service 10856b
        sys.stderr.write("ERROR: " + msg + "\n")
Packit Service 10856b
Packit Service 10856b
Packit Service 10856b
def remove_prefix(text, prefix):
Packit Service 10856b
    if isinstance(prefix, str):
Packit Service 10856b
        prefix = [prefix, ]
Packit Service 10856b
Packit Service 10856b
    for p in prefix:
Packit Service 10856b
        pfx = p if p.endswith(os.sep) else p + os.sep
Packit Service 10856b
        if text.startswith(pfx):
Packit Service 10856b
            return text[len(pfx):]
Packit Service 10856b
Packit Service 10856b
    return text
Packit Service 10856b
Packit Service 10856b
Packit Service 10856b
def file_walk(args, yield_dirs=False):
Packit Service 10856b
    for content in args:
Packit Service 10856b
        if os.path.isdir(content):
Packit Service 10856b
            if yield_dirs:
Packit Service 10856b
                yield ("", content)
Packit Service 10856b
            for root, dirs, files in os.walk(content):
Packit Service 10856b
                if yield_dirs:
Packit Service 10856b
                    for f in dirs:
Packit Service 10856b
                        p = os.path.join(root, f)
Packit Service 10856b
                        yield (remove_prefix(p, content), p)
Packit Service 10856b
                for f in files:
Packit Service 10856b
                    p = os.path.join(root, f)
Packit Service 10856b
                    yield (remove_prefix(p, content), p)
Packit Service 10856b
        elif os.path.exists(content):
Packit Service 10856b
            yield ("", content)
Packit Service 10856b
        else:
Packit Service 10856b
            raise IOError(errno.ENOENT, os.strerror(errno.ENOENT), content)
Packit Service 10856b
Packit Service 10856b
Packit Service 10856b
def cpuid_fname(c):
Packit Service 10856b
    return "%02x-%02x-%02x" % (
Packit Service 10856b
        ((c >> 16) & 0xff0) + ((c >> 8) & 0xf),
Packit Service 10856b
        ((c >> 12) & 0xf0) + ((c >> 4) & 0xf),
Packit Service 10856b
        c & 0xf)
Packit Service 10856b
Packit Service 10856b
Packit Service 10856b
def read_revs_dir(path, src=None, ret=None):
Packit Service 10856b
    if ret is None:
Packit Service 10856b
        ret = []
Packit Service 10856b
Packit Service 10856b
    ucode_re = re.compile('[0-9a-f]{2}-[0-9a-f]{2}-0[0-9a-f]$')
Packit Service 10856b
    ucode_dat_re = re.compile('microcode.*\.dat$')
Packit Service 10856b
Packit Service 10856b
    for rp, ap in file_walk([path, ]):
Packit Service 10856b
        rp_fname = os.path.basename(rp)
Packit Service 10856b
        if not ucode_re.match(rp_fname) and not ucode_dat_re.match(rp_fname):
Packit Service 10856b
            continue
Packit Service 10856b
Packit Service 10856b
        # Text-based format
Packit Service 10856b
        data = None
Packit Service 10856b
        if ucode_dat_re.match(rp_fname):
Packit Service 10856b
            data = io.BytesIO()
Packit Service 10856b
            with open(ap, "r") as f:
Packit Service 10856b
                for line in f:
Packit Service 10856b
                    if line.startswith("/"):
Packit Service 10856b
                        continue
Packit Service 10856b
                    vals = line.split(",")
Packit Service 10856b
                    for val in vals:
Packit Service 10856b
                        val = val.strip()
Packit Service 10856b
                        if not val:
Packit Service 10856b
                            continue
Packit Service 10856b
                        data.write(struct.pack("
Packit Service 10856b
            sz = data.seek(0, os.SEEK_CUR)
Packit Service 10856b
            data.seek(0, os.SEEK_SET)
Packit Service 10856b
        else:
Packit Service 10856b
            sz = os.stat(ap).st_size
Packit Service 10856b
Packit Service 10856b
        try:
Packit Service 10856b
            with data or open(ap, "rb") as f:
Packit Service 10856b
                log_info("Processing %s" % ap)
Packit Service 10856b
                offs = 0
Packit Service 10856b
                while offs < sz:
Packit Service 10856b
                    f.seek(offs, os.SEEK_SET)
Packit Service 10856b
                    hdr = struct.unpack("IiIIIIIIIIII", f.read(48))
Packit Service 10856b
                    ret.append({"path": rp, "src": src or path,
Packit Service 10856b
                                "cpuid": hdr[3], "pf": hdr[6], "rev": hdr[1],
Packit Service 10856b
                                "date": hdr[2], "offs": offs, "cksum": hdr[4],
Packit Service 10856b
                                "data_size": hdr[7], "total_size": hdr[8]})
Packit Service 10856b
Packit Service 10856b
                    if hdr[8] and hdr[8] - hdr[7] > 48:
Packit Service 10856b
                        f.seek(hdr[7], os.SEEK_CUR)
Packit Service 10856b
                        ext_tbl = struct.unpack("IIIII", f.read(20))
Packit Service 10856b
                        log_status("Found %u extended signatures for %s:%#x" %
Packit Service 10856b
                                   (ext_tbl[0], rp, offs), level=1)
Packit Service 10856b
Packit Service 10856b
                        cur_offs = offs + hdr[7] + 48 + 20
Packit Service 10856b
                        ext_sig_cnt = 0
Packit Service 10856b
                        while cur_offs < offs + hdr[8] \
Packit Service 10856b
                                and ext_sig_cnt <= ext_tbl[0]:
Packit Service 10856b
                            ext_sig = struct.unpack("III", f.read(12))
Packit Service 10856b
                            ret.append({"path": rp, "src": src or path,
Packit Service 10856b
                                        "cpuid": ext_sig[0], "pf": ext_sig[1],
Packit Service 10856b
                                        "rev": hdr[1], "date": hdr[2],
Packit Service 10856b
                                        "offs": offs, "ext_offs": cur_offs,
Packit Service 10856b
                                        "cksum": hdr[4],
Packit Service 10856b
                                        "ext_cksum": ext_sig[2],
Packit Service 10856b
                                        "data_size": hdr[7],
Packit Service 10856b
                                        "total_size": hdr[8]})
Packit Service 10856b
                            log_status(("Got ext sig %#x/%#x for " +
Packit Service 10856b
                                        "%s:%#x:%#x/%#x") %
Packit Service 10856b
                                       (ext_sig[0], ext_sig[1], rp, offs,
Packit Service 10856b
                                        hdr[3], hdr[6]), level=2)
Packit Service 10856b
Packit Service 10856b
                            cur_offs += 12
Packit Service 10856b
                            ext_sig_cnt += 1
Packit Service 10856b
Packit Service 10856b
                    offs += hdr[8] or 2048
Packit Service 10856b
        except Exception as e:
Packit Service 10856b
            log_error("a problem occurred while processing %s: %s" % (ap, e),
Packit Service 10856b
                      level=1)
Packit Service 10856b
Packit Service 10856b
    return ret
Packit Service 10856b
Packit Service 10856b
Packit Service 10856b
def read_revs_rpm(path, ret=None):
Packit Service 10856b
    if ret is None:
Packit Service 10856b
        ret = []
Packit Service 10856b
Packit Service 10856b
    dir_tmp = tempfile.mkdtemp()
Packit Service 10856b
Packit Service 10856b
    log_status("Trying to extract files from RPM \"%s\"..." % path,
Packit Service 10856b
               level=1)
Packit Service 10856b
Packit Service 10856b
    rpm2cpio = Popen(args=["rpm2cpio", path], stdout=PIPE, stderr=PIPE,
Packit Service 10856b
                     close_fds=True)
Packit Service 10856b
    cpio = Popen(args=["cpio", "-idmv", "*??-??-??", "*microcode*.dat"],
Packit Service 10856b
                 cwd=dir_tmp, stdin=rpm2cpio.stdout,
Packit Service 10856b
                 stdout=PIPE, stderr=STDOUT)
Packit Service 10856b
    out, cpio_stderr = cpio.communicate()
Packit Service 10856b
    rpm2cpio_out, rpm2cpio_err = rpm2cpio.communicate()
Packit Service 10856b
Packit Service 10856b
    rpm2cpio_ret = rpm2cpio.returncode
Packit Service 10856b
    cpio_ret = cpio.returncode
Packit Service 10856b
Packit Service 10856b
    log_info("rpm2cpio exit code: %d, cpio exit code: %d" %
Packit Service 10856b
             (rpm2cpio_ret, cpio_ret))
Packit Service 10856b
    if rpm2cpio_err:
Packit Service 10856b
        log_info("rpm2cpio stderr:\n%s" % rpm2cpio_err, level=3)
Packit Service 10856b
    if out:
Packit Service 10856b
        log_info("cpio output:\n%s" % out, level=3)
Packit Service 10856b
    if cpio_stderr:
Packit Service 10856b
        log_info("cpio stderr:\n%s" % cpio_stderr, level=3)
Packit Service 10856b
Packit Service 10856b
    if rpm2cpio_ret == 0 and cpio_ret == 0:
Packit Service 10856b
        ret = read_revs_dir(dir_tmp, path)
Packit Service 10856b
Packit Service 10856b
    shutil.rmtree(dir_tmp)
Packit Service 10856b
Packit Service 10856b
    return ret
Packit Service 10856b
Packit Service 10856b
Packit Service 10856b
def read_revs(path, ret=None):
Packit Service 10856b
    if ret is None:
Packit Service 10856b
        ret = []
Packit Service 10856b
    if os.path.isdir(path):
Packit Service 10856b
        return read_revs_dir(path, ret)
Packit Service 10856b
    else:
Packit Service 10856b
        return read_revs_rpm(path, ret)
Packit Service 10856b
Packit Service 10856b
Packit Service 10856b
def gen_mc_map(mc_data, merge=False, merge_path=False):
Packit Service 10856b
    """
Packit Service 10856b
    Converts an array of microcode file information to a map with path/sig/pf
Packit Service 10856b
    as a key.
Packit Service 10856b
Packit Service 10856b
    merge: whether to leave only the newest mc variant in the map or leave all
Packit Service 10856b
           possible variants.
Packit Service 10856b
    """
Packit Service 10856b
    res = dict()
Packit Service 10856b
Packit Service 10856b
    for mc in mc_data:
Packit Service 10856b
        key = (None if merge_path else mc["path"], mc["cpuid"], mc["pf"])
Packit Service 10856b
Packit Service 10856b
        if key not in res:
Packit Service 10856b
            res[key] = dict()
Packit Service 10856b
Packit Service 10856b
        cpuid = mc["cpuid"]
Packit Service 10856b
        cur_pf = mc["pf"]
Packit Service 10856b
        pid = 1
Packit Service 10856b
        while cur_pf > 0:
Packit Service 10856b
            if cur_pf & 1 and not (merge and pid in res[key]
Packit Service 10856b
                                   and res[key][pid]["rev"][0] >= mc["rev"]):
Packit Service 10856b
                if pid not in res[cpuid] or merge:
Packit Service 10856b
                    res[cpuid][pid] = []
Packit Service 10856b
                res[cpuid][pid].append(mc)
Packit Service 10856b
Packit Service 10856b
            cur_pf = cur_pf / 2
Packit Service 10856b
            pid = pid * 2
Packit Service 10856b
Packit Service 10856b
    return res
Packit Service 10856b
Packit Service 10856b
Packit Service 10856b
def gen_fn_map(mc_data, merge=False, merge_path=False):
Packit Service 10856b
    res = dict()
Packit Service 10856b
Packit Service 10856b
    for mc in mc_data:
Packit Service 10856b
        key = (None if merge_path else mc["path"], mc["cpuid"], mc["pf"])
Packit Service 10856b
        if key in res:
Packit Service 10856b
            log_warn("Duplicate path/cpuid/pf: %s/%#x/%#x" % key)
Packit Service 10856b
        else:
Packit Service 10856b
            res[key] = []
Packit Service 10856b
        if merge and len(res[key]):
Packit Service 10856b
            if mc["rev"] > res[key][0]["rev"]:
Packit Service 10856b
                res[key][0] = mc
Packit Service 10856b
        else:
Packit Service 10856b
            res[key].append(mc)
Packit Service 10856b
Packit Service 10856b
    return res
Packit Service 10856b
Packit Service 10856b
Packit Service 10856b
def revcmp(a, b):
Packit Service 10856b
    return b["rev"] - a["rev"]
Packit Service 10856b
Packit Service 10856b
Packit Service 10856b
class ChangeLogEntry:
Packit Service 10856b
    ADDED = 0
Packit Service 10856b
    REMOVED = 1
Packit Service 10856b
    UPDATED = 2
Packit Service 10856b
    DOWNGRADED = 3
Packit Service 10856b
    OTHER = 4
Packit Service 10856b
Packit Service 10856b
Packit Service 10856b
def mc_stripped_path(mc):
Packit Service 10856b
    paths = ("usr/share/microcode_ctl/ucode_with_caveats/intel",
Packit Service 10856b
             "usr/share/microcode_ctl/ucode_with_caveats",
Packit Service 10856b
             "usr/share/microcode_ctl",
Packit Service 10856b
             "lib/firmware",
Packit Service 10856b
             "etc/firmware",
Packit Service 10856b
             )
Packit Service 10856b
Packit Service 10856b
    return remove_prefix(mc["path"], paths)
Packit Service 10856b
Packit Service 10856b
Packit Service 10856b
class mcnm:
Packit Service 10856b
    MCNM_ABBREV = 0
Packit Service 10856b
    MCNM_FAMILIES = 1
Packit Service 10856b
    MCNM_MODELS = 2
Packit Service 10856b
    MCNM_FAMILIES_MODELS = 3
Packit Service 10856b
    MCNM_CODENAME = 4
Packit Service 10856b
Packit Service 10856b
Packit Service 10856b
def get_mc_cnames(mc, cmap, mode=mcnm.MCNM_ABBREV):
Packit Service 10856b
    if not isinstance(mc, dict):
Packit Service 10856b
        mc = mc_from_mc_key(mc)
Packit Service 10856b
    sig = mc["cpuid"]
Packit Service 10856b
    pf = mc["pf"]
Packit Service 10856b
    res = []
Packit Service 10856b
Packit Service 10856b
    if not cmap:
Packit Service 10856b
        return None
Packit Service 10856b
    if sig not in cmap:
Packit Service 10856b
        log_info("No codename information for sig %#x" % sig)
Packit Service 10856b
        return None
Packit Service 10856b
Packit Service 10856b
    cnames = cmap[sig]
Packit Service 10856b
Packit Service 10856b
    if mode in (mcnm.MCNM_FAMILIES, mcnm.MCNM_MODELS,
Packit Service 10856b
                mcnm.MCNM_FAMILIES_MODELS):
Packit Service 10856b
        for c in cnames:
Packit Service 10856b
            if not (pf & c["pf_mask"]):
Packit Service 10856b
                continue
Packit Service 10856b
            for m, f in ((mcnm.MCNM_FAMILIES, "families"),
Packit Service 10856b
                         (mcnm.MCNM_MODELS, "models")):
Packit Service 10856b
                if m & mode == 0:
Packit Service 10856b
                    continue
Packit Service 10856b
                if f not in c or not c[f]:
Packit Service 10856b
                    log_info("No %s for sig %#x in %r" % (f, sig, c))
Packit Service 10856b
                    continue
Packit Service 10856b
Packit Service 10856b
                res.append(c[f])
Packit Service 10856b
Packit Service 10856b
        return ", ".join(res) or None
Packit Service 10856b
Packit Service 10856b
    steppings = dict()
Packit Service 10856b
    suffices = dict()
Packit Service 10856b
    for c in cnames:
Packit Service 10856b
        if pf and not (pf & c["pf_mask"]):
Packit Service 10856b
            continue
Packit Service 10856b
Packit Service 10856b
        if mode == mcnm.MCNM_ABBREV and "abbrev" in c and c["abbrev"]:
Packit Service 10856b
            cname = c["abbrev"]
Packit Service 10856b
        else:
Packit Service 10856b
            cname = c["codename"]
Packit Service 10856b
Packit Service 10856b
        if cname not in suffices:
Packit Service 10856b
            suffices[cname] = set()
Packit Service 10856b
        if "variant" in c and c["variant"]:
Packit Service 10856b
            suffices[cname] |= set(c["variant"])
Packit Service 10856b
Packit Service 10856b
        if cname not in steppings:
Packit Service 10856b
            steppings[cname] = set()
Packit Service 10856b
        if c["stepping"]:
Packit Service 10856b
            steppings[cname] |= set(c["stepping"])
Packit Service 10856b
Packit Service 10856b
    for cname in sorted(steppings.keys()):
Packit Service 10856b
        cname_str = cname
Packit Service 10856b
        if len(suffices[cname]):
Packit Service 10856b
            cname_str += "-" + "/".join(sorted(suffices[cname]))
Packit Service 10856b
        if len(steppings[cname]):
Packit Service 10856b
            cname_str += " " + "/".join(sorted(steppings[cname]))
Packit Service 10856b
        res.append(cname_str)
Packit Service 10856b
Packit Service 10856b
    return ", ".join(res) or None
Packit Service 10856b
Packit Service 10856b
Packit Service 10856b
def mc_from_mc_key(k):
Packit Service 10856b
    return dict(zip(("path", "cpuid", "pf"), k))
Packit Service 10856b
Packit Service 10856b
Packit Service 10856b
def mc_path(mc, pf_sfx=True, midword=None, cmap=None):
Packit Service 10856b
    if not isinstance(mc, dict):
Packit Service 10856b
        mc = mc_from_mc_key(mc)
Packit Service 10856b
    path = mc_stripped_path(mc) if mc["path"] is not None else None
Packit Service 10856b
    cpuid_fn = cpuid_fname(mc["cpuid"])
Packit Service 10856b
    fname = os.path.basename(mc["path"] or cpuid_fn)
Packit Service 10856b
    midword = "" if midword is None else " " + midword
Packit Service 10856b
    cname = get_mc_cnames(mc, cmap)
Packit Service 10856b
    cname_str = " (" + cname + ")" if cname else ""
Packit Service 10856b
Packit Service 10856b
    if pf_sfx:
Packit Service 10856b
        sfx = "/0x%02x" % mc["pf"]
Packit Service 10856b
    else:
Packit Service 10856b
        sfx = ""
Packit Service 10856b
Packit Service 10856b
    if not path or path == os.path.join("intel-ucode", cpuid_fn):
Packit Service 10856b
        return "%s%s%s%s" % (fname, sfx, cname_str, midword)
Packit Service 10856b
    else:
Packit Service 10856b
        return "%s%s%s%s (in %s)" % (cpuid_fn, sfx, cname_str, midword, path)
Packit Service 10856b
Packit Service 10856b
Packit Service 10856b
def gen_changelog_file(old, new):
Packit Service 10856b
    pass
Packit Service 10856b
Packit Service 10856b
Packit Service 10856b
def mc_cmp(old_mc, new_mc):
Packit Service 10856b
    res = []
Packit Service 10856b
Packit Service 10856b
    old_mc_revs = [x["rev"] for x in old_mc]
Packit Service 10856b
    new_mc_revs = [x["rev"] for x in new_mc]
Packit Service 10856b
    common = set(old_mc_revs) & set(new_mc_revs)
Packit Service 10856b
    old_rev_list = [x for x in sorted(old_mc_revs) if x not in common]
Packit Service 10856b
    new_rev_list = [x for x in sorted(new_mc_revs) if x not in common]
Packit Service 10856b
Packit Service 10856b
    if len(old_rev_list) != 1 or len(new_rev_list) != 1:
Packit Service 10856b
        for i in new_mc:
Packit Service 10856b
            if i["rev"] in new_rev_list:
Packit Service 10856b
                res.append((ChangeLogEntry.ADDED, None, i))
Packit Service 10856b
        for i in old_mc:
Packit Service 10856b
            if i["rev"] in old_rev_list:
Packit Service 10856b
                res.append((ChangeLogEntry.REMOVED, i, None))
Packit Service 10856b
    else:
Packit Service 10856b
        for old in old_mc:
Packit Service 10856b
            if old["rev"] == old_rev_list[0]:
Packit Service 10856b
                break
Packit Service 10856b
        for new in new_mc:
Packit Service 10856b
            if new["rev"] == new_rev_list[0]:
Packit Service 10856b
                break
Packit Service 10856b
        if new["rev"] > old["rev"]:
Packit Service 10856b
            res.append((ChangeLogEntry.UPDATED, old, new))
Packit Service 10856b
        elif new["rev"] < old["rev"]:
Packit Service 10856b
            res.append((ChangeLogEntry.DOWNGRADED, old, new))
Packit Service 10856b
Packit Service 10856b
    return res
Packit Service 10856b
Packit Service 10856b
Packit Service 10856b
def gen_changelog(old, new):
Packit Service 10856b
    res = []
Packit Service 10856b
Packit Service 10856b
    old_map = gen_fn_map(old)
Packit Service 10856b
    new_map = gen_fn_map(new)
Packit Service 10856b
Packit Service 10856b
    old_files = set(old_map.keys())
Packit Service 10856b
    new_files = set(new_map.keys())
Packit Service 10856b
Packit Service 10856b
    both = old_files & new_files
Packit Service 10856b
    added = new_files - old_files
Packit Service 10856b
    removed = old_files - new_files
Packit Service 10856b
Packit Service 10856b
    for f in sorted(added):
Packit Service 10856b
        p = mc_path(new_map[f][0])
Packit Service 10856b
        for old_f in sorted(removed):
Packit Service 10856b
            old_p = mc_path(old_map[old_f][0])
Packit Service 10856b
            if p == old_p and f[1] == old_f[1] and f[2] == old_f[2]:
Packit Service 10856b
                log_info("Matched %s (%s and %s)" %
Packit Service 10856b
                         (p, old_map[old_f][0]["path"], new_map[f][0]["path"]))
Packit Service 10856b
                added.remove(f)
Packit Service 10856b
                removed.remove(old_f)
Packit Service 10856b
Packit Service 10856b
                res += mc_cmp(old_map[old_f], new_map[f])
Packit Service 10856b
Packit Service 10856b
    for f in sorted(added):
Packit Service 10856b
        for i in new_map[f]:
Packit Service 10856b
            res.append((ChangeLogEntry.ADDED, None, i))
Packit Service 10856b
    for f in sorted(removed):
Packit Service 10856b
        for i in old_map[f]:
Packit Service 10856b
            res.append((ChangeLogEntry.REMOVED, i, None))
Packit Service 10856b
    for f in sorted(both):
Packit Service 10856b
        res += mc_cmp(old_map[f], new_map[f])
Packit Service 10856b
Packit Service 10856b
    return res
Packit Service 10856b
Packit Service 10856b
Packit Service 10856b
def mc_date(mc):
Packit Service 10856b
    if isinstance(mc, dict):
Packit Service 10856b
        mc = mc["date"]
Packit Service 10856b
    return "%04x-%02x-%02x" % (mc & 0xffff, mc >> 24, (mc >> 16) & 0xff)
Packit Service 10856b
Packit Service 10856b
Packit Service 10856b
def mc_rev(mc, date=None):
Packit Service 10856b
    '''
Packit Service 10856b
    While revision is signed for comparison purposes, historically
Packit Service 10856b
    it is printed as unsigned,  Oh well.
Packit Service 10856b
    '''
Packit Service 10856b
    global print_date
Packit Service 10856b
Packit Service 10856b
    if mc["rev"] < 0:
Packit Service 10856b
        rev = 2**32 + mc["rev"]
Packit Service 10856b
    else:
Packit Service 10856b
        rev = mc["rev"]
Packit Service 10856b
Packit Service 10856b
    if date if date is not None else print_date:
Packit Service 10856b
        return "%#x (%s)" % (rev, mc_date(mc))
Packit Service 10856b
    else:
Packit Service 10856b
        return "%#x" % rev
Packit Service 10856b
Packit Service 10856b
Packit Service 10856b
def print_changelog(clog, cmap, args):
Packit Service 10856b
    for e, old, new in sorted(clog):
Packit Service 10856b
        if e == ChangeLogEntry.ADDED:
Packit Service 10856b
            print("Addition of %s at revision %s" %
Packit Service 10856b
                  (mc_path(new, midword="microcode", cmap=cmap), mc_rev(new)))
Packit Service 10856b
        elif e == ChangeLogEntry.REMOVED:
Packit Service 10856b
            print("Removal of %s at revision %s" %
Packit Service 10856b
                  (mc_path(old, midword="microcode", cmap=cmap), mc_rev(old)))
Packit Service 10856b
        elif e == ChangeLogEntry.UPDATED:
Packit Service 10856b
            print("Update of %s from revision %s up to %s" %
Packit Service 10856b
                  (mc_path(old, midword="microcode", cmap=cmap),
Packit Service 10856b
                   mc_rev(old), mc_rev(new)))
Packit Service 10856b
        elif e == ChangeLogEntry.DOWNGRADED:
Packit Service 10856b
                print("Downgrade of %s from revision %s down to %s" %
Packit Service 10856b
                      (mc_path(old, midword="microcode", cmap=cmap),
Packit Service 10856b
                       mc_rev(old), mc_rev(new)))
Packit Service 10856b
        elif e == ChangeLogEntry.OTHER:
Packit Service 10856b
            print("Other change in %s:" % old["path"])
Packit Service 10856b
            print("  old: %#x/%#x: rev %s (offs %#x)" %
Packit Service 10856b
                  (old["cpuid"], old["pf"], mc_rev(old), old["offs"]))
Packit Service 10856b
            print("  new: %#x/%#x: rev %s (offs %#x)" %
Packit Service 10856b
                  (new["cpuid"], new["pf"], mc_rev(new), new["offs"]))
Packit Service 10856b
Packit Service 10856b
Packit Service 10856b
class TableStyles:
Packit Service 10856b
    TS_CSV = 0
Packit Service 10856b
    TS_FANCY = 1
Packit Service 10856b
Packit Service 10856b
Packit Service 10856b
def print_line(line, column_sz):
Packit Service 10856b
    print(" | ".join([str(x).ljust(column_sz[i])
Packit Service 10856b
                      for i, x in zip(itertools.count(),
Packit Service 10856b
                                      itertools.chain(line,
Packit Service 10856b
                                      [""] * (len(column_sz) -
Packit Service 10856b
                                              len(line))))]).rstrip())
Packit Service 10856b
Packit Service 10856b
Packit Service 10856b
def print_table(items, header=[], style=TableStyles.TS_CSV):
Packit Service 10856b
    if style == TableStyles.TS_CSV:
Packit Service 10856b
        for i in items:
Packit Service 10856b
            print(";".join(i))
Packit Service 10856b
    elif style == TableStyles.TS_FANCY:
Packit Service 10856b
        column_sz = list(reduce(lambda x, y:
Packit Service 10856b
                                map(max, izip_longest(x, y, fillvalue=0)),
Packit Service 10856b
                                [[len(x) for x in i]
Packit Service 10856b
                                 for i in itertools.chain(header, items)]))
Packit Service 10856b
        for i in header:
Packit Service 10856b
            print_line(i, column_sz)
Packit Service 10856b
        if header:
Packit Service 10856b
            print("-+-".join(["-" * x for x in column_sz]))
Packit Service 10856b
        for i in items:
Packit Service 10856b
            print_line(i, column_sz)
Packit Service 10856b
Packit Service 10856b
Packit Service 10856b
def print_summary(revs, cmap, args):
Packit Service 10856b
    m = gen_fn_map(revs)
Packit Service 10856b
    cnames_mode = mcnm.MCNM_ABBREV if args.abbrev else mcnm.MCNM_CODENAME
Packit Service 10856b
Packit Service 10856b
    header = []
Packit Service 10856b
    if args.header:
Packit Service 10856b
        header.append(["Path", "Offset", "Ext. Offset", "CPUID",
Packit Service 10856b
                       "Platform ID Mask", "Revision", "Date", "Checksum",
Packit Service 10856b
                       "Codenames"] +
Packit Service 10856b
                      (["Models"] if args.models else []))
Packit Service 10856b
    tbl = []
Packit Service 10856b
    for k in sorted(m.keys()):
Packit Service 10856b
        for mc in m[k]:
Packit Service 10856b
            tbl.append([mc_stripped_path(mc),
Packit Service 10856b
                        "0x%x" % mc["offs"],
Packit Service 10856b
                        "0x%x" % mc["ext_offs"] if "ext_offs" in mc else "-",
Packit Service 10856b
                        "0x%05x" % mc["cpuid"],
Packit Service 10856b
                        "0x%02x" % mc["pf"],
Packit Service 10856b
                        mc_rev(mc, date=False),
Packit Service 10856b
                        mc_date(mc),
Packit Service 10856b
                        "0x%08x" % mc["cksum"],
Packit Service 10856b
                        get_mc_cnames(mc, cmap, cnames_mode) or ""] +
Packit Service 10856b
                       ([get_mc_cnames(mc, cmap,
Packit Service 10856b
                                       mcnm.MCNM_FAMILIES_MODELS)]
Packit Service 10856b
                        if args.models else []))
Packit Service 10856b
Packit Service 10856b
    print_table(tbl, header, style=TableStyles.TS_FANCY)
Packit Service 10856b
Packit Service 10856b
Packit Service 10856b
def read_codenames_file(path):
Packit Service 10856b
    '''
Packit Service 10856b
    Supports two formats: new and old
Packit Service 10856b
     * old: tab-separated. Field order:
Packit Service 10856b
       Segment, (unused), Codename, (dash-separated) Stepping,
Packit Service 10856b
       Platform ID mask, CPUID, (unused) Update link, (unused) Specs link
Packit Service 10856b
     * new: semicolon-separated; support comments.  Distinguished
Packit Service 10856b
       by the first line that starts with octothorp.  Field order:
Packit Service 10856b
       Segment, Unused, Codename, Stepping, Platform ID mask, CPUID,
Packit Service 10856b
       Abbreviation, Variant(s), Families, Models
Packit Service 10856b
    '''
Packit Service 10856b
    old_fields = ["segment", "_", "codename", "stepping", "pf_mask", "sig",
Packit Service 10856b
                  "_update", "_specs"]
Packit Service 10856b
    new_fields = ["segment", "_", "codename", "stepping", "pf_mask", "sig",
Packit Service 10856b
                  "abbrev", "variant", "families", "models"]
Packit Service 10856b
    new_fmt = False
Packit Service 10856b
    field_names = old_fields
Packit Service 10856b
Packit Service 10856b
    res = dict()
Packit Service 10856b
Packit Service 10856b
    try:
Packit Service 10856b
        with open(path, "r") as f:
Packit Service 10856b
            for line in f:
Packit Service 10856b
                line = line.strip()
Packit Service 10856b
                if len(line) == 0:
Packit Service 10856b
                    continue
Packit Service 10856b
                if line[0] == '#':
Packit Service 10856b
                    new_fmt = True
Packit Service 10856b
                    field_names = new_fields
Packit Service 10856b
                    continue
Packit Service 10856b
Packit Service 10856b
                fields = line.split(";" if new_fmt else "\t",
Packit Service 10856b
                                    1 + len(field_names))
Packit Service 10856b
                fields = dict(zip(field_names, fields))
Packit Service 10856b
                if "sig" not in fields:
Packit Service 10856b
                    log_warn("Skipping %r (from \"%s\")" % (fields, line))
Packit Service 10856b
                    continue
Packit Service 10856b
Packit Service 10856b
                sig = fields["sig"] = int(fields["sig"], 16)
Packit Service 10856b
                fields["pf_mask"] = int(fields["pf_mask"], 16)
Packit Service 10856b
                fields["stepping"] = fields["stepping"].split(",")
Packit Service 10856b
                if "variant" in fields:
Packit Service 10856b
                    if fields["variant"]:
Packit Service 10856b
                        fields["variant"] = fields["variant"].split(",")
Packit Service 10856b
                    else:
Packit Service 10856b
                        fields["variant"] = []
Packit Service 10856b
Packit Service 10856b
                if sig not in res:
Packit Service 10856b
                    res[sig] = list()
Packit Service 10856b
                res[sig].append(fields)
Packit Service 10856b
    except Exception as e:
Packit Service 10856b
        log_error("a problem occurred while reading code names: %s" % e)
Packit Service 10856b
Packit Service 10856b
    return res
Packit Service 10856b
Packit Service 10856b
Packit Service 10856b
def print_discrepancies(rev_map, deps, cmap, args):
Packit Service 10856b
    """
Packit Service 10856b
    rev_map: dict "name": revs
Packit Service 10856b
    deps: list of tuples (name, parent/None)
Packit Service 10856b
    """
Packit Service 10856b
    sigs = set()
Packit Service 10856b
Packit Service 10856b
    for p, r in rev_map.items():
Packit Service 10856b
        sigs |= set(r.keys())
Packit Service 10856b
Packit Service 10856b
    if args.header:
Packit Service 10856b
        header1 = ["sig"]
Packit Service 10856b
        if args.print_vs:
Packit Service 10856b
            header2 = [""]
Packit Service 10856b
        for p, n, d in deps:
Packit Service 10856b
            header1.append(n)
Packit Service 10856b
            if args.print_vs:
Packit Service 10856b
                add = ""
Packit Service 10856b
                if d:
Packit Service 10856b
                    for pd, nd, dd in deps:
Packit Service 10856b
                        if pd == d:
Packit Service 10856b
                            add = "(vs. %s)" % nd
Packit Service 10856b
                            break
Packit Service 10856b
                header2.append(add)
Packit Service 10856b
        if args.models:
Packit Service 10856b
            header1.append("Model names")
Packit Service 10856b
            if args.print_vs:
Packit Service 10856b
                header2.append("")
Packit Service 10856b
    header = [header1] + ([header2] if args.print_vs else [])
Packit Service 10856b
Packit Service 10856b
    tbl = []
Packit Service 10856b
    for s in sorted(sigs):
Packit Service 10856b
        out = [mc_path(s)]
Packit Service 10856b
        print_out = not args.print_filter
Packit Service 10856b
        print_date = args.min_date is None
Packit Service 10856b
Packit Service 10856b
        for p, n, d in deps:
Packit Service 10856b
            cur = dict([(x["rev"], x) for x in rev_map[p][s]]) \
Packit Service 10856b
                  if s in rev_map[p] else []
Packit Service 10856b
            v = "/".join([mc_rev(y) for x, y in sorted(cur.items())]) \
Packit Service 10856b
                if cur else "-"
Packit Service 10856b
            if d is not None:
Packit Service 10856b
                prev = [x["rev"] for x in rev_map[d][s]] if s in rev_map[d] \
Packit Service 10856b
                        else []
Packit Service 10856b
                if [x for x in cur if x not in prev]:
Packit Service 10856b
                    v += " (*)"
Packit Service 10856b
                    print_out = True
Packit Service 10856b
            if args.min_date is not None and s in rev_map[p]:
Packit Service 10856b
                for x in rev_map[p][s]:
Packit Service 10856b
                    print_date |= mc_date(x) > args.min_date
Packit Service 10856b
            out.append(v)
Packit Service 10856b
Packit Service 10856b
        if print_out and print_date:
Packit Service 10856b
            if args.models:
Packit Service 10856b
                out.append(get_mc_cnames(s, cmap) or "")
Packit Service 10856b
            tbl.append(out)
Packit Service 10856b
Packit Service 10856b
    print_table(tbl, header, style=TableStyles.TS_FANCY)
Packit Service 10856b
Packit Service 10856b
Packit Service 10856b
def cmd_summary(args):
Packit Service 10856b
    revs = []
Packit Service 10856b
    for p in args.filelist:
Packit Service 10856b
        revs = read_revs(p, ret=revs)
Packit Service 10856b
Packit Service 10856b
    codenames_map = read_codenames_file(args.codenames)
Packit Service 10856b
Packit Service 10856b
    print_summary(revs, codenames_map, args)
Packit Service 10856b
Packit Service 10856b
    return 0
Packit Service 10856b
Packit Service 10856b
Packit Service 10856b
def cmd_changelog(args):
Packit Service 10856b
    codenames_map = read_codenames_file(args.codenames)
Packit Service 10856b
    base_path = args.filelist[0]
Packit Service 10856b
    upd_path = args.filelist[1]
Packit Service 10856b
Packit Service 10856b
    base = read_revs(base_path)
Packit Service 10856b
    upd = read_revs(upd_path)
Packit Service 10856b
Packit Service 10856b
    print_changelog(gen_changelog(base, upd), codenames_map, args)
Packit Service 10856b
Packit Service 10856b
    return 0
Packit Service 10856b
Packit Service 10856b
Packit Service 10856b
def cmd_discrepancies(args):
Packit Service 10856b
    """
Packit Service 10856b
    filenames:
Packit Service 10856b
     * "<" prefix (possibly multiple times) to refer to a previous entry
Packit Service 10856b
       to compare against
Packit Service 10856b
     * "[name]" prefix is a name reference
Packit Service 10856b
    """
Packit Service 10856b
    codenames_map = read_codenames_file(args.codenames)
Packit Service 10856b
    rev_map = dict()
Packit Service 10856b
    deps = list()
Packit Service 10856b
    cur = -1
Packit Service 10856b
Packit Service 10856b
    for path in args.filelist:
Packit Service 10856b
        orig_path = path
Packit Service 10856b
        name = None
Packit Service 10856b
        cur += 1
Packit Service 10856b
        dep = None
Packit Service 10856b
        while True:
Packit Service 10856b
            if path[0] == '<':
Packit Service 10856b
                path = path[1:]
Packit Service 10856b
                dep = cur - 1 if dep is None else dep - 1
Packit Service 10856b
            elif path[0] == '[' and path.find(']') > 0:
Packit Service 10856b
                pos = path.find(']')
Packit Service 10856b
                name = path[1:pos]
Packit Service 10856b
                path = path[pos + 1:]
Packit Service 10856b
            else:
Packit Service 10856b
                break
Packit Service 10856b
        if name is None:
Packit Service 10856b
            name = path
Packit Service 10856b
        if dep is not None and dep < 0:
Packit Service 10856b
            log_error("Incorrect dep reference for '%s' (points to index %d)" %
Packit Service 10856b
                      (orig_path, dep))
Packit Service 10856b
            return 1
Packit Service 10856b
        deps.append((path, name, deps[dep][0] if dep is not None else None))
Packit Service 10856b
        rev_map[path] = gen_fn_map(read_revs(path), merge=args.merge,
Packit Service 10856b
                                   merge_path=True)
Packit Service 10856b
Packit Service 10856b
    print_discrepancies(rev_map, deps, codenames_map, args)
Packit Service 10856b
Packit Service 10856b
    return 0
Packit Service 10856b
Packit Service 10856b
Packit Service 10856b
def parse_cli():
Packit Service 10856b
    root_parser = argparse.ArgumentParser(prog="gen_updates",
Packit Service 10856b
                                          description="Intel CPU Microcode " +
Packit Service 10856b
                                          "parser")
Packit Service 10856b
    root_parser.add_argument("-C", "--codenames", default='codenames',
Packit Service 10856b
                             help="Code names file")
Packit Service 10856b
    root_parser.add_argument("-v", "--verbose", action="count", default=0,
Packit Service 10856b
                             help="Increase output verbosity")
Packit Service 10856b
Packit Service 10856b
    cmdparsers = root_parser.add_subparsers(title="Commands",
Packit Service 10856b
                                            help="main gen_updates commands")
Packit Service 10856b
Packit Service 10856b
    parser_s = cmdparsers.add_parser("summary",
Packit Service 10856b
                                     help="Generate microcode summary")
Packit Service 10856b
    parser_s.add_argument("-a", "--abbreviate", action="store_const",
Packit Service 10856b
                          dest="abbrev", const=True, default=True,
Packit Service 10856b
                          help="Abbreviate code names")
Packit Service 10856b
    parser_s.add_argument("-A", "--no-abbreviate", action="store_const",
Packit Service 10856b
                          dest="abbrev", const=False,
Packit Service 10856b
                          help="Do not abbreviate code names")
Packit Service 10856b
    parser_s.add_argument("-m", "--print-models", action="store_const",
Packit Service 10856b
                          dest="models", const=True, default=False,
Packit Service 10856b
                          help="Print models")
Packit Service 10856b
    parser_s.add_argument("-M", "--no-print-models",
Packit Service 10856b
                          action="store_const", dest="models",
Packit Service 10856b
                          const=False, help="Do not print models")
Packit Service 10856b
    parser_s.add_argument("-H", "--no-print-header",
Packit Service 10856b
                          action="store_const", dest="header",
Packit Service 10856b
                          const=False, default=True,
Packit Service 10856b
                          help="Do not print hader")
Packit Service 10856b
    parser_s.add_argument("filelist", nargs="*", default=[],
Packit Service 10856b
                          help="List or RPMs/directories to process")
Packit Service 10856b
    parser_s.set_defaults(func=cmd_summary)
Packit Service 10856b
Packit Service 10856b
    parser_c = cmdparsers.add_parser("changelog",
Packit Service 10856b
                                     help="Generate changelog")
Packit Service 10856b
    parser_c.add_argument("filelist", nargs=2,
Packit Service 10856b
                          help="RPMs/directories to compare")
Packit Service 10856b
    parser_c.set_defaults(func=cmd_changelog)
Packit Service 10856b
Packit Service 10856b
    parser_d = cmdparsers.add_parser("discrepancies",
Packit Service 10856b
                                     help="Generate discrepancies")
Packit Service 10856b
    parser_d.add_argument("-s", "--merge-revs", action="store_const",
Packit Service 10856b
                          dest="merge", const=True, default=False,
Packit Service 10856b
                          help="Merge revisions that come" +
Packit Service 10856b
                               " from different files")
Packit Service 10856b
    parser_d.add_argument("-S", "--no-merge-revs", action="store_const",
Packit Service 10856b
                          dest="merge", const=False,
Packit Service 10856b
                          help="Do not Merge revisions that come" +
Packit Service 10856b
                               " from different files")
Packit Service 10856b
    parser_d.add_argument("-v", "--print-vs", action="store_const",
Packit Service 10856b
                          dest="print_vs", const=True, default=False,
Packit Service 10856b
                          help="Print base version ")
Packit Service 10856b
    parser_d.add_argument("-V", "--no-print-vs", action="store_const",
Packit Service 10856b
                          dest="print_vs", const=False,
Packit Service 10856b
                          help="Do not Merge revisions that come" +
Packit Service 10856b
                               " from different files")
Packit Service 10856b
    parser_d.add_argument("-m", "--print-models", action="store_const",
Packit Service 10856b
                          dest="models", const=True, default=True,
Packit Service 10856b
                          help="Print model names")
Packit Service 10856b
    parser_d.add_argument("-M", "--no-print-models", action="store_const",
Packit Service 10856b
                          dest="models", const=False,
Packit Service 10856b
                          help="Do not print model names")
Packit Service 10856b
    parser_d.add_argument("-H", "--no-print-header", action="store_const",
Packit Service 10856b
                          dest="header", const=False, default=True,
Packit Service 10856b
                          help="Do not print hader")
Packit Service 10856b
    parser_d.add_argument("-a", "--print-all-files", action="store_const",
Packit Service 10856b
                          dest="print_filter", const=False, default=True,
Packit Service 10856b
                          help="Print all files")
Packit Service 10856b
    parser_d.add_argument("-c", "--print-changed-files", action="store_const",
Packit Service 10856b
                          dest="print_filter", const=True,
Packit Service 10856b
                          help="Print only changed files")
Packit Service 10856b
    parser_d.add_argument("-d", "--min-date", action="store",
Packit Service 10856b
                          help="Minimum date filter")
Packit Service 10856b
    parser_d.add_argument("filelist", nargs='*',
Packit Service 10856b
                          help="RPMs/directories to compare")
Packit Service 10856b
    parser_d.set_defaults(func=cmd_discrepancies)
Packit Service 10856b
Packit Service 10856b
    args = root_parser.parse_args()
Packit Service 10856b
    if not hasattr(args, "func"):
Packit Service 10856b
        root_parser.print_help()
Packit Service 10856b
        return None
Packit Service 10856b
    return args
Packit Service 10856b
Packit Service 10856b
Packit Service 10856b
def main():
Packit Service 10856b
    args = parse_cli()
Packit Service 10856b
    if args is None:
Packit Service 10856b
        return 1
Packit Service 10856b
Packit Service 10856b
    return args.func(args)
Packit Service 10856b
Packit Service 10856b
Packit Service 10856b
if __name__ == "__main__":
Packit Service 10856b
    sys.exit(main())