diff --git a/README.md b/README.md index 9e55479..c22fdb7 100644 --- a/README.md +++ b/README.md @@ -1,3 +1,4 @@ +[![Build Status](https://travis-ci.org/snapshotmanager/boom.svg?branch=master)](https://travis-ci.org/snapshotmanager/boom) # Boom Boom is a *boot manager* for Linux systems using boot loaders that @@ -58,7 +59,7 @@ parameters. This project is hosted at: - * http://github.com/bmr-cymru/boom + * http://github.com/snapshotmanager/boom For the latest version, to contribute, and for more information, please visit the project pages or join the mailing list. @@ -66,7 +67,7 @@ the project pages or join the mailing list. To clone the current master (development) branch run: ``` -git clone git://github.com/bmr-cymru/boom.git +git clone git://github.com/snapshotmanager/boom.git ``` ## Reporting bugs @@ -87,27 +88,20 @@ the available options and commands. ### Builds and packages Binary packages for Fedora and Red Hat Enterprise Linux are available -from the [copr repository][9]. These builds use the RPM spec file -distributed in the git repository and include all the necessary -library modules, binaries, and configuration files needed to install -and use boom. +from the distribution repositories. -To enable the repository on Fedora, run: +Red Hat Enterprise Linux 7: ``` -# dnf copr enable bmr/boom +# yum -y install lvm2-python-boom ``` -The python2 and python3 versions of boom may be installed by running: +Red Hat Enterprise Linux 8 and Fedora: ``` -# dnf -y install python2-boom python3-boom +# dnf -y install boom-boot ``` -Note that although both python 2 and 3 versions of the library are -provided only one package contains the `boom` binary, depending on -the system default python runtime for that distribution version. - ## The boom command The `boom` command is the main interface to the boom boot manager. @@ -308,6 +302,36 @@ When editing a BootEntry, the `boot_id` will change: this is because the options that define an entry form the entry's identity. The new `boot_id` is written to the terminal on success. +### Boot image cache + +Boom can optionally back up the boot images used by a boom BootEntry +so that the entry can still be used if an operating system update +removes the kernel or initramfs image used. + +To use backup images the boot image cache must be enabled in the +`boom.conf` configuration file and the `--backup` option should +be given to the `boom entry` `create`, `edit`, or `clone` +subcommands. + +When `--backup` is used boom will make a copy of each boot image +used by the new entry by adding a '.boomN' suffix (where `N` is a +number) to the file name. The new BootEntry uses the backup copy +of the image rather than the original file name. + +If the `auto_clean` configuration option is set cache entries are +automatically removed when they are no longer in use by any +BootEntry. + +#### boom cache command + +The `boom cache` command gives information about the paths and +images stored in the boom boot iamge cache. The `boom cache list` +command gives information on cache entries in a tabular report +format similar to other `list` commands. + +Detailed information on selected cache entries is provided by the +`boom cache show` command. + ### Reporting commands The `boom entry list` and `boom host|profile list` commands generate @@ -700,12 +724,11 @@ and [Read the Docs][6]. Installation and user documentation will be added in a future update. [0]: https://systemd.io/BOOT_LOADER_SPECIFICATION - [1]: https://github.com/bmr-cymru/snapshot-boot-docs - [2]: https://github.com/bmr-cymru/boom/issues + [1]: https://github.com/snapshotmanager/snapshot-boot-docs + [2]: https://github.com/snapshotmanager/boom/issues [3]: https://www.redhat.com/mailman/listinfo/dm-devel [4]: https://boom.readthedocs.org/en/latest/index.html# [5]: http://sphinx-doc.org/ [6]: https://www.readthedocs.org/ [7]: https://boom.readthedocs.io/en/latest/boom.html#module-boom.command [8]: https://boom.readthedocs.io/en/latest/boom.html - [9]: https://copr.fedorainfracloud.org/coprs/bmr/boom/ diff --git a/boom.spec b/boom.spec index c8288b5..a5797ed 100644 --- a/boom.spec +++ b/boom.spec @@ -2,13 +2,14 @@ %global sphinx_docs 1 Name: boom -Version: 1.0 +Version: 1.1 Release: 1%{?dist} Summary: %{summary} License: GPLv2 URL: https://github.com/snapshotmanager/boom -Source0: https://github.com/snapshotmanager/boom/archive/%{version}.tar.gz +#Source0: https://github.com/snapshotmanager/boom/archive/%{version}.tar.gz +Source0: boom-%{version}.tar.gz BuildArch: noarch @@ -79,7 +80,7 @@ include this support in both Red Hat Enterprise Linux 7 and Fedora). This package provides integration scripts for grub2 bootloader. %prep -%setup -q -n boom-%{commit} +%setup -q -n boom-%{version} # NOTE: Do not use backup extension - MANIFEST.in is picking them %build @@ -106,6 +107,7 @@ install -m 644 etc/default/boom ${RPM_BUILD_ROOT}/etc/default install -d -m 700 ${RPM_BUILD_ROOT}/boot/boom/profiles install -d -m 700 ${RPM_BUILD_ROOT}/boot/boom/hosts install -d -m 700 ${RPM_BUILD_ROOT}/boot/loader/entries +install -d -m 700 ${RPM_BUILD_ROOT}/boot/boom/cache install -m 644 examples/boom.conf ${RPM_BUILD_ROOT}/boot/boom mkdir -p ${RPM_BUILD_ROOT}/%{_mandir}/man8 @@ -141,6 +143,7 @@ rm doc/conf.py %config(noreplace) /boot/boom/boom.conf %dir /boot/boom/profiles %dir /boot/boom/hosts +%dir /boot/boom/cache %dir /boot/loader/entries %files grub2 @@ -151,6 +154,15 @@ rm doc/conf.py %changelog +* Wed May 13 2020 Bryn M. Reeves = 1.1 +- Bump release + +* Tue May 12 2020 Bryn M. Reeves = 1.1-0.1.beta +- Bump release + +* Mon May 11 2020 Bryn M. Reeves - 1.0-2 +- Include boom/cache directory in package + * Wed Nov 27 2019 Bryn M. Reeves - 1.0-1 - Bump release for boom-1.0 diff --git a/boom/__init__.py b/boom/__init__.py index eb5f776..fa17610 100644 --- a/boom/__init__.py +++ b/boom/__init__.py @@ -35,6 +35,6 @@ from __future__ import print_function from ._boom import * from ._boom import __all__ -__version__ = "1.0" +__version__ = "1.1" # vim: set et ts=4 sw=4 : diff --git a/boom/_boom.py b/boom/_boom.py index 4a84b78..690386e 100644 --- a/boom/_boom.py +++ b/boom/_boom.py @@ -31,6 +31,12 @@ DEFAULT_BOOM_DIR = "boom" #: The root directory for Boom configuration files. DEFAULT_BOOM_PATH = path_join(DEFAULT_BOOT_PATH, DEFAULT_BOOM_DIR) +#: The default directory name for the Boom cache. +DEFAULT_CACHE_DIR = "cache" + +#: The path to the root directory of the Boom cache. +DEFAULT_CACHE_PATH = path_join(DEFAULT_BOOM_PATH, DEFAULT_CACHE_DIR) + #: Configuration file mode BOOT_CONFIG_MODE = 0o644 @@ -82,6 +88,84 @@ FORMAT_KEYS = [ FMT_OS_VERSION, FMT_OS_VERSION_ID ] + +# +# Taken from python3.7/stat.py - for compatibility with py2.7 +# + +_S_IFDIR = 0o040000 # directory +_S_IFCHR = 0o020000 # character device +_S_IFBLK = 0o060000 # block device +_S_IFREG = 0o100000 # regular file +_S_IFIFO = 0o010000 # fifo (named pipe) +_S_IFLNK = 0o120000 # symbolic link +_S_IFSOC = 0o140000 # socket file + +_S_ISUID = 0o4000 # set UID bit +_S_ISGID = 0o2000 # set GID bit +_S_ENFMT = _S_ISGID # file locking enforcement +_S_ISVTX = 0o1000 # sticky bit +_S_IREAD = 0o0400 # Unix V7 synonym for _S_IRUSR +_S_IWRITE = 0o0200 # Unix V7 synonym for _S_IWUSR +_S_IEXEC = 0o0100 # Unix V7 synonym for _S_IXUSR +_S_IRWXU = 0o0700 # mask for owner permissions +_S_IRUSR = 0o0400 # read by owner +_S_IWUSR = 0o0200 # write by owner +_S_IXUSR = 0o0100 # execute by owner +_S_IRWXG = 0o0070 # mask for group permissions +_S_IRGRP = 0o0040 # read by group +_S_IWGRP = 0o0020 # write by group +_S_IXGRP = 0o0010 # execute by group +_S_IRWXO = 0o0007 # mask for others (not in group) permissions +_S_IROTH = 0o0004 # read by others +_S_IWOTH = 0o0002 # write by others +_S_IXOTH = 0o0001 # execute by others + +_filemode_table = ( + ((_S_IFLNK, "l"), + (_S_IFREG, "-"), + (_S_IFBLK, "b"), + (_S_IFDIR, "d"), + (_S_IFCHR, "c"), + (_S_IFIFO, "p")), + + ((_S_IRUSR, "r"),), + ((_S_IWUSR, "w"),), + ((_S_IXUSR | _S_ISUID, "s"), + (_S_ISUID, "S"), + (_S_IXUSR, "x")), + + ((_S_IRGRP, "r"),), + ((_S_IWGRP, "w"),), + ((_S_IXGRP | _S_ISGID, "s"), + (_S_ISGID, "S"), + (_S_IXGRP, "x")), + + ((_S_IROTH, "r"),), + ((_S_IWOTH, "w"),), + ((_S_IXOTH | _S_ISVTX, "t"), + (_S_ISVTX, "T"), + (_S_IXOTH, "x")) +) + + +def boom_filemode(mode): + """Convert a file's mode to a string of the form '-rwxrwxrwx'.""" + perm = [] + for table in _filemode_table: + for bit, char in table: + if mode & bit == bit: + perm.append(char) + break + else: + perm.append("-") + return "".join(perm) + + +# +# Logging +# + BOOM_LOG_DEBUG = logging.DEBUG BOOM_LOG_INFO = logging.INFO BOOM_LOG_WARN = logging.WARNING @@ -106,10 +190,14 @@ BOOM_DEBUG_PROFILE = 1 BOOM_DEBUG_ENTRY = 2 BOOM_DEBUG_REPORT = 4 BOOM_DEBUG_COMMAND = 8 -BOOM_DEBUG_ALL = (BOOM_DEBUG_PROFILE | - BOOM_DEBUG_ENTRY | - BOOM_DEBUG_REPORT | - BOOM_DEBUG_COMMAND) +BOOM_DEBUG_CACHE = 16 +BOOM_DEBUG_ALL = ( + BOOM_DEBUG_PROFILE + | BOOM_DEBUG_ENTRY + | BOOM_DEBUG_REPORT + | BOOM_DEBUG_COMMAND + | BOOM_DEBUG_CACHE +) __debug_mask = 0 @@ -201,6 +289,10 @@ class BoomConfig(object): legacy_format = "grub1" legacy_sync = True + cache_enable = True + cache_auto_clean = True + cache_path = DEFAULT_CACHE_PATH + def __str__(self): """Return a string representation of this ``BoomConfig`` in boom.conf (INI) notation. @@ -213,7 +305,12 @@ class BoomConfig(object): cstr += '[legacy]\n' cstr += 'enable = %s\n' % self.legacy_enable cstr += 'format = %s\n' % self.legacy_format - cstr += 'sync = %s' % self.legacy_sync + cstr += 'sync = %s\n\n' % self.legacy_sync + + cstr += '[cache]\n' + cstr += 'enable = %s\n' % self.cache_enable + cstr += 'auto_clean = %s\n' % self.cache_auto_clean + cstr += 'cache_path = %s\n' % self.cache_path return cstr @@ -221,15 +318,20 @@ class BoomConfig(object): """Return a string representation of this ``BoomConfig`` in BoomConfig initialiser notation. """ - cstr = ('BoomConfig(boot_path="%s",boom_path="%s",' % + cstr = ('BoomConfig(boot_path="%s", boom_path="%s", ' % (self.boot_path, self.boom_path)) - cstr += ('enable_legacy=%s,legacy_format="%s",' % + cstr += ('enable_legacy=%s, legacy_format="%s", ' % (self.legacy_enable, self.legacy_format)) - cstr += 'legacy_sync=%s)' % self.legacy_sync + cstr += 'legacy_sync=%s, ' % self.legacy_sync + cstr += 'cache_enable=%s, ' % self.cache_enable + cstr += 'auto_clean=%s, ' % self.cache_auto_clean + cstr += 'cache_path="%s")' % self.cache_path + return cstr def __init__(self, boot_path=None, boom_path=None, legacy_enable=None, - legacy_format=None, legacy_sync=None): + legacy_format=None, legacy_sync=None, cache_enable=None, + cache_auto_clean=None, cache_path=None): """Initialise a new ``BoomConfig`` object with the supplied configuration values, or defaults for any unset arguments. @@ -238,12 +340,19 @@ class BoomConfig(object): :param legacy_enable: enable legacy bootloader support :param legacy_format: the legacy bootlodaer format to write :param legacy_sync: the legacy sync mode + :param cache_enable: enable boot image cache + :param cache_auto_clean: automatically clean up unused boot + images + :param cache_path: the path to the boot image cache """ self.boot_path = boot_path or self.boot_path self.boom_path = boom_path or self.boom_path self.legacy_enable = legacy_enable or self.legacy_enable self.legacy_format = legacy_format or self.legacy_format self.legacy_sync = legacy_sync or self.legacy_sync + self.cache_enable = cache_enable or self.cache_enable + self.cache_auto_clean = cache_auto_clean or self.cache_auto_clean + self.cache_path = cache_path or self.cache_path __config = BoomConfig() @@ -296,6 +405,15 @@ def get_boom_path(): return __config.boom_path +def get_cache_path(): + """Return the currently configured boot file system path. + + :returns: the path to the /boot file system. + :rtype: str + """ + return __config.cache_path + + def set_boot_path(boot_path): """Sets the location of the boot file system to ``boot_path``. @@ -365,6 +483,38 @@ def set_boom_path(boom_path): __config.boom_path = boom_path set_boom_config_path(__config.boom_path) + cache_path = path_join(boom_path, "cache") + if path_exists(cache_path): + set_cache_path(cache_path) + + +def set_cache_path(cache_path): + """Set the location of the boom image cache directory. + + Set the location of the boom image cache path stored in + the active configuration to ``cache_path``. This defaults to the + 'cache/' sub-directory in the boom configuration directory + ``config.boom_path``: this may be overridden by calling this + function with a different path. + + :param cache_path: the path to the 'cache/' directory containing + cached boot images. + :returns: ``None`` + :raises: ValueError if ``cache_path`` does not exist. + """ + global __config + err_str = "Cache path %s does not exist" % cache_path + if isabs(cache_path) and not path_exists(cache_path): + raise ValueError(err_str) + elif not path_exists(path_join(__config.cache_path, cache_path)): + raise ValueError(err_str) + + if not isabs(cache_path): + cache_path = path_join(__config.cache_path, cache_path) + + __config.cache_path = cache_path + _log_debug("Set cache path to: %s" % cache_path) + def get_boom_config_path(): """Return the currently configured boom configuration file path. @@ -465,10 +615,14 @@ class Selection(object): host_add_opts = None host_del_opts = None + # Cache fields + path = None + img_id = None + #: Selection criteria applying to BootEntry objects entry_attrs = [ "boot_id", "title", "version", "machine_id", "linux", "initrd", "efi", - "options", "devicetree" + "options", "devicetree", "path" ] #: Selection criteria applying to BootParams objects @@ -489,7 +643,12 @@ class Selection(object): "host_add_opts", "host_del_opts", "machine_id" ] - all_attrs = entry_attrs + params_attrs + profile_attrs + host_attrs + #: Cache selection supports a subset of entry_attrs + cache_attrs = ["version", "linux", "initrd", "path", "timestamp", "img_id"] + + all_attrs = ( + entry_attrs + params_attrs + profile_attrs + host_attrs + cache_attrs + ) def __str__(self): """Format this ``Selection`` object as a human readable string. @@ -527,7 +686,8 @@ class Selection(object): os_uname_pattern=None, os_kernel_pattern=None, os_initramfs_pattern=None, host_id=None, host_name=None, host_label=None, host_short_name=None, - host_add_opts=None, host_del_opts=None): + host_add_opts=None, host_del_opts=None, path=None, + timestamp=None, img_id=None): """Initialise a new Selection object. Initialise a new Selection object with the specified selection @@ -559,6 +719,9 @@ class Selection(object): :param host_short_name: The host short name to match :param host_add_opts: Host add options to match :param host_del_opts: Host del options to match + :param path: An cache image path to match + :param timestamp: A cache entry timestamp to match + :param img_id: A cache image identifier to match :returns: A new Selection instance :rtype: Selection """ @@ -588,6 +751,9 @@ class Selection(object): self.host_short_name = host_short_name self.host_add_opts = host_add_opts self.host_del_opts = host_del_opts + self.path = path + self.timestamp = timestamp + self.img_id = img_id @classmethod def from_cmd_args(cls, args): @@ -629,7 +795,7 @@ class Selection(object): os_version_id=args.os_version_id, os_options=args.os_options, os_uname_pattern=args.uname_pattern, - host_id=args.host_profile) + host_id=args.host_id) _log_debug("Initialised %s from arguments" % repr(s)) return s @@ -647,7 +813,7 @@ class Selection(object): return hasattr(self, attr) and getattr(self, attr) is not None def check_valid_selection(self, entry=False, params=False, - profile=False, host=False): + profile=False, host=False, cache=False): """Check a Selection for valid criteria. Check this ``Selection`` object to ensure it contains only @@ -660,6 +826,7 @@ class Selection(object): :param params: ``Selection`` may include BootParams data :param profile: ``Selection`` may include OsProfile data :param host: ``Selection`` may include Host data + :param cache: ``Selection`` may include Cache data :returns: ``None`` on success :rtype: ``NoneType`` :raises: ``ValueError`` if excluded criteria are present @@ -675,6 +842,8 @@ class Selection(object): valid_attrs += self.profile_attrs if host: valid_attrs += self.host_attrs + if cache: + valid_attrs += self.cache_attrs for attr in self.all_attrs: if self.__attr_has_value(attr) and attr not in valid_attrs: @@ -833,7 +1002,7 @@ def load_profiles_for_class(profile_class, profile_type, :returns: None """ profile_files = listdir(profiles_path) - _log_info("Loading %s profiles from %s" % (profile_type, profiles_path)) + _log_debug("Loading %s profiles from %s" % (profile_type, profiles_path)) for pf in profile_files: if not pf.endswith(".%s" % profile_ext): continue @@ -843,6 +1012,8 @@ def load_profiles_for_class(profile_class, profile_type, except Exception as e: _log_warn("Failed to load %s from '%s': %s" % (profile_class.__name__, pf_path, e)) + if get_debug_mask(): + raise e continue @@ -875,8 +1046,10 @@ __all__ = [ # Path configuration 'get_boot_path', 'get_boom_path', + 'get_cache_path', 'set_boot_path', 'set_boom_path', + 'set_cache_path', 'set_boom_config_path', 'get_boom_config_path', @@ -896,6 +1069,7 @@ __all__ = [ 'BOOM_DEBUG_ENTRY', 'BOOM_DEBUG_REPORT', 'BOOM_DEBUG_COMMAND', + 'BOOM_DEBUG_CACHE', 'BOOM_DEBUG_ALL', # Utility routines @@ -904,7 +1078,8 @@ __all__ = [ 'parse_btrfs_subvol', 'find_minimum_sha_prefix', 'min_id_width', - 'load_profiles_for_class' + 'load_profiles_for_class', + 'boom_filemode' ] # vim: set et ts=4 sw=4 diff --git a/boom/bootloader.py b/boom/bootloader.py index d35a717..1ed7157 100644 --- a/boom/bootloader.py +++ b/boom/bootloader.py @@ -148,8 +148,8 @@ KEY_MAP = { #: Default values for optional keys OPTIONAL_KEY_DEFAULTS = { BOOM_ENTRY_GRUB_USERS: "$grub_users", - BOOM_ENTRY_GRUB_ARG: "kernel", - BOOM_ENTRY_GRUB_CLASS: "--unrestricted", + BOOM_ENTRY_GRUB_ARG: "--unrestricted", + BOOM_ENTRY_GRUB_CLASS: "kernel", BOOM_ENTRY_GRUB_ID: None } @@ -352,7 +352,7 @@ def _grub2_get_env(name): _log_error("Could not obtain grub2 environment: %s" % e) return "" - for line in out.splitlines(): + for line in out.decode('utf8').splitlines(): (env_name, value) = line.split('=', 1) if name == env_name: return value.strip() @@ -826,7 +826,7 @@ def load_entries(machine_id=None): drop_entries() - _log_info("Loading boot entries from '%s'" % entries_path) + _log_debug("Loading boot entries from '%s'" % entries_path) for entry in listdir(entries_path): if not entry.endswith(".conf"): continue @@ -840,8 +840,10 @@ def load_entries(machine_id=None): except Exception as e: _log_info("Could not load BootEntry '%s': %s" % (entry_path, e)) + if get_debug_mask(): + raise e - _log_info("Loaded %d entries" % len(_entries)) + _log_debug("Loaded %d entries" % len(_entries)) def write_entries(): @@ -920,7 +922,13 @@ def select_entry(s, be): return False if s.machine_id and be.machine_id != s.machine_id: return False - + if s.linux and be.linux != s.linux: + return False + if s.initrd and be.initrd != s.initrd: + return False + if s.path: + if s.path != be.linux and s.path != be.initrd: + return False if not select_params(s, be.bp): return False @@ -1071,10 +1079,9 @@ class BootEntry(object): key_fmt = '%s%s"%s"' if quote else '%s%s%s' key_fmt += tail - if attr == "options" and expand: - attr_val = getattr(self, "expand_options") - else: - attr_val = getattr(self, attr) + attr_val = getattr(self, attr) + if expand: + attr_val = _expand_vars(attr_val) if bls: key_data = (_transform_key(attr), sep, attr_val) @@ -1501,6 +1508,7 @@ class BootEntry(object): if self.disp_boot_id != match.group(2): _log_info("Entry file name does not match boot_id: %s" % entry_basename) + self.read_only = True self._last_path = entry_file self._unwritten = False diff --git a/boom/cache.py b/boom/cache.py new file mode 100644 index 0000000..302b669 --- /dev/null +++ b/boom/cache.py @@ -0,0 +1,787 @@ +# Copyright (C) 2020 Red Hat, Inc., Bryn M. Reeves +# +# cache.py - Boom boot image cache +# +# This file is part of the boom project. +# +# This copyrighted material is made available to anyone wishing to use, +# modify, copy, or redistribute it subject to the terms and conditions +# of the GNU General Public License v.2. +# +# You should have received a copy of the GNU Lesser General Public License +# along with this program; if not, write to the Free Software Foundation, +# Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA +"""The ``boom.cache`` module defines classes, constants and functions +for maintaining an on-disk cache of kernel, initramfs and auxiliary +images required to load boom-defined boot entries. +""" +from __future__ import print_function + +from boom import * +from boom.bootloader import * + +from hashlib import sha1 +from os import chmod, chown, fdatasync, listdir, stat, unlink +from stat import S_ISREG, ST_MODE, ST_UID, ST_GID, ST_MTIME +from os.path import ( + join as path_join, exists as path_exists, sep as path_sep, + basename, dirname +) +from json import load as json_load, dump as json_dump +from errno import ENOENT +import shutil +import logging + +# Module logging configuration +_log = logging.getLogger(__name__) +_log.set_debug_mask(BOOM_DEBUG_CACHE) + +_log_debug = _log.debug +_log_debug_cache = _log.debug_masked +_log_info = _log.info +_log_warn = _log.warning +_log_error = _log.error + +#: Block size for hashing image files +_hash_size = 1024**2 + +#: The name of the JSON cache index file +_CACHE_INDEX = "cacheindex.json" + +#: The extension used for cached image files +_IMAGE_EXT = ".img" + +# +# Constants for access to cache metadata dictionaries +# + +#: Path timestamp +PATH_TS = "path_ts" +#: Path mode +PATH_MODE = "path_mode" +#: Path user ID +PATH_UID = "path_uid" +#: Path group ID +PATH_GID = "path_gid" +#: Path attribute map +PATH_ATTRS = "path_attrs" + +#: Image timestamp +IMAGE_TS = "image_ts" + +# Cache states +#: Path is cached +CACHE_CACHED = "CACHED" +#: Path is cached and missing from /boot +CACHE_MISSING = "MISSING" +#: Path is cached but image missing or damaged +CACHE_BROKEN = "BROKEN" +#: Path is cached and has been restored to /boot +CACHE_RESTORED = "RESTORED" + +# +# Path names in the boom boot image cache +# +# img_path - The path to an image as specified in a BLS snippet, or an +# argument to cache_path(), relative to the root of the +# /boot file system. E.g. "/vmlinuz-5.0.0". +# +# boot_path - The absolute path to an image relative to the host root +# file system. E.g. "/boot/vmlinuz-5.0.0". +# +# cache_path - The absolute path to a cached boot image relative to the +# host root file system. E.g. +# "/boot/boom/cache/1562375e4d022e814ba39521d0852e490b7c07f8.img" +# + +#: The index of img_path names to lists of image identifiers. +_index = {} + +#: Mapping of img_path to path metadata dictionaries +_paths = {} + +#: Mapping of image identifiers to image metadata. +_images = {} + + +def _make_relative(img_path): + """Convert an image path with a leading path separator into + a relative path fragment suitable for passing to the + os.path.join() function. + + :param img_path: The path to convert. + :returns: The path without any leading path separator. + """ + if img_path[0] == path_sep: + img_path = img_path[1:] + return img_path + + +def _image_path_to_boot(img_path): + """Convert an image path relative to /boot into an absolute + path to the corresponding boot image. + + :param img_path: The path to the image relative to the + /boot file system. + :returns: The absolute path to the image including the + current /boot file system mount point. + :rtype: str + """ + img_path = _make_relative(img_path) + return path_join(get_boot_path(), img_path) + + +def _image_id_to_cache_path(img_id): + """Convert an image path relative to /boot into a path + for the corresponding cache entry. + + :param img_id: The SHA1 digest of the image + :returns: The cache path for the image found at ``img_path``. + :rtype: str + """ + return path_join(get_cache_path(), "%s%s" % (img_id, _IMAGE_EXT)) + + +def _image_id_from_file(img_file): + """Calculate the image identifier (SHA1) for the image file + open at ``img_file``. + + :param img_file: An file-like object open for binary reading. + :returns: The image identifier for ``img_file``. + :rtype: str + """ + digest = sha1() + while True: + hashdata = img_file.read(_hash_size) + if not hashdata: + break + digest.update(hashdata) + return digest.hexdigest() + + +def _image_id_from_path(img_path): + """Calculate the image identifier (SHA1) for the image found + at ``img_path``. + + :param img_path: The absolute path to the on-disk image. + :returns: The image identifier for ``img_path``. + :rtype: str + """ + with open(img_path, "rb") as img_file: + return _image_id_from_file(img_file) + + +def drop_cache(): + """Discard the in-memory cache state. Calling this function has + no effect on the persistent cache state but will free all + in-memory represenatations and clear the cache index. + """ + global _index, _paths, _images + _index = {} + _paths = {} + _images = {} + + +def _load_image_ids(cache_path): + """Read the set of image_id values from the cache directory. + + :returns: A list of image_id values. + """ + ids = [] + for entry in listdir(cache_path): + if not entry.endswith(_IMAGE_EXT): + continue + ids.append(entry.rstrip(_IMAGE_EXT)) + return ids + + +def load_cache(verify=True, digests=False): + """Read the state of the on-disk boot image cache into memory. + """ + global _index, _paths, _images + drop_cache() + + cache_path = get_cache_path() + + index_path = path_join(cache_path, _CACHE_INDEX) + + _log_debug("Loading cache entries from '%s'" % index_path) + + # Get the set of known image_id values + ids = _load_image_ids(cache_path) + + cachedata = {} + try: + with open(index_path, "r") as index_file: + cachedata = json_load(index_file) + except IOError as e: + if e.errno != ENOENT: + raise e + _log_debug("No metadata found: starting empty cache") + return + + index = cachedata["index"] + + for path in index.keys(): + for image_id in index[path]: + if image_id not in ids: + _log_warn("Image identifier '%s' not found in cache" + " for path %s" % (image_id, path)) + # mark as broken + + paths = cachedata["paths"] + for path in paths.keys(): + if path not in index: + _log_warn("No image for path '%s' found in cache" % path) + # mark as broken + + images = cachedata["images"] + for image_id in images.keys(): + if image_id not in ids: + _log_warn("Found unknown image_id '%s'" % image_id) + # clean up? + + _log_debug("Loaded %d cache paths and %d images" % + (len(paths), len(sum(index.values(), [])))) + + _index = index + _paths = paths + _images = images + + +def write_cache(): + """Write the current in-memory state of the cache to disk. + """ + cache_path = get_cache_path() + + index_path = path_join(cache_path, _CACHE_INDEX) + + cachedata = { + "index": _index, + "paths": _paths, + "images": _images + } + + with open(index_path, "w") as index_file: + json_dump(cachedata, index_file) + fdatasync(index_file.fileno()) + + +def _insert_copy(boot_path, cache_path): + """Insert an image into the cache by physical data copy. + """ + shutil.copy2(boot_path, cache_path) + + +def _insert(boot_path, cache_path): + """Insert an image into the cache. + + :param boot_path: The absolute path to the image to add. + :param cache_path: The cache path at which to insert. + :returns: None + """ + try: + # FIXME: implement hard link support with fall-back to copy. + _insert_copy(boot_path, cache_path) + except Exception as e: + _log_error("Error copying '%s' to cache: %s" % (boot_path, e)) + raise e + + +def _remove_boot(boot_path): + """Remove a boom restored boot image from /boot. + """ + boot_dir = dirname(boot_path) + dot_path = _RESTORED_DOT_PATTERN % basename(boot_path) + if not path_exists(path_join(boot_dir, dot_path)): + raise ValueError("'%s' is not boom managed" % boot_path) + unlink(boot_path) + unlink(path_join(boot_dir, dot_path)) + + +def _remove_copy(cache_path): + """Remove an image copy from the cache store. + """ + unlink(cache_path) + + +def _remove(cache_path): + """Remove an image from the cache store. + + :param cache_path: The path to the image to be removed. + :returns: None + """ + if not cache_path.startswith(get_cache_path()): + raise ValueError("'%s' is not a boom cache path" % cache_path) + try: + _remove_copy(cache_path) + except Exception as e: + _log_error("Error removing cache image '%s': %s" % + (cache_path, e)) + raise e + + +def _insert_path(path, img_id, mode, uid, gid, attrs): + """Insert a path into the path map and index dictionaries. + """ + global _paths, _index + + if path not in _paths: + _paths[path] = {} + _paths[path][PATH_MODE] = mode + _paths[path][PATH_UID] = uid + _paths[path][PATH_GID] = gid + _paths[path][PATH_ATTRS] = attrs + + # Add the img_id to the list of images for this path + if path in _index and img_id not in _index[path]: + _index[path].append(img_id) + else: + _index[path] = [img_id] + + +def _cache_path(img_path, update=True, backup=None): + """Add an image to the boom boot image cache. + + :param img_path: The path to the on-disk boot image relative to + the configured /boot directory. + :returns: None + """ + def this_entry(): + """Return a new CacheEntry object representing the newly cached + path. + """ + return CacheEntry(img_path, _paths[img_path], [(img_id, image_ts)]) + + global _index, _paths, _images + + boot_path = _image_path_to_boot(img_path) + st = stat(boot_path) + + if not S_ISREG(st[ST_MODE]): + _log_error("Image at path '%s' is not a regular file." % img_path) + raise ValueError("'%s' is not a regular file" % img_path) + + img_id = _image_id_from_path(boot_path) + cache_path = _image_id_to_cache_path(img_id) + image_ts = st[ST_MTIME] + + img_path = backup or img_path + + # Already present? + if img_path in _paths: + _log_info("Path '%s' already cached." % img_path) + if not update: + return + + if img_path in _paths and img_id in _index[img_path]: + _log_info("Image with img_id=%s already cached for path '%s'" % + (img_id[0:6], img_path)) + return this_entry() + _log_info("Adding new image with img_id=%s for path '%s'" % + (img_id[0:6], img_path)) + + path_mode = st[ST_MODE] + path_uid = st[ST_UID] + path_gid = st[ST_GID] + path_attrs = {} # FIXME xattr support + + # Physically cache the image + _insert_copy(boot_path, cache_path) + + # Set cache entry metadata + _images[img_id] = {IMAGE_TS: image_ts} + _insert_path(img_path, img_id, path_mode, path_uid, path_gid, path_attrs) + write_cache() + + return this_entry() + + +def cache_path(img_path, update=True): + """Add an image to the boom boot image cache. + + :param img_path: The path to the on-disk boot image relative to + the configured /boot directory. + :returns: None + """ + _log_debug_cache("Caching path '%s'" % img_path) + return _cache_path(img_path, update=update) + + +def backup_path(img_path, backup_path): + """Back up an image to the boom boot image cache. + + :param img_path: The path to the on-disk boot image relative to + the configured /boot directory. + :param backup_path: The path where the backup image will be created. + :returns: None + """ + _log_debug_cache("Backing up path '%s' as '%s'" % (img_path, backup_path)) + ce = _cache_path(img_path, backup=backup_path) + ce.restore() + return ce + + +def uncache_path(img_path, force=False): + """Remove paths from the boot image cache. + + Remove ``img_path`` from the boot image cache and discard any + unused images referenced by the cache entry. Images that are + shared with other cached paths are not removed. + + :param img_path: The cached path to remove + """ + global _index, _paths, _images + + if img_path not in _paths: + raise ValueError("Path '%s' is not cached." % img_path) + + boot_path = _image_path_to_boot(img_path) + img_id = _index[img_path][0] + ts = _images[img_id][IMAGE_TS] + + ce = CacheEntry(img_path, _paths[img_path], [(img_id, ts)]) + count = ce.count + + if count and not force: + _log_info("Retaining cache path '%s' used by %d boot entries" % + (img_path, count)) + return + + if count: + _log_warn("Uncaching path '%s' used by %d boot entries" + % (img_path, count)) + + # Remove entry from the path index and metadata + images = _index.pop(img_path) + _paths.pop(img_path) + # Clean up unused images + for img_id in images: + all_images = sum(_index.values(), []) + # Shared image? + if img_id not in all_images: + _images.pop(img_id) + cache_path = _image_id_to_cache_path(img_id) + _remove(cache_path) + if _is_restored(boot_path): + _remove_boot(boot_path) + + write_cache() + + +def clean_cache(): + """Remove unused cache entries. + + Iterate over the set of cache entries and remove any paths + that are not referenced by any BootEntry, and remove all + images that are not referenced by a path. + """ + ces = find_cache_paths() + nr_unused = 0 + for ce in ces: + if not ce.count: + nr_unused += 1 + ce.uncache() + if nr_unused: + _log_info("Removed %d unused cache entries" % nr_unused) + + +#: Boom restored dot file pattern +_RESTORED_DOT_PATTERN = ".%s.boomrestored" + + +def _is_restored(boot_path): + """Return ``True`` if ``boot_path`` was restored by boom, or + ``False`` otherwise. + + :param boot_path: The absolute path to a boot image. + """ + boot_dir = dirname(boot_path) + dot_path = _RESTORED_DOT_PATTERN % basename(boot_path) + return path_exists(path_join(boot_dir, dot_path)) + + +class CacheEntry(object): + """In-memory representation of cached boot image. + """ + #: The image path for this CacheEntry + path = None + + @property + def img_id(self): + return self.images[0][0] + + @property + def disp_img_id(self): + shas = set([img_id for (img_id, ts) in self.images]) + width = find_minimum_sha_prefix(shas, 7) + return self.images[0][0][0:width] + + @property + def mode(self): + """The file system mode for this CacheEntry. + """ + return self._pathdata[PATH_MODE] + + @property + def uid(self): + """The file system uid for this CacheEntry. + """ + return self._pathdata[PATH_UID] + + @property + def gid(self): + """The file system gid for this CacheEntry. + """ + return self._pathdata[PATH_GID] + + @property + def attrs(self): + """The dictionary of extended attrs for this CacheEntry. + """ + return self._pathdata[PATH_ATTRS] + + @property + def timestamp(self): + """The timestamp of the most recent image for this CacheEntry. + """ + return self.images[0][1] + + @property + def state(self): + """Return a string representing the state of this cache entry. + """ + boot_path = _image_path_to_boot(self.path) + cache_path = _image_id_to_cache_path(self.images[0][0]) + boot_exists = path_exists(boot_path) + cache_exists = path_exists(cache_path) + if boot_exists and cache_exists: + boot_path_id = _image_id_from_path(boot_path) + if _is_restored(boot_path) and self.img_id == boot_path_id: + return CACHE_RESTORED + else: + return CACHE_CACHED + if cache_exists and not boot_exists: + return CACHE_MISSING + if boot_exists and not cache_exists: + return CACHE_BROKEN + return CACHE_UNKNOWN + + @property + def count(self): + """Return the current number of boot entries that reference + this cache entry. + """ + entries = find_entries(Selection(path=self.path)) + # Ignore foreign boot entries + entries = [entry for entry in entries if not entry.read_only] + return len(entries) + + #: The list of cached images for this CacheEntry sorted by increasing age + images = [] + + def __init__(self, path, pathdata, images): + """Initialise a CacheEntry object with information from + the on-disk cache. + """ + self.path = path + self._pathdata = pathdata + self.images = images + + def __str__(self): + fmt = "Path: %s\nImage ID: %s\nMode: %s\nUid: %d Gid: %d\nTs: %d" + return fmt % ( + self.path, + self.disp_img_id, + boom_filemode(self.mode), + self.uid, self.gid, + self.timestamp + ) + + def __repr__(self): + shas = set([img_id for (img_id, ts) in self.images]) + width = find_minimum_sha_prefix(shas, 7) + rep = '"%s", %s, %s' % ( + self.path, + self._pathdata, + # FIXME: properly generate minimum abrevs + [(img_id[0:width - 1], ts) for (img_id, ts) in self.images] + ) + return "CacheEntry(" + rep + ")" + + def restore(self, dest=None): + """Restore this CacheEntry to the /boot file system. + """ + img_id = self.images[0][0] + if dest: + if dest not in _index: + _insert_path(dest, img_id, self.mode, self.uid, self.gid, + self.attrs) + self.path = dest + write_cache() + boot_path = _image_path_to_boot(self.path) + cache_path = _image_id_to_cache_path(img_id) + dot_path = _RESTORED_DOT_PATTERN % basename(boot_path) + boot_dir = dirname(boot_path) + + if self.state not in (CACHE_MISSING, CACHE_RESTORED): + raise ValueError("Restore failed: CacheEntry state is not " + "%s or %s" % (CACHE_MISSING, CACHE_RESTORED)) + + shutil.copy2(cache_path, boot_path) + try: + chown(boot_path, self.uid, self.gid) + chmod(boot_path, self.mode) + except OSError as e: + try: + unlink(boot_path) + except OSError: + pass + raise e + + try: + dot_file = open(path_join(boot_dir, dot_path), "w") + dot_file.close() + except OSError as e: + try: + unlink(boot_path) + except OSError: + pass + raise e + + def purge(self): + """Remove the boom restored image copy from the /boot file system. + """ + boot_path = _image_path_to_boot(self.path) + if self.state is not CACHE_RESTORED: + raise ValueError("Purge failed: CacheEntry state is not RESTORED") + _remove_boot(boot_path) + + def uncache(self): + """Remove this CacheEntry from the boot image cache. + """ + uncache_path(self.path) + + +def select_cache_entry(s, ce): + """Test CacheEntry against Selection criteria. + + Test the supplied ``CacheEntry`` against the selection criteria + in ``s`` and return ``True`` if it passes, or ``False`` + otherwise. + + :param s: The selection criteria + :param be: The CacheEntry to test + :rtype: bool + :returns: True if CacheEntry passes selection or ``False`` + otherwise. + """ + # Version matches if version string is contained in image path. + if s.version and s.version not in ce.path: + return False + + # Image path match is an exact match. + if s.linux and s.linux != ce.path: + return False + if s.initrd and s.initrd != ce.path: + return False + if s.path and s.path != ce.path: + return False + if s.timestamp and s.timestamp != ce.timestamp: + return False + if s.img_id and s.img_id not in ce.img_id: + return False + return True + + +def _find_cache_entries(selection=None, images=False): + """Find cache entries matching selection criteria. + + Return a list of ``CacheEntry`` objects matching the supplied + ``selection`` criteria. If ``images`` is ``True`` a separate + entry is returned for each image in the cache: otherwise one + ``CacheEntry`` object is returned for each cached path. + + ;param selection: cache entry selection criteria. + :param images: return results by images instead of paths. + :returns: A list of matching ``CacheEntry`` objects. + """ + global _index, _paths, _images + matches = [] + + if not _index: + load_cache() + + # Use null search criteria if unspecified + selection = selection if selection else Selection() + + selection.check_valid_selection(cache=True) + + _log_debug_cache("Finding cache entries for %s" % repr(selection)) + + for path in _index: + def ts_key(val): + return val[1] + + def tuplicate(img_id): + """Return a (img_id, image_ts) tuple for the given img_id. + """ + return (img_id, _images[img_id][IMAGE_TS]) + + entry_images = [tuplicate(img_id) for img_id in _index[path]] + # Sort images list from newest to oldest + entry_images = sorted(entry_images, reverse=True, key=ts_key) + if not images: + ce = CacheEntry(path, _paths[path], entry_images) + if select_cache_entry(selection, ce): + matches.append(ce) + else: + for img_tuple in entry_images: + ce = CacheEntry(path, _paths[path], [img_tuple]) + if select_cache_entry(selection, ce): + matches.append(ce) + + return matches + + +def find_cache_paths(selection=None): + """Find cache entries matching selection criteria. + + Return a list of ``CacheEntry`` objects matching the supplied + ``selection`` criteria, one for each path that exists in the + cache. For each cached path a ``CacheEntry`` object is returned + with a list of images that are cached for that path. The image + list is sorted by timestamp with the most recent entry first. + + ;param selection: cache entry selection criteria. + :returns: A list of matching ``CacheEntry`` objects. + """ + matches = _find_cache_entries(selection=selection, images=False) + _log_debug_cache("Found %d cached paths" % len(matches)) + return matches + + +def find_cache_images(selection=None): + """Find cache entries matching selection criteria. + + Return a list of ``CacheEntry`` objects matching the supplied + ``selection`` criteria, one for each image that exists in the + cache. Each ``CacheEntry`` object is returned with a list + containing a single image. + + ;param selection: cache entry selection criteria. + :returns: A list of matching ``CacheEntry`` objects. + """ + matches = _find_cache_entries(selection=selection, images=True) + _log_debug_cache("Found %d cached images" % len(matches)) + return matches + + +__all__ = [ + "CACHE_CACHED", "CACHE_MISSING", "CACHE_BROKEN", "CACHE_RESTORED", + "drop_cache", "load_cache", "write_cache", + "cache_path", "backup_path", "uncache_path", "clean_cache", + "find_cache_paths", "find_cache_images" +] + +# vim: set et ts=4 sw=4 : diff --git a/boom/command.py b/boom/command.py index a113c21..dc9abae 100644 --- a/boom/command.py +++ b/boom/command.py @@ -33,9 +33,10 @@ from boom.bootloader import * from boom.hostprofile import * from boom.legacy import * from boom.config import * +from boom.cache import * from os import environ, uname, getcwd -from os.path import basename, exists as path_exists, isabs, join +from os.path import basename, exists as path_exists, isabs, join, sep from argparse import ArgumentParser import platform import logging @@ -83,8 +84,10 @@ class BoomReportObj(object): be = None osp = None hp = None + ce = None - def __init__(self, boot_entry=None, os_profile=None, host_profile=None): + def __init__(self, boot_entry=None, os_profile=None, host_profile=None, + cache_entry=None): """Initialise new BoomReportObj objects. Construct a new BoomReportObj object containing the @@ -96,6 +99,7 @@ class BoomReportObj(object): self.be = boot_entry self.osp = os_profile self.hp = host_profile + self.ce = cache_entry #: BootEntry report object type @@ -106,6 +110,8 @@ BR_PROFILE = 2 BR_PARAMS = 4 #: HostProfile report object type BR_HOST = 8 +#: CacheEntry report object type +BR_CACHE = 16 #: Report object type table for ``boom.command`` reports. _report_obj_types = [ @@ -116,7 +122,9 @@ _report_obj_types = [ BoomReportObjType( BR_PARAMS, "Boot parameters", "param_", lambda o: o.be.bp), BoomReportObjType( - BR_HOST, "Host profiles", "host_", lambda o: o.hp) + BR_HOST, "Host profiles", "host_", lambda o: o.hp), + BoomReportObjType( + BR_CACHE, "Cache entries", "cache_", lambda o: o.ce) ] # @@ -317,6 +325,38 @@ _default_entry_fields = "bootid,version,osname,rootdev" _verbose_entry_fields = (_default_entry_fields + ",options,machineid") +#: Fields derived from CacheEntry data +_cache_fields = [ + BoomFieldType( + BR_CACHE, "imgid", "ImageID", "Image identifier", 7, + REP_SHA, lambda f, d: f.report_sha(d.img_id)), + BoomFieldType( + BR_CACHE, "path", "Path", "Image path", 24, + REP_STR, lambda f, d: f.report_str(d.path)), + BoomFieldType( + BR_CACHE, "mode", "Mode", "Path mode", 8, + REP_STR, lambda f, d: f.report_str(boom_filemode(d.mode))), + BoomFieldType( + BR_CACHE, "uid", "User", "User ID", 5, + REP_NUM, lambda f, d: f.report_num(d.uid)), + BoomFieldType( + BR_CACHE, "gid", "Group", "Group ID", 5, + REP_NUM, lambda f, d: f.report_num(d.gid)), + BoomFieldType( + BR_CACHE, "ts", "Timestamp", "Timestamp", 10, + REP_NUM, lambda f, d: f.report_num(d.timestamp)), + BoomFieldType( + BR_CACHE, "state", "State", "State", 8, + REP_STR, lambda f, d: f.report_str(d.state)), + BoomFieldType( + BR_CACHE, "count", "Count", "Use Count", 5, + REP_NUM, lambda f, d: f.report_num(d.count)) +] + +_default_cache_fields = "path,imgid,ts,state" +_verbose_cache_fields = "path,imgid,ts,mode,uid,gid,state,count" + + def _get_machine_id(): """Return the current host's machine-id. @@ -466,14 +506,81 @@ def _merge_add_del_opts(orig_opts, opts): # Command driven API: BootEntry and OsProfile management and reporting. # +# Boot image cache modes + +#: Use original image (no caching) +I_NONE = None +I_CACHE = "cache" +I_BACKUP = "backup" + + # # BootEntry manipulation # +def _find_backup_name(img_path): + """Generate a new, unique backup pathname. + """ + img_backup = ("%s.boom" % img_path)[1:] + "%d" + + def _backup_img(backup_nr): + return sep + img_backup % backup_nr + + def _backup_path(backup_nr): + return join(get_boot_path(), img_backup[1:] % backup_nr) + + backup_nr = 0 + while path_exists(_backup_path(backup_nr)): + if find_cache_paths(Selection(path=_backup_img(backup_nr))): + break + backup_nr += 1 + return sep + img_backup % backup_nr + + +def _cache_image(img_path, backup): + """Cache the image found at ``img_path`` and optionally create + a backup copy. + """ + if "." in img_path: + ext = img_path.rsplit(".", 1)[1] + if ext.startswith("boom") and ext[4:].isdigit(): + if find_cache_paths(Selection(path=img_path)): + return img_path + try: + if backup: + img_backup = _find_backup_name(img_path) + _log_debug("backing up '%s' as '%s'" % (img_path, img_backup)) + ce = backup_path(img_path, img_backup) + return img_backup + else: + ce = cache_path(img_path) + except (OSError, ValueError) as e: + _log_error("Could not cache path %s: %s" % (img_path, e)) + raise e + return img_path + + +def _find_one_entry(select): + """Find exactly one entry, and raise ValueError if zero or more + than one entry is found. + + :param: An instance of ``Selection`` specificying match criteria. + :returns: A single instance of ``BootEntry`` + :raises: ValueError if selection results are empty or non-unique + """ + bes = find_entries(select) + if not bes: + raise ValueError("No matching entry found for boot ID %s" % + select.boot_id) + if len(bes) > 1: + raise ValueError("Selection criteria must match exactly one entry") + return bes[0] + + def create_entry(title, version, machine_id, root_device, lvm_root_lv=None, btrfs_subvol_path=None, btrfs_subvol_id=None, profile=None, add_opts=None, del_opts=None, write=True, architecture=None, - expand=False, allow_no_dev=False): + expand=False, allow_no_dev=False, images=I_NONE): """Create new boot loader entry. Create the specified boot entry in the configured loader directory. @@ -494,6 +601,8 @@ def create_entry(title, version, machine_id, root_device, lvm_root_lv=None, :param architecture: An optional BLS architecture string. :param expand: Expand bootloader environment variables. :param allow_no_dev: Accept a non-existent or invalid root dev. + :param images: Whether to cache or backup boot images in the new + entry. :returns: a ``BootEntry`` object corresponding to the new entry. :rtype: ``BootEntry`` :raises: ``ValueError`` if either required values are missing or @@ -515,6 +624,11 @@ def create_entry(title, version, machine_id, root_device, lvm_root_lv=None, if not profile: raise ValueError("Cannot create entry without OsProfile.") + bc = get_boom_config() + if images is not I_NONE and not bc.cache_enable: + raise BoomConfigError("Cannot use images=%s with image cache disabled" + " (config.cache_enable=False)" % images) + add_opts = add_opts.split() if add_opts else [] del_opts = del_opts.split() if del_opts else [] @@ -530,6 +644,10 @@ def create_entry(title, version, machine_id, root_device, lvm_root_lv=None, osprofile=profile, boot_params=bp, architecture=architecture, allow_no_dev=allow_no_dev) + if images in (I_BACKUP, I_CACHE): + be.initrd = _cache_image(be.initrd, images == I_BACKUP) + be.linux = _cache_image(be.linux, images == I_BACKUP) + if find_entries(Selection(boot_id=be.boot_id)): raise ValueError("Entry already exists (boot_id=%s)." % be.disp_boot_id) @@ -579,7 +697,7 @@ def clone_entry(selection=None, title=None, version=None, machine_id=None, root_device=None, lvm_root_lv=None, btrfs_subvol_path=None, btrfs_subvol_id=None, profile=None, architecture=None, add_opts=None, del_opts=None, - write=True, expand=False, allow_no_dev=False): + write=True, expand=False, allow_no_dev=False, images=I_NONE): """Clone an existing boot loader entry. Create the specified boot entry in the configured loader directory @@ -621,15 +739,12 @@ def clone_entry(selection=None, title=None, version=None, machine_id=None, "machine_id, root_device, lvm_root_lv, " "btrfs_subvol_path, btrfs_subvol_id, profile") - bes = find_entries(selection=selection) - if not bes: - raise ValueError("No matching entry found for boot ID %s" % - selection.boot_id) + bc = get_boom_config() + if images is not I_NONE and not bc.cache_enable: + raise BoomConfigError("Cannot use images=%s with image cache disabled" + " (config.cache_enable=False)" % images) - if len(bes) > 1: - raise ValueError("clone criteria must match exactly one entry") - - be = bes[0] + be = _find_one_entry(selection) _log_debug("Cloning entry with boot_id='%s'" % be.disp_boot_id) @@ -658,14 +773,32 @@ def clone_entry(selection=None, title=None, version=None, machine_id=None, osprofile=profile, boot_params=bp, architecture=architecture, allow_no_dev=allow_no_dev) - if find_entries(Selection(boot_id=clone_be.boot_id)): - raise ValueError("Entry already exists (boot_id=%s)." % - clone_be.disp_boot_id) orig_be = find_entries(selection)[0] if orig_be.options != orig_be.expand_options: clone_be.options = orig_be.options + # Clone optional keys allowed by profile + for optional_key in orig_be._osp.optional_keys.split(): + if optional_key in clone_be._osp.optional_keys: + if hasattr(orig_be, optional_key): + setattr(clone_be, optional_key, + getattr(orig_be, optional_key)) + + # Boot image overrides? + if orig_be.initrd != clone_be.initrd: + clone_be.initrd = orig_be.initrd + if orig_be.linux != clone_be.linux: + clone_be.linux = orig_be.linux + + if images in (I_BACKUP, I_CACHE): + clone_be.initrd = _cache_image(clone_be.initrd, images == I_BACKUP) + clone_be.linux = _cache_image(clone_be.linux, images == I_BACKUP) + + if find_entries(Selection(boot_id=clone_be.boot_id)): + raise ValueError("Entry already exists (boot_id=%s)." % + clone_be.disp_boot_id) + if write: clone_be.write_entry(expand=expand) __write_legacy() @@ -676,7 +809,7 @@ def clone_entry(selection=None, title=None, version=None, machine_id=None, def edit_entry(selection=None, title=None, version=None, machine_id=None, root_device=None, lvm_root_lv=None, btrfs_subvol_path=None, btrfs_subvol_id=None, profile=None, architecture=None, - add_opts=None, del_opts=None, expand=False): + add_opts=None, del_opts=None, expand=False, images=I_NONE): """Edit an existing boot loader entry. Modify an existing BootEntry by changing one or more of the @@ -714,19 +847,15 @@ def edit_entry(selection=None, title=None, version=None, machine_id=None, "machine_id, root_device, lvm_root_lv, " "btrfs_subvol_path, btrfs_subvol_id, profile") + bc = get_boom_config() + if images is not I_NONE and not bc.cache_enable: + raise BoomConfigError("Cannot use images=%s with image cache disabled" + " (config.cache_enable=False)" % images) + # Discard all selection criteria but boot_id. selection = Selection(boot_id=selection.boot_id) - bes = find_entries(selection=selection) - - if not bes: - raise ValueError("No matching entry found for boot ID %s" % - selection.boot_id) - - if len(bes) > 1: - raise ValueError("edit criteria must match exactly one entry") - - be = bes[0] + be = _find_one_entry(selection) _log_debug("Editing entry with boot_id='%s'" % be.disp_boot_id) @@ -752,6 +881,10 @@ def edit_entry(selection=None, title=None, version=None, machine_id=None, be.bp.add_opts = add_opts be.bp.del_opts = del_opts + if images in (I_BACKUP, I_CACHE): + be.initrd = _cache_image(be.initrd, images == I_BACKUP) + be.linux = _cache_image(be.linux, images == I_BACKUP) + be.update_entry(expand=expand) __write_legacy() @@ -812,7 +945,7 @@ def print_entries(selection=None, output_fields=None, opts=None, output_fields = _expand_fields(_default_entry_fields, output_fields) bes = find_entries(selection=selection) - selected = [BoomReportObj(be, be._osp, None) for be in bes] + selected = [BoomReportObj(be, be._osp, None, None) for be in bes] entry_fields = _expand_entry_fields if expand else _entry_fields report_fields = entry_fields + _profile_fields + _params_fields @@ -901,12 +1034,21 @@ def _uname_heuristic(name, version_id): :returns: ``True`` if uname pattern heuristics should be used for this OS or ``False`` otherwise. """ + el_uname = "el" + fc_uname = "fc" _name_to_uname = { - "Red Hat Enterprise Server": "el", - "Red Hat Enterprise Workstation": "el", - "Fedora": "fc" + "Red Hat Enterprise Linux": el_uname, + "Red Hat Enterprise Linux Server": el_uname, + "Red Hat Enterprise Linux Workstation": el_uname, + "Fedora": fc_uname } + # Strip trailing minor version ident from elX_Y + if "_" in version_id: + version_id = version_id[0:version_id.find("_")] + if "." in version_id: + version_id = version_id[0:version_id.find(".")] + if name in _name_to_uname: return "%s%s" % (_name_to_uname[name], version_id) return None @@ -1276,7 +1418,7 @@ def print_profiles(selection=None, opts=None, output_fields=None, output_fields = _expand_fields(_default_profile_fields, output_fields) osps = find_profiles(selection=selection) - selected = [BoomReportObj(None, osp, None) for osp in osps] + selected = [BoomReportObj(None, osp, None, None) for osp in osps] report_fields = _profile_fields return _do_print_type(report_fields, selected, output_fields=output_fields, @@ -1582,12 +1724,78 @@ def print_hosts(selection=None, opts=None, output_fields=None, output_fields = _expand_fields(_default_host_fields, output_fields) hps = find_host_profiles(selection=selection) - selected = [BoomReportObj(None, None, hp) for hp in hps] + selected = [BoomReportObj(None, None, hp, None) for hp in hps] report_fields = _host_fields return _do_print_type(report_fields, selected, output_fields=output_fields, opts=opts, sort_keys=sort_keys) +def _print_cache(find_fn, selection=None, opts=None, output_fields=None, + sort_keys=None, expand=False): + """Print cache entries (with or without images) matching selection + criteria. + + Selection criteria may be expressed via a ``Selection`` object + passed to the call using the ``Selection`` parameter. + + :param selection: A Selection object giving selection + criteria for the operation + :param output_fields: a comma-separated list of output fields + :param opts: output formatting and control options + :param sort_keys: a comma-separated list of sort keys + :param expand: unused + :returns: the number of matching profiles output + :rtype: int + """ + output_fields = _expand_fields(_default_cache_fields, output_fields) + + ces = find_fn(selection=selection) + selected = [BoomReportObj(None, None, None, ce) for ce in ces] + report_fields = _cache_fields + return _do_print_type(report_fields, selected, output_fields=output_fields, + opts=opts, sort_keys=sort_keys) + + +def print_cache(selection=None, opts=None, output_fields=None, + sort_keys=None, expand=False): + """Print cache entries matching selection criteria. + + Selection criteria may be expressed via a ``Selection`` object + passed to the call using the ``Selection`` parameter. + + :param selection: A Selection object giving selection + criteria for the operation + :param output_fields: a comma-separated list of output fields + :param opts: output formatting and control options + :param sort_keys: a comma-separated list of sort keys + :param expand: unused + :returns: the number of matching profiles output + :rtype: int + """ + return _print_cache(find_cache_paths, selection=selection, opts=opts, + output_fields=output_fields, sort_keys=sort_keys) + + +def print_cache_images(selection=None, opts=None, output_fields=None, + sort_keys=None, expand=False): + """Print cache entries and images matching selection criteria. + + Selection criteria may be expressed via a ``Selection`` object + passed to the call using the ``Selection`` parameter. + + :param selection: A Selection object giving selection + criteria for the operation + :param output_fields: a comma-separated list of output fields + :param opts: output formatting and control options + :param sort_keys: a comma-separated list of sort keys + :param expand: unused + :returns: the number of matching profiles output + :rtype: int + """ + return _print_cache(find_cache_images, selection=selection, opts=opts, + output_fields=output_fields, sort_keys=sort_keys) + + def show_legacy(selection=None, loader=BOOM_LOADER_GRUB1): """Print boot entries in legacy boot loader formats. @@ -1728,6 +1936,8 @@ def _create_cmd(cmd_args, select, opts, identifier): arch = cmd_args.architecture + images = I_BACKUP if cmd_args.backup else I_NONE + try: be = create_entry(title, version, machine_id, root_device, lvm_root_lv=lvm_root_lv, @@ -1736,7 +1946,7 @@ def _create_cmd(cmd_args, select, opts, identifier): add_opts=add_opts, del_opts=del_opts, architecture=arch, write=False, expand=cmd_args.expand_variables, - allow_no_dev=no_dev) + allow_no_dev=no_dev, images=images) except BoomRootDeviceError as brde: print(brde) @@ -1824,23 +2034,21 @@ def _clone_cmd(cmd_args, select, opts, identifier): return 1 title = cmd_args.title - version = cmd_args.version root_device = cmd_args.root_device lvm_root_lv = cmd_args.root_lv subvol = cmd_args.btrfs_subvolume (btrfs_subvol_path, btrfs_subvol_id) = _subvol_from_arg(subvol) - if not cmd_args.machine_id: - # Use host machine-id by default - machine_id = _get_machine_id() - if not machine_id: - print("Could not determine machine_id") - return 1 - else: - machine_id = cmd_args.machine_id - # Discard all selection criteria but boot_id. select = Selection(boot_id=select.boot_id) + try: + be = _find_one_entry(select) + except ValueError as e: + print(e) + return 1 + + version = cmd_args.version or be.version + machine_id = cmd_args.machine_id or be.machine_id profile = _find_profile(cmd_args, version, machine_id, "clone") @@ -1849,6 +2057,8 @@ def _clone_cmd(cmd_args, select, opts, identifier): arch = cmd_args.architecture + images = I_BACKUP if cmd_args.backup else I_NONE + try: be = clone_entry(select, title=title, version=version, machine_id=machine_id, root_device=root_device, @@ -1857,13 +2067,16 @@ def _clone_cmd(cmd_args, select, opts, identifier): btrfs_subvol_id=btrfs_subvol_id, profile=profile, add_opts=add_opts, del_opts=del_opts, architecture=arch, expand=cmd_args.expand_variables, - allow_no_dev=cmd_args.no_dev) + allow_no_dev=cmd_args.no_dev, images=images) except ValueError as e: print(e) return 1 + # Command-line overrides take precedence over any overridden values + # in the cloned BootEntry. _apply_profile_overrides(be, cmd_args) + _apply_optional_keys(be, cmd_args) try: be.write_entry(expand=cmd_args.expand_variables) @@ -1983,20 +2196,21 @@ def _edit_cmd(cmd_args, select, opts, identifier): return 1 title = cmd_args.title - version = cmd_args.version root_device = cmd_args.root_device lvm_root_lv = cmd_args.root_lv subvol = cmd_args.btrfs_subvolume (btrfs_subvol_path, btrfs_subvol_id) = _subvol_from_arg(subvol) - if not cmd_args.machine_id: - # Use host machine-id by default - machine_id = _get_machine_id() - if not machine_id: - print("Could not determine machine_id") - return 1 - else: - machine_id = cmd_args.machine_id + # Discard all selection criteria but boot_id. + select = Selection(boot_id=select.boot_id) + try: + be = _find_one_entry(select) + except ValueError as e: + print(e) + return 1 + + version = cmd_args.version or be.version + machine_id = cmd_args.machine_id or be.machine_id profile = _find_profile(cmd_args, version, machine_id, "edit") @@ -2013,6 +2227,8 @@ def _edit_cmd(cmd_args, select, opts, identifier): print(e) return 1 + # Command-line overrides take precedence over any overridden values + # in the edited BootEntry. _apply_profile_overrides(be, cmd_args) try: @@ -2543,6 +2759,51 @@ def _edit_host_cmd(cmd_args, select, opts, identifier): return 0 +def _show_cache_cmd(cmd_args, select, opts, identifier): + """Show cache command handler. + + Show the cache entries that match the given selection criteria + in human readable form. Each matching entry is printed as a + multi-line record. + + :param cmd_args: Command line arguments for the command + :param select: Selection criteria for the profiles to show. + :returns: integer status code returned from ``main()`` + """ + if identifier is not None: + select = Selection(path=identifier) + + try: + find_fn = find_cache_images if cmd_args.verbose else find_cache_paths + ces = find_fn(selection=select) + except ValueError as e: + print(e) + return 1 + first = True + for ce in ces: + ws = "" if first else "\n" + ce_str = str(ce) + ce_str = _str_indent(ce_str, 2) + print("%sCache Entry (img_id=%s)\n%s" % (ws, ce.disp_img_id, ce_str)) + first = False + return 0 + + +def _list_cache_cmd(cmd_args, select, opts, identifier): + """List cache command handler. + + List the cache entries that match the given selection criteria + as a tabular report, with one entry per row. + + :param cmd_args: Command line arguments for the command + :param select: Selection criteria fore the profiles to list + :returns: integer status code returned from ``main()`` + """ + print_fn = print_cache if not cmd_args.verbose else print_cache_images + return _generic_list_cmd(cmd_args, select, opts, _verbose_cache_fields, + print_fn) + + def _write_legacy_cmd(cmd_args, select, opts, identifier): if identifier: print("write legacy does not accept a boot_id") @@ -2592,6 +2853,7 @@ WRITE_CMD = "write" ENTRY_TYPE = "entry" PROFILE_TYPE = "profile" HOST_TYPE = "host" +CACHE_TYPE = "cache" LEGACY_TYPE = "legacy" _boom_entry_commands = [ @@ -2621,6 +2883,11 @@ _boom_host_commands = [ (EDIT_CMD, _edit_host_cmd) ] +_boom_cache_commands = [ + (SHOW_CMD, _show_cache_cmd), + (LIST_CMD, _list_cache_cmd) +] + _boom_legacy_commands = [ (WRITE_CMD, _write_legacy_cmd), (CLEAR_CMD, _clear_legacy_cmd), @@ -2631,10 +2898,23 @@ _boom_command_types = [ (ENTRY_TYPE, _boom_entry_commands), (PROFILE_TYPE, _boom_profile_commands), (HOST_TYPE, _boom_host_commands), + (CACHE_TYPE, _boom_cache_commands), (LEGACY_TYPE, _boom_legacy_commands) ] +def _get_command_verbs(): + """Return the set of command verbs known to boom. + """ + verbs = set() + all_cmds = [_boom_entry_commands, _boom_profile_commands, + _boom_host_commands, _boom_cache_commands, + _boom_legacy_commands] + for cmd_list in all_cmds: + verbs.update([cmd[0] for cmd in cmd_list]) + return verbs + + def _id_from_arg(cmd_args, cmdtype, cmd): if cmd == CREATE_CMD: if cmdtype == ENTRY_TYPE: @@ -2755,6 +3035,8 @@ def main(args): help="Additional kernel options to append", type=str) parser.add_argument("--architecture", metavar="ARCH", default=None, help="An optional BLS architecture string", type=str) + parser.add_argument("--backup", action="store_true", + help="Back up boot images used by entry") parser.add_argument("-b", "--boot-id", "--bootid", metavar="BOOT_ID", type=str, help="The BOOT_ID of a boom boot entry") parser.add_argument("--boot-dir", "--bootdir", metavar="PATH", type=str, @@ -2787,9 +3069,7 @@ def main(args): help="Take os-release values from the running host", action="store_true") parser.add_argument("-P", "--host-profile", metavar="PROFILE", type=str, - help="A boom host profile identifier") - parser.add_argument("--host-id", metavar="HOSTID", type=str, - help="A host profile identifier") + help="A boom host profile identifier", dest="host_id") parser.add_argument("--host-name", metavar="HOSTNAME", type=str, help="The host name associated with a host profile") parser.add_argument("-i", "--initrd", metavar="IMG", type=str, @@ -2858,6 +3138,23 @@ def main(args): parser.add_argument("-v", "--version", metavar="VERSION", type=str, help="The kernel version of a boom " "boot entry") + + if len(args) < 3: + parser.print_usage() + print("Too few arguments: %s" % " ".join(args[1:])) + return 1 + + cmd_types = [cmdtype[0] for cmdtype in _boom_command_types] + cmd_verbs = _get_command_verbs() + + type_arg = args[1] if len(args) > 1 else "" + cmd_arg = args[2] if len(args) > 2 else "" + + if type_arg not in cmd_types or cmd_arg not in cmd_verbs: + parser.print_usage() + print("Unknown command: %s %s" % (type_arg, cmd_arg)) + return 1 + try: cmd_args = parser.parse_args(args=args[1:]) except SystemExit as e: @@ -2870,6 +3167,9 @@ def main(args): return 1 setup_logging(cmd_args) cmd_type = _match_cmd_type(cmd_args.type) + if not cmd_type: + print("Unknown command type: %s" % cmd_args.type) + return 1 if cmd_args.boot_dir or BOOM_BOOT_PATH_ENV in environ: boot_path = cmd_args.boot_dir or environ[BOOM_BOOT_PATH_ENV] @@ -2881,6 +3181,8 @@ def main(args): if cmd_args.config: set_boom_config_path(cmd_args.config) + bc = load_boom_config() + if not path_exists(get_boom_path()): _log_error("Configuration directory '%s' not found." % get_boom_path()) @@ -2906,6 +3208,10 @@ def main(args): boom_entries_path()) return 1 + if cmd_type[0] == CACHE_TYPE and not bc.cache_enable: + _log_error("Boot image cache disabled (config.cache_enable=False)") + return 1 + # Parse an LV name from root_lv and re-write the root_device if found if cmd_args.root_lv: try: @@ -2931,16 +3237,19 @@ def main(args): # No valid VG name pass - if not cmd_type: - print("Unknown command type: %s" % cmd_args.type) - return 1 - type_cmds = cmd_type[1] command = _match_command(cmd_args.command, type_cmds) if not command: print("Unknown command: %s %s" % (cmd_type[0], cmd_args.command)) return 1 + if cmd_args.backup and not bc.cache_enable: + print("--backup specified but cache disabled" + " (config.cache_enable=False)") + return 1 + elif cmd_args.backup: + load_cache() + select = Selection.from_cmd_args(cmd_args) opts = _report_opts_from_args(cmd_args) identifier = _id_from_arg(cmd_args, cmd_type[0], command[0]) @@ -2954,6 +3263,12 @@ def main(args): except Exception as e: _log_error("Command failed: %s" % e) + if bc.cache_enable and bc.cache_auto_clean: + try: + clean_cache() + except Exception as e: + _log_error("Could not clean boot image cache: %s" % e) + shutdown_logging() return status @@ -2969,7 +3284,11 @@ __all__ = [ # HostProfile manipulation 'create_host', 'delete_hosts', 'clone_host', 'edit_host', - 'list_hosts', 'print_hosts' + 'list_hosts', 'print_hosts', + + # Cache manipulation + 'print_cache', 'print_cache_images', + 'I_NONE', 'I_CACHE', 'I_BACKUP' ] # vim: set et ts=4 sw=4 : diff --git a/boom/config.py b/boom/config.py index 4ed3d14..58614c8 100644 --- a/boom/config.py +++ b/boom/config.py @@ -31,7 +31,7 @@ import logging try: # Python2 from ConfigParser import SafeConfigParser as ConfigParser, ParsingError -except ModuleNotFoundError: +except ImportError: # Python3 from configparser import ConfigParser, ParsingError @@ -66,6 +66,9 @@ _CFG_BOOM_ROOT = "boom_root" _CFG_LEGACY_ENABLE = "enable" _CFG_LEGACY_FMT = "format" _CFG_LEGACY_SYNC = "sync" +_CFG_SECT_CACHE = "cache" +_CFG_CACHE_ENABLE = "enable" +_CFG_CACHE_PATH = "cache_path" def _read_boom_config(path=None): @@ -116,6 +119,16 @@ def _read_boom_config(path=None): sync = cfg.get(_CFG_SECT_LEGACY, _CFG_LEGACY_SYNC) bc.legacy_sync = any([t for t in trues if t in sync]) + if cfg.has_section(_CFG_SECT_CACHE): + if cfg.has_option(_CFG_SECT_CACHE, _CFG_CACHE_ENABLE): + _log_debug("Found cache.enable") + enable = cfg.get(_CFG_SECT_CACHE, _CFG_CACHE_ENABLE) + bc.cache_enable = any([t for t in trues if t in enable]) + + if cfg.has_option(_CFG_SECT_CACHE, _CFG_CACHE_PATH): + _log_debug("Found cache.cache_path") + bc.cache_path = cfg.get(_CFG_SECT_CACHE, _CFG_CACHE_PATH) + _log_debug("read configuration: %s" % repr(bc)) bc._cfg = cfg return bc @@ -132,6 +145,7 @@ def load_boom_config(path=None): """ bc = _read_boom_config(path=path) set_boom_config(bc) + return bc def _sync_config(bc, cfg): diff --git a/boom/hostprofile.py b/boom/hostprofile.py index cd56f89..3d195d7 100644 --- a/boom/hostprofile.py +++ b/boom/hostprofile.py @@ -207,7 +207,7 @@ def load_host_profiles(): load_profiles_for_class(HostProfile, "Host", profiles_path, "host") _host_profiles_loaded = True - _log_info("Loaded %d host profiles" % len(_host_profiles)) + _log_debug("Loaded %d host profiles" % len(_host_profiles)) def write_host_profiles(force=False): diff --git a/boom/osprofile.py b/boom/osprofile.py index 8b68f93..1893676 100644 --- a/boom/osprofile.py +++ b/boom/osprofile.py @@ -219,16 +219,16 @@ def drop_profiles(): :returns: None """ global _profiles, _profiles_by_id, _profiles_loaded - nr_profiles = len(_profiles) - 1 + nr_profiles = len(_profiles) - 1 if _profiles else 0 _profiles = [] _profiles_by_id = {} _null_profile = OsProfile(name="", short_name="", version="", version_id="") - _profiles.append(_null_profile) _profiles_by_id[_null_profile.os_id] = _null_profile - _log_info("Dropped %d profiles" % nr_profiles) + if nr_profiles: + _log_info("Dropped %d profiles" % nr_profiles) _profiles_loaded = False @@ -249,7 +249,7 @@ def load_profiles(): global _profiles_loaded drop_profiles() load_profiles_for_class(OsProfile, "Os", boom_profiles_path(), "profile") - _log_info("Loaded %d profiles" % (len(_profiles) - 1)) + _log_debug("Loaded %d profiles" % (len(_profiles) - 1)) _profiles_loaded = True @@ -666,7 +666,7 @@ class BoomProfile(object): comment = "" ptype = self.__class__.__name__ - _log_debug("Loading %sProfile from '%s'" % + _log_debug("Loading %s from '%s'" % (ptype, basename(profile_file))) with open(profile_file, "r") as pf: for line in pf: @@ -1037,6 +1037,9 @@ class BoomProfile(object): @options.setter def options(self, value): + if "root=" not in value: + raise ValueError("OsProfile.options must include root= " + "device option") self._profile_data[BOOM_OS_OPTIONS] = value self._dirty() @@ -1218,7 +1221,7 @@ class BoomProfile(object): return try: unlink(profile_path) - _log_debug("Deleted %sProfile(id='%s')" % (ptype, profile_id)) + _log_debug("Deleted %s(id='%s')" % (ptype, profile_id)) except Exception as e: _log_error("Error removing %s file '%s': %s" % (ptype, profile_path, e)) @@ -1362,6 +1365,12 @@ class OsProfile(BoomProfile): if key not in profile_data: raise ValueError(err_str % key) + if BOOM_OS_OPTIONS not in profile_data: + raise ValueError(err_str % BOOM_OS_OPTIONS) + elif "root=" not in profile_data[BOOM_OS_OPTIONS]: + raise ValueError("OsProfile.options must include root= " + "device option") + root_opts = [key for key in OS_ROOT_KEYS if key in profile_data] if not any(root_opts): root_opts_err = err_str % "ROOT_OPTS" @@ -1402,7 +1411,7 @@ class OsProfile(BoomProfile): comment = "" ptype = self.__class__.__name__ - _log_debug("Loading %sProfile from '%s'" % + _log_debug("Loading %s from '%s'" % (ptype, basename(profile_file))) with open(profile_file, "r") as pf: for line in pf: @@ -1416,10 +1425,7 @@ class OsProfile(BoomProfile): comment = "" self._comments = comments - try: - self._from_data(profile_data, dirty=False) - except ValueError as e: - raise ValueError(str(e) + "in %s" % profile_file) + self._from_data(profile_data, dirty=False) def __init__(self, name=None, short_name=None, version=None, version_id=None, profile_file=None, profile_data=None, diff --git a/boom/report.py b/boom/report.py index 0ae6072..8e55aa3 100644 --- a/boom/report.py +++ b/boom/report.py @@ -301,7 +301,7 @@ class BoomField(object): """ if value is not None and not isinstance(value, num_types): raise TypeError("Value for report_num() must be a numeric type.") - report_string = str(value) if value else "" + report_string = str(value) sort_value = value if value is not None else -1 self.set_value(report_string, sort_value=sort_value) diff --git a/doc/boom.rst b/doc/boom.rst index 91c2c89..8f6ad1c 100644 --- a/doc/boom.rst +++ b/doc/boom.rst @@ -60,6 +60,17 @@ boom.command module :show-inheritance: +boom.cache module +----------------- + +.. automodule:: boom.cache + :members: + :special-members: + :exclude-members: __dict__, __module__, __weakref__ + :undoc-members: + :show-inheritance: + + boom.report module ------------------ @@ -72,4 +83,3 @@ boom.report module :show-inheritance: - diff --git a/doc/conf.py b/doc/conf.py index f12214c..cccdcfa 100644 --- a/doc/conf.py +++ b/doc/conf.py @@ -56,7 +56,7 @@ master_doc = 'index' # General information about the project. project = u'Boom' -copyright = u'2017, Bryn M. Reeves' +copyright = u'2017-2020, Red Hat, Inc.' author = u'Bryn M. Reeves' # The version info for the project you're documenting, acts as replacement for @@ -64,9 +64,9 @@ author = u'Bryn M. Reeves' # built documents. # # The short X.Y version. -version = u'0.1' +version = u'1.1' # The full version, including alpha/beta/rc tags. -release = u'0.1.0' +release = u'1.1' # The language for content autogenerated by Sphinx. Refer to documentation # for a list of supported languages. diff --git a/examples/profiles/4abe4f7fd38fc7506cbe2f42b1b85d54af3b29b1-rhel8.profile b/examples/profiles/4abe4f7fd38fc7506cbe2f42b1b85d54af3b29b1-rhel8.profile new file mode 120000 index 0000000..667bd04 --- /dev/null +++ b/examples/profiles/4abe4f7fd38fc7506cbe2f42b1b85d54af3b29b1-rhel8.profile @@ -0,0 +1 @@ +../../tests/boom/profiles/4abe4f7fd38fc7506cbe2f42b1b85d54af3b29b1-rhel8.profile \ No newline at end of file diff --git a/examples/profiles/4aff687826c2b83409c0cbd097a93c405681e227-rhel7.8.profile b/examples/profiles/4aff687826c2b83409c0cbd097a93c405681e227-rhel7.8.profile new file mode 120000 index 0000000..637d9fe --- /dev/null +++ b/examples/profiles/4aff687826c2b83409c0cbd097a93c405681e227-rhel7.8.profile @@ -0,0 +1 @@ +../../tests/boom/profiles/4aff687826c2b83409c0cbd097a93c405681e227-rhel7.8.profile \ No newline at end of file diff --git a/examples/profiles/72e3679e3556b8e1620dd3b6e5d2a1f508e5ba43-rhel7.7.profile b/examples/profiles/72e3679e3556b8e1620dd3b6e5d2a1f508e5ba43-rhel7.7.profile new file mode 120000 index 0000000..42447e9 --- /dev/null +++ b/examples/profiles/72e3679e3556b8e1620dd3b6e5d2a1f508e5ba43-rhel7.7.profile @@ -0,0 +1 @@ +../../tests/boom/profiles/72e3679e3556b8e1620dd3b6e5d2a1f508e5ba43-rhel7.7.profile \ No newline at end of file diff --git a/examples/profiles/8896596a45fcc9e36e9c87aee77ab3e422da2635-fedora30.profile b/examples/profiles/8896596a45fcc9e36e9c87aee77ab3e422da2635-fedora30.profile new file mode 120000 index 0000000..c0466bd --- /dev/null +++ b/examples/profiles/8896596a45fcc9e36e9c87aee77ab3e422da2635-fedora30.profile @@ -0,0 +1 @@ +../../tests/boom/profiles/8896596a45fcc9e36e9c87aee77ab3e422da2635-fedora30.profile \ No newline at end of file diff --git a/man/man5/boom.5 b/man/man5/boom.5 index d191b02..757f0e1 100644 --- a/man/man5/boom.5 +++ b/man/man5/boom.5 @@ -35,6 +35,25 @@ format (currently only \fbgrub\fP syntax is supported). If the value of the \fBsync\fP key is true the legacy configuration will be automatically written whenever entries are added, removed, or modified. +.TP +.B cache +The cache section contains settings that control the behavior of the +boom boot image cache. The \fBenable\fP key enables or disables the +boot image cache: if the cache is disabled then the \fBboom cache\fP +command and cache-related options such as \fB\-\-backup\fP are not +available. + +The path to the on-disk cache may be changed by setting the +\fBcache_path\fP key. By default the cache is located in a +subdirectory of /boot at /boot/boom/cache. Alternately a separate +file system may be used to contain the cache (mounted at this path +or another location specified in \fBboom.conf\fP). In this case the +cache file system should be configured to be mounted at the same +location in all installed operating system instances. + +If the \fBauto_clean\fP key is enabled boom will automatically +remove cache entries when they are no longer used by a boom boot +entry. . .SH AUTHORS . @@ -42,9 +61,9 @@ Bryn M. Reeves . .SH SEE ALSO . -Boom project page: https://github.com/bmr-cymru/boom +Boom project page: https://github.com/snapshotmanager/boom .br -Boot to snapshot documentation: https://github.com/bmr-cymru/snapshot-boot-docs +Boot to snapshot documentation: https://github.com/snapshotmanager/snapshot-boot-docs .br LVM2 resource page: https://www.sourceware.org/lvm2/ .br diff --git a/man/man8/boom.8 b/man/man8/boom.8 index 28bad24..7b862e7 100644 --- a/man/man8/boom.8 +++ b/man/man8/boom.8 @@ -8,7 +8,6 @@ . RI [ create | delete | clone | show | list | edit ] .. . -. .de ARG_LEGACY_TYPES . RI legacy .. @@ -17,7 +16,15 @@ . RI [ write | clear | show ] .. . +.de ARG_CACHE_TYPES +. RI cache +.. . +.de ARG_CACHE_COMMAND +. RI [ list | show ] +.. +. +.. .SH NAME . Boom \(em linux boot manager @@ -49,6 +56,17 @@ Boom \(em linux boot manager . .HP .B boom +.de CMD_CACHE_COMMAND +. ad l +. ARG_CACHE_TYPES +. ARG_CACHE_COMMAND +. ad b +.. +.CMD_CACHE_COMMAND + +. +.HP +.B boom .de CMD_ENTRY_CREATE . ad l . BR entry @@ -553,6 +571,42 @@ Boom \(em linux boot manager . ad b .. .CMD_LEGACY_SHOW + +. +.HP +.B boom +.de CMD_CACHE_LIST +. ad l +. BR cache +. BR \fBlist +. IR [ img_id ] +. RB [ --image +. IR img_id ] +. RB [ --linux +. IR kernel_path ] +. RB [ --initrd +. IR initrd_path ] +. ad b +.. +.CMD_CACHE_LIST +. +.HP +.B boom +.de CMD_CACHE_SHOW +. ad l +. BR cache +. BR \fBshow +. IR [ img_id ] +. RB [ --image +. IR img_id ] +. RB [ --linux +. IR kernel_path ] +. RB [ --initrd +. IR initrd_path ] +. ad b +.. +.CMD_CACHE_SHOW + . .PD .ad b @@ -1168,6 +1222,43 @@ Display the selected boot entries as they would appear in the configured legacy boot loader format. The normal command line selection options may be used to control the set of entries written to the terminal. + +.SH BOOT IMAGE CACHE +Boom can optionally cache or back up the images used by a boom +BootEntry. This allows an entry to be booted in the case that a +subsequent update has removed the original kernel and initramfs +images and can be used to recover an earlier system state from +a snapshot following even major operating system updates. +. +.HP +.B boom +.CMD_CACHE_LIST +.br +Output a tabular report of paths present in the boot image cache. + +Displays a report with one cache entry per line, containing fields +describing the properties of the cache entry. + +The list of fields to display is given with \fB--options\fP as a comma +separated list of field names. To obtain a list of available fields run +'\fBboom host list -o help\fP'. If the list of fields begins with the +'\fB+\fP' character the specified fields are appended to the default +field list. Otherwise the given list of fields replaces the default set +of report fields. + +Report output may be sorted by multiple user-defined keys using +the \fB--sort\fP option. The option expects a comma separated list +of keys, with optional '\fB+\fP' and '\fB-\fP' prefixes indicating +ascending and descending sort for that field respectively. +. +.HP +.B boom +.CMD_CACHE_SHOW +.br +Display matching cache entries on standard output. + +Entries matching selection criteria are printed in a compact multi-line +format. . .SH REPORT FIELDS . @@ -1268,6 +1359,54 @@ The kernel command line options template for this OS profile. .B profilepath The absolute path to this OS Profile's on-disk configuration file. . +.SS Host Profile fields +. +Host Profile fields provide access to the details of a profile's +configuration including identity fields and the template strings +used to generate entries. This includes all fields available in +the OS Profile report as well as additional Host Profile identity +fields. +.TP +.B hostid +Host profile identifier. +.TP +.B hostname +The hostname of this host profile. +.TP +.B label +The label of this host profile. +. +.SS Cache Entry fields +. +Cache entry fields provide information on the paths and images +stored in the boom boot image cache. +.TP +.B imgid +Image identifier. +.TP +.B path +Path to the cached image, relative to the boot file system. +.TP +.B mode +Path file system mode in human-readable format. +.TP +.B uid +Image owner user identifier. +.TP +.B gid +Image owner group identifier. +.TP +.B ts +Image timestamp. The mtime of the image file at the time it was added +to the cache. +.TP +.B state +A string description of the cache entry state: \fBCACHED\fp, +\fBMISSING\fP, \fBRESTORED\fP, or \fBBROKEN\fp. +.TP +.B count +The number of boot entries that reference this boot image. +. .SH REPORTING COMMANDS Both the \fBentry list\fP and \fBprofile list\fP commands use a common reporting system to display the results of the query. The selection of @@ -1573,9 +1712,9 @@ Bryn M. Reeves . .SH SEE ALSO . -Boom project page: https://github.com/bmr-cymru/boom +Boom project page: https://github.com/snapshotmanager/boom .br -Boot to snapshot documentation: https://github.com/bmr-cymru/snapshot-boot-docs +Boot to snapshot documentation: https://github.com/snapshotmanager/snapshot-boot-docs .br BootLoader Specification: https://systemd.io/BOOT_LOADER_SPECIFICATION .br diff --git a/setup.py b/setup.py index 2cda1f1..8191f8a 100644 --- a/setup.py +++ b/setup.py @@ -9,7 +9,7 @@ setup( description=("""The Boom Boot Manager."""), author='Bryn M. Reeves', author_email='bmr@redhat.com', - url='https://github.com/bmr-cymru/boom', + url='https://github.com/snapshotmanager/boom', license="GPLv2", test_suite="tests", scripts=['bin/boom'], diff --git a/tests/.initramfs-3.10.1-1.el7.img.boomrestored b/tests/.initramfs-3.10.1-1.el7.img.boomrestored new file mode 100644 index 0000000..e69de29 --- /dev/null +++ b/tests/.initramfs-3.10.1-1.el7.img.boomrestored diff --git a/tests/__init__.py b/tests/__init__.py index 8b56bb3..2f9114b 100644 --- a/tests/__init__.py +++ b/tests/__init__.py @@ -13,7 +13,7 @@ # Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA from os.path import join, abspath -from os import geteuid, getegid, makedirs +from os import environ, getcwd, geteuid, getegid, makedirs from subprocess import Popen, PIPE import shutil import errno @@ -56,6 +56,15 @@ def reset_boom_paths(): """ boom.set_boot_path(BOOT_ROOT_TEST) +def set_mock_path(): + """Set the PATH environment variable to tests/bin to include mock + binaries used in the boom test suite. + """ + os_path = environ['PATH'] + os_path = join(getcwd(), "tests/bin") + ":" + os_path + environ['PATH'] = os_path + + # Mock objects class MockArgs(object): @@ -63,7 +72,8 @@ class MockArgs(object): """ add_opts = "" architecture = None - boot_id = "12345678" + backup = False + boot_id = None btrfs_opts = "" btrfs_subvolume = "23" command = "" @@ -222,7 +232,7 @@ def have_grub1(): __all__ = [ 'BOOT_ROOT_TEST', 'SANDBOX_PATH', 'rm_sandbox', 'mk_sandbox', 'reset_sandbox', 'reset_boom_paths', - 'get_logical_volume', 'get_root_lv', 'have_root_lv', + 'set_mock_path', 'get_logical_volume', 'get_root_lv', 'have_root_lv', 'MockArgs', 'have_root', 'have_lvm', 'have_grub1' ] diff --git a/tests/bin/grub2-editenv b/tests/bin/grub2-editenv new file mode 100755 index 0000000..b27e707 --- /dev/null +++ b/tests/bin/grub2-editenv @@ -0,0 +1,14 @@ +#!/bin/bash + +if [ "$1" == "list" ]; then + cat < +# +# cache_tests.py - Boom cache API tests. +# +# This file is part of the boom project. +# +# This copyrighted material is made available to anyone wishing to use, +# modify, copy, or redistribute it subject to the terms and conditions +# of the GNU General Public License v.2. +# +# You should have received a copy of the GNU Lesser General Public License +# along with this program; if not, write to the Free Software Foundation, +# Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA +import unittest +import logging +from sys import stdout +from os import listdir, makedirs, unlink +from os.path import abspath, basename, dirname, exists, join +from glob import glob +import shutil +import re + +# Python3 moves StringIO to io +try: + from StringIO import StringIO +except: + from io import StringIO + +log = logging.getLogger() +log.level = logging.DEBUG +log.addHandler(logging.FileHandler("test.log")) + +from boom import * +from boom.osprofile import * +from boom.bootloader import * +from boom.hostprofile import * +from boom.command import * +from boom.config import * +from boom.report import * +from boom.cache import * + +# For access to non-exported members +import boom.cache + +from tests import * + +BOOT_ROOT_TEST = abspath("./tests") +config = BoomConfig() +config.legacy_enable = False +config.legacy_sync = False +set_boom_config(config) +set_boot_path(BOOT_ROOT_TEST) + +debug_masks = ['profile', 'entry', 'report', 'command', 'all'] + + +class CacheHelperTests(unittest.TestCase): + """Test internal boom.cache helpers. Cases in this class must + not modify on-disk state and do not use a unique test + fixture. + """ + def test__make_relative_with_non_abs_path(self): + path = "not/an/absolute/path" + self.assertEqual(path, boom.cache._make_relative(path)) + + def test__make_relative_with_abs_path(self): + path = "/vmlinuz" + self.assertEqual(path[1:], boom.cache._make_relative(path)) + + def test__make_relative_root_is_empty_string(self): + path = "/" + self.assertEqual("", boom.cache._make_relative(path)) + + def test__image_path_to_boot(self): + image_path = "vmlinuz" + xboot_path = join(get_boot_path(), image_path) + boot_path = boom.cache._image_path_to_boot(join("/", image_path)) + self.assertEqual(boot_path, xboot_path) + + def test__image_id_to_cache_path(self): + img_id = "47dc6ad4ea9ca5453e607987d49c33858bd553e0" + xcache_file = "47dc6ad4ea9ca5453e607987d49c33858bd553e0.img" + self.assertEqual(boom.cache._image_id_to_cache_path(img_id), + join(get_cache_path(), xcache_file)) + + def test__image_id_from_path(self): + img_path = join(get_boot_path(), "vmlinuz-5.5.5-1.fc30.x86_64") + ximg_id = "fdfb8e5a3857adca47f25ee47078bad4a757cc92" + self.assertEqual(boom.cache._image_id_from_path(img_path), ximg_id) + + def test__image_id_from_bad_path_raises(self): + img_path = "/qux/qux/qux" # non-existent + with self.assertRaises(IOError) as cm: + boom.cache._image_id_from_path(img_path) + +class CacheTests(unittest.TestCase): + """Test boom.command APIs + """ + + # Master BLS loader directory for sandbox + loader_path = join(BOOT_ROOT_TEST, "loader") + + # Master boom configuration path for sandbox + boom_path = join(BOOT_ROOT_TEST, "boom") + + # Master grub configuration path for sandbox + grub_path = join(BOOT_ROOT_TEST, "grub") + + # Test fixture init/cleanup + def setUp(self): + """Set up a test fixture for the CommandTests class. + + Defines standard objects for use in these tests. + """ + reset_sandbox() + + # Sandbox paths + boot_sandbox = join(SANDBOX_PATH, "boot") + boom_sandbox = join(SANDBOX_PATH, "boot/boom") + grub_sandbox = join(SANDBOX_PATH, "boot/grub") + loader_sandbox = join(SANDBOX_PATH, "boot/loader") + + # Initialise sandbox from master + makedirs(boot_sandbox) + shutil.copytree(self.boom_path, boom_sandbox) + shutil.copytree(self.loader_path, loader_sandbox) + shutil.copytree(self.grub_path, grub_sandbox) + + # Copy boot images + images = glob(join(BOOT_ROOT_TEST, "initramfs*")) + images += glob(join(BOOT_ROOT_TEST, "vmlinuz*")) + for image in images: + def _dotfile(img_path): + pattern = ".%s.boomrestored" + img_name = basename(img_path) + img_dir = dirname(img_path) + return join(img_dir, pattern % img_name) + + shutil.copy2(image, boot_sandbox) + if exists(_dotfile(image)): + shutil.copy2(_dotfile(image), boot_sandbox) + + # Set boom paths + set_boot_path(boot_sandbox) + + # Tests that deal with legacy configs will enable this. + config = BoomConfig() + config.legacy_enable = False + config.legacy_sync = False + + # Reset profiles, entries, and host profiles to known state. + load_profiles() + load_entries() + load_host_profiles() + load_cache() + + def tearDown(self): + # Drop any in-memory entries and profiles modified by tests + drop_entries() + drop_profiles() + drop_host_profiles() + drop_cache() + + # Clear sandbox data + rm_sandbox() + reset_boom_paths() + + def _make_null_testimg(self, restored=False): + """Return an empty file in the configured $BOOT path for test use. + If ``restored`` is ``True`` the image will be made to look like + an image restored by boom. + """ + img_name = "testimg" + boot_path = get_boot_path() + print(boot_path) + # The img_id of an empty file is the sha1 null hash: da39a3e + img_file = open(join(boot_path, img_name), "w") + img_file.close() + + if restored: + boomrestored_name = "." + img_name + ".boomrestored" + boomrestored_file = open(join(boot_path, boomrestored_name), "w") + boomrestored_file.close() + + return img_name + + def test_drop_cache(self): + drop_cache() + self.assertEqual(len(boom.cache._index), 0) + self.assertEqual(len(boom.cache._paths), 0) + self.assertEqual(len(boom.cache._images), 0) + + def test_load_cache(self): + load_cache() + self.assertTrue(len(boom.cache._index)) + self.assertTrue(len(boom.cache._paths)) + self.assertTrue(len(boom.cache._images)) + + # Verify number of images + boom_cache_path = get_cache_path() + ximage_count = 0 + for p in listdir(boom_cache_path): + if p.endswith(".img"): + ximage_count += 1 + self.assertEqual(len(boom.cache._images), ximage_count) + + def test_load_cache_no_cacheindex(self): + # Wipe cache + unlink(join(SANDBOX_PATH, "boot/boom/cache/cacheindex.json")) + load_cache() + self.assertFalse(len(boom.cache._index)) + self.assertFalse(len(boom.cache._paths)) + self.assertFalse(len(boom.cache._images)) + + def test_write_cache(self): + # Re-write the current cache state + write_cache() + + # Write an empty cache + drop_cache() + write_cache() + + def test__insert(self): + img_name = self._make_null_testimg() + img_path = join(get_boot_path(), img_name) + + cache_name = "da39a3ee5e6b4b0d3255bfef95601890afd80709.img" + cache_path = join(get_cache_path(), cache_name) + + boom.cache._insert(img_path, cache_path) + self.assertTrue(exists(cache_path)) + + def test__insert_bad_path_raises(self): + img_path = "/qux/qux/qux" + + cache_name = "da39a3ee5e6b4b0d3255bfef95601890afd80709.img" + cache_path = join(get_cache_path(), cache_name) + + with self.assertRaises(IOError) as cm: + boom.cache._insert(img_path, cache_path) + + def test__remove_boot(self): + img_name = self._make_null_testimg(restored=True) + img_path = join(get_boot_path(), img_name) + + self.assertTrue(exists(img_path)) + boom.cache._remove_boot(img_path) + self.assertFalse(exists(img_path)) + + def test__remove_boot_bad_path_raises(self): + with self.assertRaises(ValueError) as cm: + boom.cache._remove_boot("nosuch.img") + + def test__remove_boot_not_restored_raises(self): + img_name = self._make_null_testimg(restored=False) + img_path = join(get_boot_path(), img_name) + + with self.assertRaises(ValueError) as cm: + boom.cache._remove_boot(img_path) + + def test__remove(self): + img_name = self._make_null_testimg() + img_path = join(get_boot_path(), img_name) + + cache_name = "da39a3ee5e6b4b0d3255bfef95601890afd80709.img" + cache_path = join(get_cache_path(), cache_name) + + boom.cache._insert(img_path, cache_path) + self.assertTrue(exists(cache_path)) + + boom.cache._remove(cache_path) + self.assertFalse(exists(cache_path)) + + def test__remove_nonex_path_raises(self): + cache_name = "da39a3ee5e6b4b0d3255bfef95601890afd80709.img" + cache_path = join(get_cache_path(), cache_name) + + with self.assertRaises(OSError) as cm: + boom.cache._remove(cache_path) + + def test__remove_bad_path_raises(self): + cache_path = "/da39a3ee5e6b4b0d3255bfef95601890afd80709.img" + + with self.assertRaises(ValueError) as cm: + boom.cache._remove(cache_path) + + def test_cache_path(self): + img_name = self._make_null_testimg(restored=False) + img_path = join("/", img_name) + + cache_path(img_path) + + ces = find_cache_paths(Selection(path=img_path)) + self.assertEqual(len(ces), 1) + + def test_cache_path_dupe(self): + img_name = self._make_null_testimg(restored=False) + img_path = join("/", img_name) + + ce1 = cache_path(img_path) + ce2 = cache_path(img_path) + + self.assertEqual(repr(ce1), repr(ce2)) + + def test_cache_path_nonex_path_raises(self): + img_name = "nonexistent" + img_path = join("/", img_name) + + with self.assertRaises(OSError) as cm: + cache_path(img_path) + + def test_cache_path_nonreg_path_raises(self): + img_path = "/" + + with self.assertRaises(ValueError) as cm: + cache_path(img_path) + + def test_backup_path(self): + img_name = self._make_null_testimg(restored=False) + img_path = join("/", img_name) + backup_img = img_path + ".boom0" + + backup_path(img_path, backup_img) + + # Assert backup is in cache + ces = find_cache_paths(Selection(path=backup_img)) + self.assertEqual(len(ces), 1) + + # Assert original is not in cache + ces = find_cache_paths(Selection(path=img_path)) + self.assertEqual(len(ces), 0) + + def test_uncache_path(self): + img_name = self._make_null_testimg(restored=False) + img_path = join("/", img_name) + + cache_path(img_path) + + ces = find_cache_paths(Selection(path=img_path)) + self.assertEqual(len(ces), 1) + + uncache_path(img_path) + + ces = find_cache_paths(Selection(path=img_path)) + self.assertEqual(len(ces), 0) + + + def test_uncache_path_not_cached(self): + img_name = self._make_null_testimg(restored=False) + img_path = join("/", img_name) + + with self.assertRaises(ValueError) as cm: + uncache_path(img_path) + + def test_uncache_in_use(self): + img_path = "/vmlinuz-4.16.11-100.fc26.x86_64" + uncache_path(img_path) + + ces = find_cache_paths(Selection(path=img_path)) + self.assertEqual(len(ces), 1) + + def test_uncache_in_use_force(self): + img_path = "/vmlinuz-4.16.11-100.fc26.x86_64" + uncache_path(img_path, force=True) + + ces = find_cache_paths(Selection(path=img_path)) + self.assertEqual(len(ces), 0) + + def test_uncache_restored(self): + img_path = "/initramfs-3.10.1-1.el7.img" + uncache_path(img_path) + + ces = find_cache_paths(Selection(path=img_path)) + self.assertEqual(len(ces), 1) + + def test_uncache_restored_force(self): + img_path = "/initramfs-3.10.1-1.el7.img" + uncache_path(img_path, force=True) + + ces = find_cache_paths(Selection(path=img_path)) + self.assertEqual(len(ces), 0) + + def test_clean_cache(self): + clean_cache() + + def test_clean_cache_unrefed(self): + cache_path("/initramfs-2.6.0.img") + clean_cache() + + def test_cache_state_missing_and_restore(self): + img_path="/initramfs-2.6.0.img" + ce = cache_path(img_path) + unlink(join(get_boot_path(), img_path[1:])) + self.assertEqual(ce.state, CACHE_MISSING) + ce.restore() + self.assertEqual(ce.state, CACHE_RESTORED) + + def test_cache_state_missing_and_restore_with_dest(self): + img_path="/initramfs-2.6.0.img" + ce = cache_path(img_path) + unlink(join(get_boot_path(), img_path[1:])) + self.assertEqual(ce.state, CACHE_MISSING) + ce.restore(dest=img_path + ".boom0") + self.assertEqual(ce.state, CACHE_RESTORED) + + def test_cache_restore_non_missing_raises(self): + # A path that is not RESTORED|MISSING + img_path = "/vmlinuz-4.16.11-100.fc26.x86_64" + ce = find_cache_paths(Selection(path=img_path))[0] + with self.assertRaises(ValueError) as cm: + ce.restore() + + def test_cache_purge_restored(self): + img_path="/initramfs-2.6.0.img" + ce = cache_path(img_path) + unlink(join(get_boot_path(), img_path[1:])) + self.assertEqual(ce.state, CACHE_MISSING) + ce.restore() + self.assertEqual(ce.state, CACHE_RESTORED) + ce.purge() + self.assertEqual(ce.state, CACHE_MISSING) + + def test_cache_purge_not_restored(self): + # A path that is not RESTORED|MISSING + img_path = "/vmlinuz-4.16.11-100.fc26.x86_64" + ce = find_cache_paths(Selection(path=img_path))[0] + self.assertEqual(ce.state, CACHE_CACHED) + with self.assertRaises(ValueError) as cm: + ce.purge() + + def test_find_cache_paths(self): + ces = find_cache_paths() + self.assertTrue(ces) + + def test_find_cache_images(self): + ces = find_cache_images() + self.assertTrue(ces) + +# vim: set et ts=4 sw=4 : diff --git a/tests/command_tests.py b/tests/command_tests.py index f55dd78..6d0c79b 100644 --- a/tests/command_tests.py +++ b/tests/command_tests.py @@ -15,7 +15,8 @@ import unittest import logging from sys import stdout from os import listdir, makedirs -from os.path import abspath, exists, join +from os.path import abspath, basename, dirname, exists, join +from glob import glob import shutil import re @@ -247,6 +248,20 @@ class CommandTests(unittest.TestCase): shutil.copytree(self.loader_path, loader_sandbox) shutil.copytree(self.grub_path, grub_sandbox) + # Copy boot images + images = glob(join(BOOT_ROOT_TEST, "initramfs*")) + images += glob(join(BOOT_ROOT_TEST, "vmlinuz*")) + for image in images: + def _dotfile(img_path): + pattern = ".%s.boomrestored" + img_name = basename(img_path) + img_dir = dirname(img_path) + return join(img_dir, pattern % img_name) + + shutil.copy2(image, boot_sandbox) + if exists(_dotfile(image)): + shutil.copy2(_dotfile(image), boot_sandbox) + # Set boom paths set_boot_path(boot_sandbox) @@ -1602,6 +1617,48 @@ class CommandTests(unittest.TestCase): r = boom.command._list_cmd(args, None, opts, None) self.assertEqual(r, 0) + def test__list_cmd_with_sort(self): + args = MockArgs() + args.sort = "bootid" + opts = boom.command._report_opts_from_args(args) + r = boom.command._list_cmd(args, Selection(), opts, None) + self.assertEqual(r, 0) + + def test__list_cmd_with_sort_ascending(self): + args = MockArgs() + args.sort = "+bootid" + opts = boom.command._report_opts_from_args(args) + r = boom.command._list_cmd(args, Selection(), opts, None) + self.assertEqual(r, 0) + + def test__list_cmd_with_sort_descending(self): + args = MockArgs() + args.sort = "-bootid" + opts = boom.command._report_opts_from_args(args) + r = boom.command._list_cmd(args, Selection(), opts, None) + self.assertEqual(r, 0) + + def test__list_cmd_with_sort_two(self): + args = MockArgs() + args.sort = "bootid,version" + opts = boom.command._report_opts_from_args(args) + r = boom.command._list_cmd(args, Selection(), opts, None) + self.assertEqual(r, 0) + + def test__list_cmd_with_sort_bad(self): + args = MockArgs() + args.sort = "qux" + opts = boom.command._report_opts_from_args(args) + r = boom.command._list_cmd(args, Selection(), opts, None) + self.assertEqual(r, 1) + + def test__list_cmd_with_options_help(self): + args = MockArgs() + args.options = "help" + opts = boom.command._report_opts_from_args(args) + r = boom.command._list_cmd(args, Selection(), opts, None) + self.assertEqual(r, 0) + def test__list_cmd_verbose(self): """Test the _list_cmd() handler with a valid entry and verbose output. @@ -1912,7 +1969,7 @@ class CommandTests(unittest.TestCase): args = MockArgs() args.profile = "d4439b7" args.uname_pattern = "nf26" - args.os_options = "boot and stuff" + args.os_options = "root=%{root_device} boot and stuff" r = boom.command._edit_profile_cmd(args, None, None, None) self.assertEqual(r, 0) @@ -1922,7 +1979,7 @@ class CommandTests(unittest.TestCase): args = MockArgs() os_id = "d4439b7" args.uname_pattern = "nf26" - args.os_options = "boot and stuff" + args.os_options = "root=%{root_device} boot and stuff" r = boom.command._edit_profile_cmd(args, None, None, os_id) self.assertEqual(r, 0) @@ -2171,6 +2228,21 @@ class CommandTests(unittest.TestCase): r = boom.command._edit_host_cmd(args, None, None, host_id) self.assertEqual(r, 0) + def test__list_cache_cmd(self): + args = MockArgs() + r = boom.command._list_cache_cmd(args, None, None, None) + + def test__list_cache_cmd_with_sort_num(self): + args = MockArgs() + args.sort = "count" + opts = boom.command._report_opts_from_args(args) + r = boom.command._list_cache_cmd(args, Selection(), opts, None) + self.assertEqual(r, 0) + + def test__show_cache_cmd(self): + args = MockArgs() + r = boom.command._show_cache_cmd(args, None, None, None) + def test_boom_main_noargs(self): args = ['bin/boom', '--help'] boom.command.main(args) diff --git a/tests/grub/grub.conf b/tests/grub/grub.conf index 96dbc19..79eb5dd 100644 --- a/tests/grub/grub.conf +++ b/tests/grub/grub.conf @@ -132,6 +132,10 @@ title title root (hd0,0) kernel vmlinuz-2.2.2-2.fc24.x86_64 root=/dev/vg_root/root ro rootflags=subvol=/snapshot/today rhgb quiet initrd initramfs-2.2.2-2.fc24.x86_64.img +title grub args + root (hd0,0) + kernel /vmlinuz-5.4.7-100.fc30.x86_64 root=/dev/vg_hex/root ro rd.lvm.lv=vg_hex/root + initrd /initramfs-5.4.7-100.fc30.x86_64.img title title root (hd0,0) kernel vmlinuz-1.1.1-1.fc24.x86_64 root=/dev/vg_root/root ro rd.lvm.lv=vg_root/root rhgb quiet diff --git a/tests/hostprofile_tests.py b/tests/hostprofile_tests.py index aa8b537..596a804 100644 --- a/tests/hostprofile_tests.py +++ b/tests/hostprofile_tests.py @@ -266,6 +266,7 @@ class HostProfileTests(unittest.TestCase): # Re-set test /boot set_boot_path(BOOT_ROOT_TEST) + @unittest.skipIf(have_root(), "DAC controls do not apply to root") def test_write_host_profiles_fail(self): load_host_profiles() # Set the /boot path to a non-writable path for the test user. diff --git a/tests/initramfs-2.6.0.img b/tests/initramfs-2.6.0.img new file mode 100644 index 0000000..2e112be Binary files /dev/null and b/tests/initramfs-2.6.0.img differ diff --git a/tests/initramfs-3.10.1-1.el7.img b/tests/initramfs-3.10.1-1.el7.img new file mode 100644 index 0000000..50c4fab Binary files /dev/null and b/tests/initramfs-3.10.1-1.el7.img differ diff --git a/tests/initramfs-4.16.11-100.fc26.x86_64.img b/tests/initramfs-4.16.11-100.fc26.x86_64.img new file mode 100644 index 0000000..a4f62bf Binary files /dev/null and b/tests/initramfs-4.16.11-100.fc26.x86_64.img differ diff --git a/tests/initramfs-5.4.7-100.fc30.x86_64.img b/tests/initramfs-5.4.7-100.fc30.x86_64.img new file mode 100644 index 0000000..3188833 Binary files /dev/null and b/tests/initramfs-5.4.7-100.fc30.x86_64.img differ diff --git a/tests/loader/entries/653b444d513a43239c37deae4f5fe644-526f54a-5.4.7-100.fc30.x86_64.conf b/tests/loader/entries/653b444d513a43239c37deae4f5fe644-526f54a-5.4.7-100.fc30.x86_64.conf new file mode 100644 index 0000000..42fece5 --- /dev/null +++ b/tests/loader/entries/653b444d513a43239c37deae4f5fe644-526f54a-5.4.7-100.fc30.x86_64.conf @@ -0,0 +1,10 @@ +#OsIdentifier: 8896596a45fcc9e36e9c87aee77ab3e422da2635 +title grub args +machine-id 653b444d513a43239c37deae4f5fe644 +version 5.4.7-100.fc30.x86_64 +linux /vmlinuz-5.4.7-100.fc30.x86_64 +initrd /initramfs-5.4.7-100.fc30.x86_64.img +options root=/dev/vg_hex/root ro rd.lvm.lv=vg_hex/root +grub_users $grub_users +grub_arg kernel +grub_class --unrestricted diff --git a/tests/report_tests.py b/tests/report_tests.py index 5d4552a..1c27522 100644 --- a/tests/report_tests.py +++ b/tests/report_tests.py @@ -54,6 +54,11 @@ _test_obj_types = [ class ReportTests(unittest.TestCase): + def test_BoomFieldType_no_type(self): + with self.assertRaises(ValueError): + bf = BoomFieldType(0, None, "None", "Nothing", 0, + REP_NUM, lambda x: x) + def test_BoomFieldType_no_name(self): with self.assertRaises(ValueError): bf = BoomFieldType(BR_NUM, None, "None", "Nothing", 0, diff --git a/tests/vmlinuz-2.6.0 b/tests/vmlinuz-2.6.0 new file mode 100644 index 0000000..1bd6eef Binary files /dev/null and b/tests/vmlinuz-2.6.0 differ diff --git a/tests/vmlinuz-3.10.1-1.el7 b/tests/vmlinuz-3.10.1-1.el7 new file mode 100644 index 0000000..24bea55 Binary files /dev/null and b/tests/vmlinuz-3.10.1-1.el7 differ diff --git a/tests/vmlinuz-4.16.11-100.fc26.x86_64 b/tests/vmlinuz-4.16.11-100.fc26.x86_64 new file mode 100644 index 0000000..fa4ddd5 Binary files /dev/null and b/tests/vmlinuz-4.16.11-100.fc26.x86_64 differ diff --git a/tests/vmlinuz-5.4.7-100.fc30.x86_64 b/tests/vmlinuz-5.4.7-100.fc30.x86_64 new file mode 100644 index 0000000..589ec70 Binary files /dev/null and b/tests/vmlinuz-5.4.7-100.fc30.x86_64 differ diff --git a/tests/vmlinuz-5.5.5-1.fc30.x86_64 b/tests/vmlinuz-5.5.5-1.fc30.x86_64 new file mode 100644 index 0000000..7372d0d Binary files /dev/null and b/tests/vmlinuz-5.5.5-1.fc30.x86_64 differ