Blame dnf/drpm.py

Packit Service 21c75c
# drpm.py
Packit Service 21c75c
# Delta RPM support
Packit Service 21c75c
#
Packit Service 21c75c
# Copyright (C) 2012-2016 Red Hat, Inc.
Packit Service 21c75c
#
Packit Service 21c75c
# This copyrighted material is made available to anyone wishing to use,
Packit Service 21c75c
# modify, copy, or redistribute it subject to the terms and conditions of
Packit Service 21c75c
# the GNU General Public License v.2, or (at your option) any later version.
Packit Service 21c75c
# This program is distributed in the hope that it will be useful, but WITHOUT
Packit Service 21c75c
# ANY WARRANTY expressed or implied, including the implied warranties of
Packit Service 21c75c
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General
Packit Service 21c75c
# Public License for more details.  You should have received a copy of the
Packit Service 21c75c
# GNU General Public License along with this program; if not, write to the
Packit Service 21c75c
# Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
Packit Service 21c75c
# 02110-1301, USA.  Any Red Hat trademarks that are incorporated in the
Packit Service 21c75c
# source code or documentation are not subject to the GNU General Public
Packit Service 21c75c
# License and may only be used or replicated with the express permission of
Packit Service 21c75c
# Red Hat, Inc.
Packit Service 21c75c
#
Packit Service 21c75c
Packit Service 21c75c
from __future__ import absolute_import
Packit Service 21c75c
from __future__ import unicode_literals
Packit Service 21c75c
from binascii import hexlify
Packit Service 21c75c
from dnf.yum.misc import unlink_f
Packit Service 21c75c
from dnf.i18n import _
Packit Service 21c75c
Packit Service 21c75c
import dnf.callback
Packit Service 21c75c
import dnf.logging
Packit Service 21c75c
import dnf.repo
Packit Service 21c75c
import hawkey
Packit Service 21c75c
import logging
Packit Service 21c75c
import libdnf.repo
Packit Service 21c75c
import os
Packit Service 21c75c
Packit Service 21c75c
APPLYDELTA = '/usr/bin/applydeltarpm'
Packit Service 21c75c
Packit Service 21c75c
logger = logging.getLogger("dnf")
Packit Service 21c75c
Packit Service 21c75c
Packit Service 21c75c
class DeltaPayload(dnf.repo.PackagePayload):
Packit Service 21c75c
    def __init__(self, delta_info, delta, pkg, progress):
Packit Service 21c75c
        super(DeltaPayload, self).__init__(pkg, progress)
Packit Service 21c75c
        self.delta_info = delta_info
Packit Service 21c75c
        self.delta = delta
Packit Service 21c75c
Packit Service 21c75c
    def __str__(self):
Packit Service 21c75c
        return os.path.basename(self.delta.location)
Packit Service 21c75c
Packit Service 21c75c
    def _end_cb(self, cbdata, lr_status, msg):
Packit Service 21c75c
        super(DeltaPayload, self)._end_cb(cbdata, lr_status, msg)
Packit Service 21c75c
        if lr_status != libdnf.repo.PackageTargetCB.TransferStatus_ERROR:
Packit Service 21c75c
            self.delta_info.enqueue(self)
Packit Service 21c75c
Packit Service 21c75c
    def _target_params(self):
Packit Service 21c75c
        delta = self.delta
Packit Service 21c75c
        ctype, csum = delta.chksum
Packit Service 21c75c
        ctype = hawkey.chksum_name(ctype)
Packit Service 21c75c
        chksum = hexlify(csum).decode()
Packit Service 21c75c
Packit Service 21c75c
        ctype_code = libdnf.repo.PackageTarget.checksumType(ctype)
Packit Service 21c75c
        if ctype_code == libdnf.repo.PackageTarget.ChecksumType_UNKNOWN:
Packit Service 21c75c
            logger.warning(_("unsupported checksum type: %s"), ctype)
Packit Service 21c75c
Packit Service 21c75c
        return {
Packit Service 21c75c
            'relative_url' : delta.location,
Packit Service 21c75c
            'checksum_type' : ctype_code,
Packit Service 21c75c
            'checksum' : chksum,
Packit Service 21c75c
            'expectedsize' : delta.downloadsize,
Packit Service 21c75c
            'base_url' : delta.baseurl,
Packit Service 21c75c
        }
Packit Service 21c75c
Packit Service 21c75c
    @property
Packit Service 21c75c
    def download_size(self):
Packit Service 21c75c
        return self.delta.downloadsize
Packit Service 21c75c
Packit Service 21c75c
    @property
Packit Service 21c75c
    def _full_size(self):
Packit Service 21c75c
        return self.pkg.downloadsize
Packit Service 21c75c
Packit Service 21c75c
    def localPkg(self):
Packit Service 21c75c
        location = self.delta.location
Packit Service 21c75c
        return os.path.join(self.pkg.repo.pkgdir, os.path.basename(location))
Packit Service 21c75c
Packit Service 21c75c
Packit Service 21c75c
class DeltaInfo(object):
Packit Service 21c75c
    def __init__(self, query, progress, deltarpm_percentage=None):
Packit Service 21c75c
        '''A delta lookup and rebuild context
Packit Service 21c75c
           query -- installed packages to use when looking up deltas
Packit Service 21c75c
           progress -- progress obj to display finished delta rebuilds
Packit Service 21c75c
        '''
Packit Service 21c75c
        self.deltarpm_installed = False
Packit Service 21c75c
        if os.access(APPLYDELTA, os.X_OK):
Packit Service 21c75c
            self.deltarpm_installed = True
Packit Service 21c75c
        try:
Packit Service 21c75c
            self.deltarpm_jobs = os.sysconf('SC_NPROCESSORS_ONLN')
Packit Service 21c75c
        except (TypeError, ValueError):
Packit Service 21c75c
            self.deltarpm_jobs = 4
Packit Service 21c75c
        if deltarpm_percentage is None:
Packit Service 21c75c
            self.deltarpm_percentage = dnf.conf.Conf().deltarpm_percentage
Packit Service 21c75c
        else:
Packit Service 21c75c
            self.deltarpm_percentage = deltarpm_percentage
Packit Service 21c75c
        self.query = query
Packit Service 21c75c
        self.progress = progress
Packit Service 21c75c
Packit Service 21c75c
        self.queue = []
Packit Service 21c75c
        self.jobs = {}
Packit Service 21c75c
        self.err = {}
Packit Service 21c75c
Packit Service 21c75c
    def delta_factory(self, po, progress):
Packit Service 21c75c
        '''Turn a po to Delta RPM po, if possible'''
Packit Service 21c75c
        if not self.deltarpm_installed:
Packit Service 21c75c
            # deltarpm is not installed
Packit Service 21c75c
            return None
Packit Service 21c75c
        if not po.repo.deltarpm or not self.deltarpm_percentage:
Packit Service 21c75c
            # drpm disabled
Packit Service 21c75c
            return None
Packit Service 21c75c
        if po._is_local_pkg():
Packit Service 21c75c
            # drpm disabled for local
Packit Service 21c75c
            return None
Packit Service 21c75c
        if os.path.exists(po.localPkg()):
Packit Service 21c75c
            # already there
Packit Service 21c75c
            return None
Packit Service 21c75c
Packit Service 21c75c
        best = po._size * self.deltarpm_percentage / 100
Packit Service 21c75c
        best_delta = None
Packit Service 21c75c
        for ipo in self.query.filter(name=po.name, arch=po.arch):
Packit Service 21c75c
            delta = po.get_delta_from_evr(ipo.evr)
Packit Service 21c75c
            if delta and delta.downloadsize < best:
Packit Service 21c75c
                best = delta.downloadsize
Packit Service 21c75c
                best_delta = delta
Packit Service 21c75c
        if best_delta:
Packit Service 21c75c
            return DeltaPayload(self, best_delta, po, progress)
Packit Service 21c75c
        return None
Packit Service 21c75c
Packit Service 21c75c
    def job_done(self, pid, code):
Packit Service 21c75c
        # handle a finished delta rebuild
Packit Service 21c75c
        logger.log(dnf.logging.SUBDEBUG, 'drpm: %d: return code: %d, %d', pid,
Packit Service 21c75c
                   code >> 8, code & 0xff)
Packit Service 21c75c
Packit Service 21c75c
        pload = self.jobs.pop(pid)
Packit Service 21c75c
        pkg = pload.pkg
Packit Service 21c75c
        if code != 0:
Packit Service 21c75c
            unlink_f(pload.pkg.localPkg())
Packit Service 21c75c
            self.err[pkg] = [_('Delta RPM rebuild failed')]
Packit Service 21c75c
        elif not pload.pkg.verifyLocalPkg():
Packit Service 21c75c
            self.err[pkg] = [_('Checksum of the delta-rebuilt RPM failed')]
Packit Service 21c75c
        else:
Packit Service 21c75c
            os.unlink(pload.localPkg())
Packit Service 21c75c
            self.progress.end(pload, dnf.callback.STATUS_DRPM, _('done'))
Packit Service 21c75c
Packit Service 21c75c
    def start_job(self, pload):
Packit Service 21c75c
        # spawn a delta rebuild job
Packit Service 21c75c
        spawn_args = [APPLYDELTA, APPLYDELTA,
Packit Service 21c75c
                      '-a', pload.pkg.arch,
Packit Service 21c75c
                      pload.localPkg(), pload.pkg.localPkg()]
Packit Service 21c75c
        pid = os.spawnl(os.P_NOWAIT, *spawn_args)
Packit Service 21c75c
        logger.log(dnf.logging.SUBDEBUG, 'drpm: spawned %d: %s', pid,
Packit Service 21c75c
                   ' '.join(spawn_args[1:]))
Packit Service 21c75c
        self.jobs[pid] = pload
Packit Service 21c75c
Packit Service 21c75c
    def enqueue(self, pload):
Packit Service 21c75c
        # process finished jobs, start new ones
Packit Service 21c75c
        while self.jobs:
Packit Service 21c75c
            pid, code = os.waitpid(-1, os.WNOHANG)
Packit Service 21c75c
            if not pid:
Packit Service 21c75c
                break
Packit Service 21c75c
            self.job_done(pid, code)
Packit Service 21c75c
        self.queue.append(pload)
Packit Service 21c75c
        while len(self.jobs) < self.deltarpm_jobs:
Packit Service 21c75c
            self.start_job(self.queue.pop(0))
Packit Service 21c75c
            if not self.queue:
Packit Service 21c75c
                break
Packit Service 21c75c
Packit Service 21c75c
    def wait(self):
Packit Service 21c75c
        '''Wait until all jobs have finished'''
Packit Service 21c75c
        while self.jobs:
Packit Service 21c75c
            pid, code = os.wait()
Packit Service 21c75c
            self.job_done(pid, code)
Packit Service 21c75c
            if self.queue:
Packit Service 21c75c
                self.start_job(self.queue.pop(0))