|
Packit Service |
21c75c |
# drpm.py
|
|
Packit Service |
21c75c |
# Delta RPM support
|
|
Packit Service |
21c75c |
#
|
|
Packit Service |
21c75c |
# Copyright (C) 2012-2016 Red Hat, Inc.
|
|
Packit Service |
21c75c |
#
|
|
Packit Service |
21c75c |
# This copyrighted material is made available to anyone wishing to use,
|
|
Packit Service |
21c75c |
# modify, copy, or redistribute it subject to the terms and conditions of
|
|
Packit Service |
21c75c |
# the GNU General Public License v.2, or (at your option) any later version.
|
|
Packit Service |
21c75c |
# This program is distributed in the hope that it will be useful, but WITHOUT
|
|
Packit Service |
21c75c |
# ANY WARRANTY expressed or implied, including the implied warranties of
|
|
Packit Service |
21c75c |
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU General
|
|
Packit Service |
21c75c |
# Public License for more details. You should have received a copy of the
|
|
Packit Service |
21c75c |
# GNU General Public License along with this program; if not, write to the
|
|
Packit Service |
21c75c |
# Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
|
|
Packit Service |
21c75c |
# 02110-1301, USA. Any Red Hat trademarks that are incorporated in the
|
|
Packit Service |
21c75c |
# source code or documentation are not subject to the GNU General Public
|
|
Packit Service |
21c75c |
# License and may only be used or replicated with the express permission of
|
|
Packit Service |
21c75c |
# Red Hat, Inc.
|
|
Packit Service |
21c75c |
#
|
|
Packit Service |
21c75c |
|
|
Packit Service |
21c75c |
from __future__ import absolute_import
|
|
Packit Service |
21c75c |
from __future__ import unicode_literals
|
|
Packit Service |
21c75c |
from binascii import hexlify
|
|
Packit Service |
21c75c |
from dnf.yum.misc import unlink_f
|
|
Packit Service |
21c75c |
from dnf.i18n import _
|
|
Packit Service |
21c75c |
|
|
Packit Service |
21c75c |
import dnf.callback
|
|
Packit Service |
21c75c |
import dnf.logging
|
|
Packit Service |
21c75c |
import dnf.repo
|
|
Packit Service |
21c75c |
import hawkey
|
|
Packit Service |
21c75c |
import logging
|
|
Packit Service |
21c75c |
import libdnf.repo
|
|
Packit Service |
21c75c |
import os
|
|
Packit Service |
21c75c |
|
|
Packit Service |
21c75c |
APPLYDELTA = '/usr/bin/applydeltarpm'
|
|
Packit Service |
21c75c |
|
|
Packit Service |
21c75c |
logger = logging.getLogger("dnf")
|
|
Packit Service |
21c75c |
|
|
Packit Service |
21c75c |
|
|
Packit Service |
21c75c |
class DeltaPayload(dnf.repo.PackagePayload):
|
|
Packit Service |
21c75c |
def __init__(self, delta_info, delta, pkg, progress):
|
|
Packit Service |
21c75c |
super(DeltaPayload, self).__init__(pkg, progress)
|
|
Packit Service |
21c75c |
self.delta_info = delta_info
|
|
Packit Service |
21c75c |
self.delta = delta
|
|
Packit Service |
21c75c |
|
|
Packit Service |
21c75c |
def __str__(self):
|
|
Packit Service |
21c75c |
return os.path.basename(self.delta.location)
|
|
Packit Service |
21c75c |
|
|
Packit Service |
21c75c |
def _end_cb(self, cbdata, lr_status, msg):
|
|
Packit Service |
21c75c |
super(DeltaPayload, self)._end_cb(cbdata, lr_status, msg)
|
|
Packit Service |
21c75c |
if lr_status != libdnf.repo.PackageTargetCB.TransferStatus_ERROR:
|
|
Packit Service |
21c75c |
self.delta_info.enqueue(self)
|
|
Packit Service |
21c75c |
|
|
Packit Service |
21c75c |
def _target_params(self):
|
|
Packit Service |
21c75c |
delta = self.delta
|
|
Packit Service |
21c75c |
ctype, csum = delta.chksum
|
|
Packit Service |
21c75c |
ctype = hawkey.chksum_name(ctype)
|
|
Packit Service |
21c75c |
chksum = hexlify(csum).decode()
|
|
Packit Service |
21c75c |
|
|
Packit Service |
21c75c |
ctype_code = libdnf.repo.PackageTarget.checksumType(ctype)
|
|
Packit Service |
21c75c |
if ctype_code == libdnf.repo.PackageTarget.ChecksumType_UNKNOWN:
|
|
Packit Service |
21c75c |
logger.warning(_("unsupported checksum type: %s"), ctype)
|
|
Packit Service |
21c75c |
|
|
Packit Service |
21c75c |
return {
|
|
Packit Service |
21c75c |
'relative_url' : delta.location,
|
|
Packit Service |
21c75c |
'checksum_type' : ctype_code,
|
|
Packit Service |
21c75c |
'checksum' : chksum,
|
|
Packit Service |
21c75c |
'expectedsize' : delta.downloadsize,
|
|
Packit Service |
21c75c |
'base_url' : delta.baseurl,
|
|
Packit Service |
21c75c |
}
|
|
Packit Service |
21c75c |
|
|
Packit Service |
21c75c |
@property
|
|
Packit Service |
21c75c |
def download_size(self):
|
|
Packit Service |
21c75c |
return self.delta.downloadsize
|
|
Packit Service |
21c75c |
|
|
Packit Service |
21c75c |
@property
|
|
Packit Service |
21c75c |
def _full_size(self):
|
|
Packit Service |
21c75c |
return self.pkg.downloadsize
|
|
Packit Service |
21c75c |
|
|
Packit Service |
21c75c |
def localPkg(self):
|
|
Packit Service |
21c75c |
location = self.delta.location
|
|
Packit Service |
21c75c |
return os.path.join(self.pkg.repo.pkgdir, os.path.basename(location))
|
|
Packit Service |
21c75c |
|
|
Packit Service |
21c75c |
|
|
Packit Service |
21c75c |
class DeltaInfo(object):
|
|
Packit Service |
21c75c |
def __init__(self, query, progress, deltarpm_percentage=None):
|
|
Packit Service |
21c75c |
'''A delta lookup and rebuild context
|
|
Packit Service |
21c75c |
query -- installed packages to use when looking up deltas
|
|
Packit Service |
21c75c |
progress -- progress obj to display finished delta rebuilds
|
|
Packit Service |
21c75c |
'''
|
|
Packit Service |
21c75c |
self.deltarpm_installed = False
|
|
Packit Service |
21c75c |
if os.access(APPLYDELTA, os.X_OK):
|
|
Packit Service |
21c75c |
self.deltarpm_installed = True
|
|
Packit Service |
21c75c |
try:
|
|
Packit Service |
21c75c |
self.deltarpm_jobs = os.sysconf('SC_NPROCESSORS_ONLN')
|
|
Packit Service |
21c75c |
except (TypeError, ValueError):
|
|
Packit Service |
21c75c |
self.deltarpm_jobs = 4
|
|
Packit Service |
21c75c |
if deltarpm_percentage is None:
|
|
Packit Service |
21c75c |
self.deltarpm_percentage = dnf.conf.Conf().deltarpm_percentage
|
|
Packit Service |
21c75c |
else:
|
|
Packit Service |
21c75c |
self.deltarpm_percentage = deltarpm_percentage
|
|
Packit Service |
21c75c |
self.query = query
|
|
Packit Service |
21c75c |
self.progress = progress
|
|
Packit Service |
21c75c |
|
|
Packit Service |
21c75c |
self.queue = []
|
|
Packit Service |
21c75c |
self.jobs = {}
|
|
Packit Service |
21c75c |
self.err = {}
|
|
Packit Service |
21c75c |
|
|
Packit Service |
21c75c |
def delta_factory(self, po, progress):
|
|
Packit Service |
21c75c |
'''Turn a po to Delta RPM po, if possible'''
|
|
Packit Service |
21c75c |
if not self.deltarpm_installed:
|
|
Packit Service |
21c75c |
# deltarpm is not installed
|
|
Packit Service |
21c75c |
return None
|
|
Packit Service |
21c75c |
if not po.repo.deltarpm or not self.deltarpm_percentage:
|
|
Packit Service |
21c75c |
# drpm disabled
|
|
Packit Service |
21c75c |
return None
|
|
Packit Service |
21c75c |
if po._is_local_pkg():
|
|
Packit Service |
21c75c |
# drpm disabled for local
|
|
Packit Service |
21c75c |
return None
|
|
Packit Service |
21c75c |
if os.path.exists(po.localPkg()):
|
|
Packit Service |
21c75c |
# already there
|
|
Packit Service |
21c75c |
return None
|
|
Packit Service |
21c75c |
|
|
Packit Service |
21c75c |
best = po._size * self.deltarpm_percentage / 100
|
|
Packit Service |
21c75c |
best_delta = None
|
|
Packit Service |
21c75c |
for ipo in self.query.filter(name=po.name, arch=po.arch):
|
|
Packit Service |
21c75c |
delta = po.get_delta_from_evr(ipo.evr)
|
|
Packit Service |
21c75c |
if delta and delta.downloadsize < best:
|
|
Packit Service |
21c75c |
best = delta.downloadsize
|
|
Packit Service |
21c75c |
best_delta = delta
|
|
Packit Service |
21c75c |
if best_delta:
|
|
Packit Service |
21c75c |
return DeltaPayload(self, best_delta, po, progress)
|
|
Packit Service |
21c75c |
return None
|
|
Packit Service |
21c75c |
|
|
Packit Service |
21c75c |
def job_done(self, pid, code):
|
|
Packit Service |
21c75c |
# handle a finished delta rebuild
|
|
Packit Service |
21c75c |
logger.log(dnf.logging.SUBDEBUG, 'drpm: %d: return code: %d, %d', pid,
|
|
Packit Service |
21c75c |
code >> 8, code & 0xff)
|
|
Packit Service |
21c75c |
|
|
Packit Service |
21c75c |
pload = self.jobs.pop(pid)
|
|
Packit Service |
21c75c |
pkg = pload.pkg
|
|
Packit Service |
21c75c |
if code != 0:
|
|
Packit Service |
21c75c |
unlink_f(pload.pkg.localPkg())
|
|
Packit Service |
21c75c |
self.err[pkg] = [_('Delta RPM rebuild failed')]
|
|
Packit Service |
21c75c |
elif not pload.pkg.verifyLocalPkg():
|
|
Packit Service |
21c75c |
self.err[pkg] = [_('Checksum of the delta-rebuilt RPM failed')]
|
|
Packit Service |
21c75c |
else:
|
|
Packit Service |
21c75c |
os.unlink(pload.localPkg())
|
|
Packit Service |
21c75c |
self.progress.end(pload, dnf.callback.STATUS_DRPM, _('done'))
|
|
Packit Service |
21c75c |
|
|
Packit Service |
21c75c |
def start_job(self, pload):
|
|
Packit Service |
21c75c |
# spawn a delta rebuild job
|
|
Packit Service |
21c75c |
spawn_args = [APPLYDELTA, APPLYDELTA,
|
|
Packit Service |
21c75c |
'-a', pload.pkg.arch,
|
|
Packit Service |
21c75c |
pload.localPkg(), pload.pkg.localPkg()]
|
|
Packit Service |
21c75c |
pid = os.spawnl(os.P_NOWAIT, *spawn_args)
|
|
Packit Service |
21c75c |
logger.log(dnf.logging.SUBDEBUG, 'drpm: spawned %d: %s', pid,
|
|
Packit Service |
21c75c |
' '.join(spawn_args[1:]))
|
|
Packit Service |
21c75c |
self.jobs[pid] = pload
|
|
Packit Service |
21c75c |
|
|
Packit Service |
21c75c |
def enqueue(self, pload):
|
|
Packit Service |
21c75c |
# process finished jobs, start new ones
|
|
Packit Service |
21c75c |
while self.jobs:
|
|
Packit Service |
21c75c |
pid, code = os.waitpid(-1, os.WNOHANG)
|
|
Packit Service |
21c75c |
if not pid:
|
|
Packit Service |
21c75c |
break
|
|
Packit Service |
21c75c |
self.job_done(pid, code)
|
|
Packit Service |
21c75c |
self.queue.append(pload)
|
|
Packit Service |
21c75c |
while len(self.jobs) < self.deltarpm_jobs:
|
|
Packit Service |
21c75c |
self.start_job(self.queue.pop(0))
|
|
Packit Service |
21c75c |
if not self.queue:
|
|
Packit Service |
21c75c |
break
|
|
Packit Service |
21c75c |
|
|
Packit Service |
21c75c |
def wait(self):
|
|
Packit Service |
21c75c |
'''Wait until all jobs have finished'''
|
|
Packit Service |
21c75c |
while self.jobs:
|
|
Packit Service |
21c75c |
pid, code = os.wait()
|
|
Packit Service |
21c75c |
self.job_done(pid, code)
|
|
Packit Service |
21c75c |
if self.queue:
|
|
Packit Service |
21c75c |
self.start_job(self.queue.pop(0))
|