# repo.py

# DNF Repository objects.

#

# Copyright (C) 2013-2016 Red Hat, Inc.

#

# This copyrighted material is made available to anyone wishing to use,

# modify, copy, or redistribute it subject to the terms and conditions of

# the GNU General Public License v.2, or (at your option) any later version.

# This program is distributed in the hope that it will be useful, but WITHOUT

# ANY WARRANTY expressed or implied, including the implied warranties of

# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General

# Public License for more details.  You should have received a copy of the

# GNU General Public License along with this program; if not, write to the

# Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA

# 02110-1301, USA.  Any Red Hat trademarks that are incorporated in the

# source code or documentation are not subject to the GNU General Public

# License and may only be used or replicated with the express permission of

# Red Hat, Inc.

#

from __future__ import absolute_import

from __future__ import unicode_literals

from dnf.i18n import ucd, _

import dnf.callback

import dnf.conf

import dnf.conf.substitutions

import dnf.const

import dnf.crypto

import dnf.exceptions

import dnf.logging

import dnf.pycomp

import dnf.util

import dnf.yum.misc

import libdnf.error

import libdnf.repo

import functools

import hashlib

import hawkey

import logging

import operator

import os

import re

import shutil

import string

import sys

import time

import traceback

_PACKAGES_RELATIVE_DIR = "packages"

_MIRRORLIST_FILENAME = "mirrorlist"

# Chars allowed in a repo ID

_REPOID_CHARS = string.ascii_letters + string.digits + '-_.:'

# Regex pattern that matches a repo cachedir and captures the repo ID

_CACHEDIR_RE = r'(?P<repoid>[%s]+)\-[%s]{16}' % (re.escape(_REPOID_CHARS),

                                                 string.hexdigits)

# Regex patterns matching any filename that is repo-specific cache data of a

# particular type.  The filename is expected to not contain the base cachedir

# path components.

CACHE_FILES = {

    'metadata': r'^%s\/.*((xml|yaml)(\.gz|\.xz|\.bz2|.zck)?|asc|cachecookie|%s)$' %

                (_CACHEDIR_RE, _MIRRORLIST_FILENAME),

    'packages': r'^%s\/%s\/.+rpm$' % (_CACHEDIR_RE, _PACKAGES_RELATIVE_DIR),

    'dbcache': r'^.+(solv|solvx)$',

}

logger = logging.getLogger("dnf")

def repo_id_invalid(repo_id):

    # :api

    """Return index of an invalid character in the repo ID (if present)."""

    first_invalid = libdnf.repo.Repo.verifyId(repo_id)

    return None if first_invalid < 0 else first_invalid

def _pkg2payload(pkg, progress, *factories):

    for fn in factories:

        pload = fn(pkg, progress)

        if pload is not None:

            return pload

    raise ValueError(_('no matching payload factory for %s') % pkg)

def _download_payloads(payloads, drpm):

    # download packages

    def _download_sort_key(payload):

        return not hasattr(payload, 'delta')

    drpm.err.clear()

    targets = [pload._librepo_target()

               for pload in sorted(payloads, key=_download_sort_key)]

    errs = _DownloadErrors()

    try:

        libdnf.repo.PackageTarget.downloadPackages(libdnf.repo.VectorPPackageTarget(targets), True)

    except RuntimeError as e:

        errs._fatal = str(e)

    drpm.wait()

    # process downloading errors

    errs._recoverable = drpm.err.copy()

    for tgt in targets:

        err = tgt.getErr()

        if err is None or err.startswith('Not finished'):

            continue

        callbacks = tgt.getCallbacks()

        payload = callbacks.package_pload

        pkg = payload.pkg

        if err == _('Already downloaded'):

            errs._skipped.add(pkg)

            continue

        pkg.repo._repo.expire()

        errs._irrecoverable[pkg] = [err]

    return errs

def _update_saving(saving, payloads, errs):

    real, full = saving

    for pload in payloads:

        pkg = pload.pkg

        if pkg in errs:

            real += pload.download_size

            continue

        real += pload.download_size

        full += pload._full_size

    return real, full

class _DownloadErrors(object):

    def __init__(self):

        self._val_irrecoverable = {}

        self._val_recoverable = {}

        self._fatal = None

        self._skipped = set()

    @property

    def _irrecoverable(self):

        if self._val_irrecoverable:

            return self._val_irrecoverable

        if self._fatal:

            return {'': [self._fatal]}

        return {}

    @property

    def _recoverable(self):

        return self._val_recoverable

    @_recoverable.setter

    def _recoverable(self, new_dct):

        self._val_recoverable = new_dct

    def _bandwidth_used(self, pload):

        if pload.pkg in self._skipped:

            return 0

        return pload.download_size

class _DetailedLibrepoError(Exception):

    def __init__(self, librepo_err, source_url):

        Exception.__init__(self)

        self.librepo_code = librepo_err.args[0]

        self.librepo_msg = librepo_err.args[1]

        self.source_url = source_url

class _NullKeyImport(dnf.callback.KeyImport):

    def _confirm(self, id, userid, fingerprint, url, timestamp):

        return True

class Metadata(object):

    def __init__(self, repo):

        self._repo = repo

    @property

    def fresh(self):

        # :api

        return self._repo.fresh()

class PackageTargetCallbacks(libdnf.repo.PackageTargetCB):

    def __init__(self, package_pload):

        super(PackageTargetCallbacks, self).__init__()

        self.package_pload = package_pload

    def end(self, status, msg):

        self.package_pload._end_cb(None, status, msg)

        return 0

    def progress(self, totalToDownload, downloaded):

        self.package_pload._progress_cb(None, totalToDownload, downloaded)

        return 0

    def mirrorFailure(self, msg, url):

        self.package_pload._mirrorfail_cb(None, msg, url)

        return 0

class PackagePayload(dnf.callback.Payload):

    def __init__(self, pkg, progress):

        super(PackagePayload, self).__init__(progress)

        self.callbacks = PackageTargetCallbacks(self)

        self.pkg = pkg

    def _end_cb(self, cbdata, lr_status, msg):

        """End callback to librepo operation."""

        status = dnf.callback.STATUS_FAILED

        if msg is None:

            status = dnf.callback.STATUS_OK

        elif msg.startswith('Not finished'):

            return

        elif lr_status == libdnf.repo.PackageTargetCB.TransferStatus_ALREADYEXISTS:

            status = dnf.callback.STATUS_ALREADY_EXISTS

        self.progress.end(self, status, msg)

    def _mirrorfail_cb(self, cbdata, err, url):

        self.progress.end(self, dnf.callback.STATUS_MIRROR, err)

    def _progress_cb(self, cbdata, total, done):

        try:

            self.progress.progress(self, done)

        except Exception:

            exc_type, exc_value, exc_traceback = sys.exc_info()

            except_list = traceback.format_exception(exc_type, exc_value, exc_traceback)

            logger.critical(''.join(except_list))

    @property

    def _full_size(self):

        return self.download_size

    def _librepo_target(self):

        pkg = self.pkg

        pkgdir = pkg.pkgdir

        dnf.util.ensure_dir(pkgdir)

        target_dct = {

            'dest': pkgdir,

            'resume': True,

            'cbdata': self,

            'progresscb': self._progress_cb,

            'endcb': self._end_cb,

            'mirrorfailurecb': self._mirrorfail_cb,

        }

        target_dct.update(self._target_params())

        return libdnf.repo.PackageTarget(

            pkg.repo._repo,

            target_dct['relative_url'],

            target_dct['dest'], target_dct['checksum_type'], target_dct['checksum'],

            target_dct['expectedsize'], target_dct['base_url'], target_dct['resume'],

            0, 0, self.callbacks)

class RPMPayload(PackagePayload):

    def __str__(self):

        return os.path.basename(self.pkg.location)

    def _target_params(self):

        pkg = self.pkg

        ctype, csum = pkg.returnIdSum()

        ctype_code = libdnf.repo.PackageTarget.checksumType(ctype)

        if ctype_code == libdnf.repo.PackageTarget.ChecksumType_UNKNOWN:

            logger.warning(_("unsupported checksum type: %s"), ctype)

        return {

            'relative_url': pkg.location,

            'checksum_type': ctype_code,

            'checksum': csum,

            'expectedsize': pkg.downloadsize,

            'base_url': pkg.baseurl,

        }

    @property

    def download_size(self):

        """Total size of the download."""

        return self.pkg.downloadsize

class RemoteRPMPayload(PackagePayload):

    def __init__(self, remote_location, conf, progress):

        super(RemoteRPMPayload, self).__init__("unused_object", progress)

        self.remote_location = remote_location

        self.remote_size = 0

        self.conf = conf

        s = (self.conf.releasever or "") + self.conf.substitutions.get('basearch')

        digest = hashlib.sha256(s.encode('utf8')).hexdigest()[:16]

        repodir = "commandline-" + digest

        self.pkgdir = os.path.join(self.conf.cachedir, repodir, "packages")

        dnf.util.ensure_dir(self.pkgdir)

        self.local_path = os.path.join(self.pkgdir, self.__str__().lstrip("/"))

    def __str__(self):

        return os.path.basename(self.remote_location)

    def _progress_cb(self, cbdata, total, done):

        self.remote_size = total

        try:

            self.progress.progress(self, done)

        except Exception:

            exc_type, exc_value, exc_traceback = sys.exc_info()

            except_list = traceback.format_exception(exc_type, exc_value, exc_traceback)

            logger.critical(''.join(except_list))

    def _librepo_target(self):

        return libdnf.repo.PackageTarget(

            self.conf._config, os.path.basename(self.remote_location),

            self.pkgdir, 0, None, 0, os.path.dirname(self.remote_location),

            True, 0, 0, self.callbacks)

    @property

    def download_size(self):

        """Total size of the download."""

        return self.remote_size

class MDPayload(dnf.callback.Payload):

    def __init__(self, progress):

        super(MDPayload, self).__init__(progress)

        self._text = ""

        self._download_size = 0

        self.fastest_mirror_running = False

        self.mirror_failures = set()

    def __str__(self):

        if dnf.pycomp.PY3:

            return self._text

        else:

            return self._text.encode('utf-8')

    def __unicode__(self):

        return self._text

    def _progress_cb(self, cbdata, total, done):

        self._download_size = total

        self.progress.progress(self, done)

    def _fastestmirror_cb(self, cbdata, stage, data):

        if stage == libdnf.repo.RepoCB.FastestMirrorStage_DETECTION:

            # pinging mirrors, this might take a while

            msg = _('determining the fastest mirror (%s hosts).. ') % data

            self.fastest_mirror_running = True

        elif stage == libdnf.repo.RepoCB.FastestMirrorStage_STATUS and self.fastest_mirror_running:

            # done.. report but ignore any errors

            msg = 'error: %s\n' % data if data else 'done.\n'

        else:

            return

        self.progress.message(msg)

    def _mirror_failure_cb(self, cbdata, msg, url, metadata):

        self.mirror_failures.add(msg)

        msg = 'error: %s (%s).' % (msg, url)

        logger.debug(msg)

    @property

    def download_size(self):

        return self._download_size

    @property

    def progress(self):

        return self._progress

    @progress.setter

    def progress(self, progress):

        if progress is None:

            progress = dnf.callback.NullDownloadProgress()

        self._progress = progress

    def start(self, text):

        self._text = text

        self.progress.start(1, 0)

    def end(self):

        self._download_size = 0

        self.progress.end(self, None, None)

# use the local cache even if it's expired. download if there's no cache.

SYNC_LAZY = libdnf.repo.Repo.SyncStrategy_LAZY

# use the local cache, even if it's expired, never download.

SYNC_ONLY_CACHE = libdnf.repo.Repo.SyncStrategy_ONLY_CACHE

# try the cache, if it is expired download new md.

SYNC_TRY_CACHE = libdnf.repo.Repo.SyncStrategy_TRY_CACHE

class RepoCallbacks(libdnf.repo.RepoCB):

    def __init__(self, repo):

        super(RepoCallbacks, self).__init__()

        self._repo = repo

        self._md_pload = repo._md_pload

    def start(self, what):

        self._md_pload.start(what)

    def end(self):

        self._md_pload.end()

    def progress(self, totalToDownload, downloaded):

        self._md_pload._progress_cb(None, totalToDownload, downloaded)

        return 0

    def fastestMirror(self, stage, ptr):

        self._md_pload._fastestmirror_cb(None, stage, ptr)

    def handleMirrorFailure(self, msg, url, metadata):

        self._md_pload._mirror_failure_cb(None, msg, url, metadata)

        return 0

    def repokeyImport(self, id, userid, fingerprint, url, timestamp):

        return self._repo._key_import._confirm(id, userid, fingerprint, url, timestamp)

class Repo(dnf.conf.RepoConf):

    # :api

    DEFAULT_SYNC = SYNC_TRY_CACHE

    def __init__(self, name=None, parent_conf=None):

        # :api

        super(Repo, self).__init__(section=name, parent=parent_conf)

        self._config.this.disown()  # _repo will be the owner of _config

        self._repo = libdnf.repo.Repo(name if name else "", self._config)

        self._md_pload = MDPayload(dnf.callback.NullDownloadProgress())

        self._callbacks = RepoCallbacks(self)

        self._callbacks.this.disown()  # _repo will be the owner of callbacks

        self._repo.setCallbacks(self._callbacks)

        self._pkgdir = None

        self._key_import = _NullKeyImport()

        self.metadata = None  # :api

        self._repo.setSyncStrategy(self.DEFAULT_SYNC)

        if parent_conf:

            self._repo.setSubstitutions(parent_conf.substitutions)

        self._substitutions = dnf.conf.substitutions.Substitutions()

        self._check_config_file_age = parent_conf.check_config_file_age \

            if parent_conf is not None else True

    @property

    def id(self):

        # :api

        return self._repo.getId()

    @property

    def repofile(self):

        # :api

        return self._repo.getRepoFilePath()

    @repofile.setter

    def repofile(self, value):

        self._repo.setRepoFilePath(value)

    @property

    def pkgdir(self):

        # :api

        if self._repo.isLocal():

            return self._repo.getLocalBaseurl()

        return self.cache_pkgdir()

    def cache_pkgdir(self):

        if self._pkgdir is not None:

            return self._pkgdir

        return os.path.join(self._repo.getCachedir(), _PACKAGES_RELATIVE_DIR)

    @pkgdir.setter

    def pkgdir(self, val):

        # :api

        self._pkgdir = val

    @property

    def _pubring_dir(self):

        return os.path.join(self._repo.getCachedir(), 'pubring')

    @property

    def load_metadata_other(self):

        return self._repo.getLoadMetadataOther()

    @load_metadata_other.setter

    def load_metadata_other(self, val):

        self._repo.setLoadMetadataOther(val)

    def __lt__(self, other):

        return self.id < other.id

    def __repr__(self):

        return "<%s %s>" % (self.__class__.__name__, self.id)

    def __setattr__(self, name, value):

        super(Repo, self).__setattr__(name, value)

    def disable(self):

        # :api

        self._repo.disable()

    def enable(self):

        # :api

        self._repo.enable()

    def add_metadata_type_to_download(self, metadata_type):

        # :api

        """Ask for additional repository metadata type to download.

        Given metadata_type is appended to the default metadata set when

        repository is downloaded.

        Parameters

        ----------

        metadata_type: string

        Example: add_metadata_type_to_download("productid")

        """

        self._repo.addMetadataTypeToDownload(metadata_type)

    def remove_metadata_type_from_download(self, metadata_type):

        # :api

        """Stop asking for this additional repository metadata type

        in download.

        Given metadata_type is no longer downloaded by default

        when this repository is downloaded.

        Parameters

        ----------

        metadata_type: string

        Example: remove_metadata_type_from_download("productid")

        """

        self._repo.removeMetadataTypeFromDownload(metadata_type)

    def get_metadata_path(self, metadata_type):

        # :api

        """Return path to the file with downloaded repository metadata of given type.

        Parameters

        ----------

        metadata_type: string

        """

        return self._repo.getMetadataPath(metadata_type)

    def get_metadata_content(self, metadata_type):

        # :api

        """Return content of the file with downloaded repository metadata of given type.

        Content of compressed metadata file is returned uncompressed.

        Parameters

        ----------

        metadata_type: string

        """

        return self._repo.getMetadataContent(metadata_type)

    def load(self):

        # :api

        """Load the metadata for this repo.

        Depending on the configuration and the age and consistence of data

        available on the disk cache, either loads the metadata from the cache or

        downloads them from the mirror, baseurl or metalink.

        This method will by default not try to refresh already loaded data if

        called repeatedly.

        Returns True if this call to load() caused a fresh metadata download.

        """

        ret = False

        try:

            ret = self._repo.load()

        except (libdnf.error.Error, RuntimeError) as e:

            if self._md_pload.mirror_failures:

                msg = "Errors during downloading metadata for repository '%s':" % self.id

                for failure in self._md_pload.mirror_failures:

                    msg += "\n  - %s" % failure

                logger.warning(msg)

            raise dnf.exceptions.RepoError(str(e))

        finally:

            self._md_pload.mirror_failures = set()

        self.metadata = Metadata(self._repo)

        return ret

    def _metadata_expire_in(self):

        """Get the number of seconds after which the cached metadata will expire.

        Returns a tuple, boolean whether there even is cached metadata and the

        number of seconds it will expire in. Negative number means the metadata

        has expired already, None that it never expires.

        """

        if not self.metadata:

            self._repo.loadCache(False)

        if self.metadata:

            if self.metadata_expire == -1:

                return True, None

            expiration = self._repo.getExpiresIn()

            if self._repo.isExpired():

                expiration = min(0, expiration)

            return True, expiration

        return False, 0

    def _set_key_import(self, key_import):

        self._key_import = key_import

    def set_progress_bar(self, progress):

        # :api

        self._md_pload.progress = progress

    def get_http_headers(self):

        # :api

        """Returns user defined http headers.

        Returns

        -------

        headers : tuple of strings

        """

        return self._repo.getHttpHeaders()

    def set_http_headers(self, headers):

        # :api

        """Sets http headers.

        Sets new http headers and rewrites existing ones.

        Parameters

        ----------

        headers : tuple or list of strings

            Example: set_http_headers(["User-Agent: Agent007", "MyFieldName: MyFieldValue"])

        """

        self._repo.setHttpHeaders(headers)

    def remote_location(self, location, schemes=('http', 'ftp', 'file', 'https')):

        """

        :param location: relative location inside the repo

        :param schemes: list of allowed protocols. Default is ('http', 'ftp', 'file', 'https')

        :return: absolute url (string) or None

        """

        def schemes_filter(url_list):

            for url in url_list:

                if schemes:

                    s = dnf.pycomp.urlparse.urlparse(url)[0]

                    if s in schemes:

                        return os.path.join(url, location.lstrip('/'))

                else:

                    return os.path.join(url, location.lstrip('/'))

            return None

        if not location:

            return None

        mirrors = self._repo.getMirrors()

        if mirrors:

            return schemes_filter(mirrors)

        elif self.baseurl:

            return schemes_filter(self.baseurl)