Blame taskcluster/taskgraph/files_changed.py

Packit f0b94e
# This Source Code Form is subject to the terms of the Mozilla Public
Packit f0b94e
# License, v. 2.0. If a copy of the MPL was not distributed with this
Packit f0b94e
# file, You can obtain one at http://mozilla.org/MPL/2.0/.
Packit f0b94e
Packit f0b94e
"""
Packit f0b94e
Support for optimizing tasks based on the set of files that have changed.
Packit f0b94e
"""
Packit f0b94e
Packit f0b94e
from __future__ import absolute_import, print_function, unicode_literals
Packit f0b94e
Packit f0b94e
import logging
Packit f0b94e
import requests
Packit f0b94e
from redo import retry
Packit f0b94e
from mozpack.path import match as mozpackmatch, join as join_path
Packit f0b94e
from mozversioncontrol import get_repository_object, InvalidRepoPath
Packit f0b94e
from subprocess import CalledProcessError
Packit f0b94e
from mozbuild.util import memoize
Packit f0b94e
Packit f0b94e
logger = logging.getLogger(__name__)
Packit f0b94e
Packit f0b94e
Packit f0b94e
@memoize
Packit f0b94e
def get_changed_files(repository, revision):
Packit f0b94e
    """
Packit f0b94e
    Get the set of files changed in the push headed by the given revision.
Packit f0b94e
    Responses are cached, so multiple calls with the same arguments are OK.
Packit f0b94e
    """
Packit f0b94e
    url = '%s/json-automationrelevance/%s' % (repository.rstrip('/'), revision)
Packit f0b94e
    logger.debug("Querying version control for metadata: %s", url)
Packit f0b94e
Packit f0b94e
    def get_automationrelevance():
Packit f0b94e
        response = requests.get(url, timeout=30)
Packit f0b94e
        return response.json()
Packit f0b94e
    contents = retry(get_automationrelevance, attempts=10, sleeptime=10)
Packit f0b94e
Packit f0b94e
    logger.debug('{} commits influencing task scheduling:'
Packit f0b94e
                 .format(len(contents['changesets'])))
Packit f0b94e
    changed_files = set()
Packit f0b94e
    for c in contents['changesets']:
Packit f0b94e
        desc = ""  # Support empty desc
Packit f0b94e
        if c['desc']:
Packit f0b94e
            desc = c['desc'].splitlines()[0].encode('ascii', 'ignore')
Packit f0b94e
        logger.debug(" {cset} {desc}".format(
Packit f0b94e
            cset=c['node'][0:12],
Packit f0b94e
            desc=desc))
Packit f0b94e
        changed_files |= set(c['files'])
Packit f0b94e
Packit f0b94e
    return changed_files
Packit f0b94e
Packit f0b94e
Packit f0b94e
def check(params, file_patterns):
Packit f0b94e
    """Determine whether any of the files changed in the indicated push to
Packit f0b94e
    https://hg.mozilla.org match any of the given file patterns."""
Packit f0b94e
    repository = params.get('head_repository')
Packit f0b94e
    revision = params.get('head_rev')
Packit f0b94e
    if not repository or not revision:
Packit f0b94e
        logger.warning("Missing `head_repository` or `head_rev` parameters; "
Packit f0b94e
                       "assuming all files have changed")
Packit f0b94e
        return True
Packit f0b94e
Packit f0b94e
    changed_files = get_changed_files(repository, revision)
Packit f0b94e
Packit f0b94e
    if 'comm_head_repository' in params:
Packit f0b94e
        repository = params.get('comm_head_repository')
Packit f0b94e
        revision = params.get('comm_head_rev')
Packit f0b94e
        if not revision:
Packit f0b94e
            logger.warning("Missing `comm_head_rev` parameters; "
Packit f0b94e
                           "assuming all files have changed")
Packit f0b94e
            return True
Packit f0b94e
Packit f0b94e
        changed_files |= {
Packit f0b94e
            join_path("comm", file) for file in
Packit f0b94e
            get_changed_files(repository, revision)
Packit f0b94e
        }
Packit f0b94e
Packit f0b94e
    for pattern in file_patterns:
Packit f0b94e
        for path in changed_files:
Packit f0b94e
            if mozpackmatch(path, pattern):
Packit f0b94e
                return True
Packit f0b94e
Packit f0b94e
    return False
Packit f0b94e
Packit f0b94e
Packit f0b94e
@memoize
Packit f0b94e
def get_locally_changed_files(repo):
Packit f0b94e
    try:
Packit f0b94e
        vcs = get_repository_object(repo)
Packit f0b94e
        return set(vcs.get_outgoing_files('AM'))
Packit f0b94e
    except (InvalidRepoPath, CalledProcessError):
Packit f0b94e
        return set()