#!/usr/bin/python3

# Build many configurations of glibc.

# Copyright (C) 2016-2018 Free Software Foundation, Inc.

# This file is part of the GNU C Library.

#

# The GNU C Library is free software; you can redistribute it and/or

# modify it under the terms of the GNU Lesser General Public

# License as published by the Free Software Foundation; either

# version 2.1 of the License, or (at your option) any later version.

#

# The GNU C Library is distributed in the hope that it will be useful,

# but WITHOUT ANY WARRANTY; without even the implied warranty of

# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU

# Lesser General Public License for more details.

#

# You should have received a copy of the GNU Lesser General Public

# License along with the GNU C Library; if not, see

# <http://www.gnu.org/licenses/>.

"""Build many configurations of glibc.

This script takes as arguments a directory name (containing a src

subdirectory with sources of the relevant toolchain components) and a

description of what to do: 'checkout', to check out sources into that

directory, 'bot-cycle', to run a series of checkout and build steps,

'bot', to run 'bot-cycle' repeatedly, 'host-libraries', to build

libraries required by the toolchain, 'compilers', to build

cross-compilers for various configurations, or 'glibcs', to build

glibc for various configurations and run the compilation parts of the

testsuite.  Subsequent arguments name the versions of components to

check out (<component>-

other than 'checkout' and 'bot-cycle', name configurations for which

compilers or glibc are to be built.

"""

import argparse

import datetime

import email.mime.text

import email.utils

import json

import os

import re

import shutil

import smtplib

import stat

import subprocess

import sys

import time

import urllib.request

try:

    os.cpu_count

except:

    import multiprocessing

    os.cpu_count = lambda: multiprocessing.cpu_count()

try:

    re.fullmatch

except:

    re.fullmatch = lambda p,s,f=0: re.match(p+"\\Z",s,f)

try:

    subprocess.run

except:

    class _CompletedProcess:

        def __init__(self, args, returncode, stdout=None, stderr=None):

            self.args = args

            self.returncode = returncode

            self.stdout = stdout

            self.stderr = stderr

    def _run(*popenargs, input=None, timeout=None, check=False, **kwargs):

        assert(timeout is None)

        with subprocess.Popen(*popenargs, **kwargs) as process:

            try:

                stdout, stderr = process.communicate(input)

            except:

                process.kill()

                process.wait()

                raise

            returncode = process.poll()

            if check and returncode:

                raise subprocess.CalledProcessError(returncode, popenargs)

        return _CompletedProcess(popenargs, returncode, stdout, stderr)

    subprocess.run = _run

class Context(object):

    """The global state associated with builds in a given directory."""

    def __init__(self, topdir, parallelism, keep, replace_sources, strip,

                 action):

        """Initialize the context."""

        self.topdir = topdir

        self.parallelism = parallelism

        self.keep = keep

        self.replace_sources = replace_sources

        self.strip = strip

        self.srcdir = os.path.join(topdir, 'src')

        self.versions_json = os.path.join(self.srcdir, 'versions.json')

        self.build_state_json = os.path.join(topdir, 'build-state.json')

        self.bot_config_json = os.path.join(topdir, 'bot-config.json')

        self.installdir = os.path.join(topdir, 'install')

        self.host_libraries_installdir = os.path.join(self.installdir,

                                                      'host-libraries')

        self.builddir = os.path.join(topdir, 'build')

        self.logsdir = os.path.join(topdir, 'logs')

        self.logsdir_old = os.path.join(topdir, 'logs-old')

        self.makefile = os.path.join(self.builddir, 'Makefile')

        self.wrapper = os.path.join(self.builddir, 'wrapper')

        self.save_logs = os.path.join(self.builddir, 'save-logs')

        self.script_text = self.get_script_text()

        if action != 'checkout':

            self.build_triplet = self.get_build_triplet()

            self.glibc_version = self.get_glibc_version()

        self.configs = {}

        self.glibc_configs = {}

        self.makefile_pieces = ['.PHONY: all\n']

        self.add_all_configs()

        self.load_versions_json()

        self.load_build_state_json()

        self.status_log_list = []

        self.email_warning = False

    def get_script_text(self):

        """Return the text of this script."""

        with open(sys.argv[0], 'r') as f:

            return f.read()

    def exec_self(self):

        """Re-execute this script with the same arguments."""

        sys.stdout.flush()

        os.execv(sys.executable, [sys.executable] + sys.argv)

    def get_build_triplet(self):

        """Determine the build triplet with config.guess."""

        config_guess = os.path.join(self.component_srcdir('gcc'),

                                    'config.guess')

        cg_out = subprocess.run([config_guess], stdout=subprocess.PIPE,

                                check=True, universal_newlines=True).stdout

        return cg_out.rstrip()

    def get_glibc_version(self):

        """Determine the glibc version number (major.minor)."""

        version_h = os.path.join(self.component_srcdir('glibc'), 'version.h')

        with open(version_h, 'r') as f:

            lines = f.readlines()

        starttext = '#define VERSION "'

        for l in lines:

            if l.startswith(starttext):

                l = l[len(starttext):]

                l = l.rstrip('"\n')

                m = re.fullmatch('([0-9]+)\.([0-9]+)[.0-9]*', l)

                return '%s.%s' % m.group(1, 2)

        print('error: could not determine glibc version')

        exit(1)

    def add_all_configs(self):

        """Add all known glibc build configurations."""

        self.add_config(arch='aarch64',

                        os_name='linux-gnu',

                        extra_glibcs=[{'variant': 'disable-multi-arch',

                                       'cfg': ['--disable-multi-arch']}])

        self.add_config(arch='aarch64_be',

                        os_name='linux-gnu')

        self.add_config(arch='alpha',

                        os_name='linux-gnu')

        self.add_config(arch='arm',

                        os_name='linux-gnueabi')

        self.add_config(arch='armeb',

                        os_name='linux-gnueabi')

        self.add_config(arch='armeb',

                        os_name='linux-gnueabi',

                        variant='be8',

                        gcc_cfg=['--with-arch=armv7-a'])

        self.add_config(arch='arm',

                        os_name='linux-gnueabihf',

                        gcc_cfg=['--with-float=hard', '--with-cpu=arm926ej-s'],

                        extra_glibcs=[{'variant': 'v7a',

                                       'ccopts': '-march=armv7-a -mfpu=vfpv3'},

                                      {'variant': 'v7a-disable-multi-arch',

                                       'ccopts': '-march=armv7-a -mfpu=vfpv3',

                                       'cfg': ['--disable-multi-arch']}])

        self.add_config(arch='armeb',

                        os_name='linux-gnueabihf',

                        gcc_cfg=['--with-float=hard', '--with-cpu=arm926ej-s'])

        self.add_config(arch='armeb',

                        os_name='linux-gnueabihf',

                        variant='be8',

                        gcc_cfg=['--with-float=hard', '--with-arch=armv7-a',

                                 '--with-fpu=vfpv3'])

        self.add_config(arch='hppa',

                        os_name='linux-gnu')

        self.add_config(arch='i686',

                        os_name='gnu')

        self.add_config(arch='ia64',

                        os_name='linux-gnu',

                        first_gcc_cfg=['--with-system-libunwind'])

        self.add_config(arch='m68k',

                        os_name='linux-gnu',

                        gcc_cfg=['--disable-multilib'])

        self.add_config(arch='m68k',

                        os_name='linux-gnu',

                        variant='coldfire',

                        gcc_cfg=['--with-arch=cf', '--disable-multilib'])

        self.add_config(arch='m68k',

                        os_name='linux-gnu',

                        variant='coldfire-soft',

                        gcc_cfg=['--with-arch=cf', '--with-cpu=54455',

                                 '--disable-multilib'])

        self.add_config(arch='microblaze',

                        os_name='linux-gnu',

                        gcc_cfg=['--disable-multilib'])

        self.add_config(arch='microblazeel',

                        os_name='linux-gnu',

                        gcc_cfg=['--disable-multilib'])

        self.add_config(arch='mips64',

                        os_name='linux-gnu',

                        gcc_cfg=['--with-mips-plt'],

                        glibcs=[{'variant': 'n32'},

                                {'arch': 'mips',

                                 'ccopts': '-mabi=32'},

                                {'variant': 'n64',

                                 'ccopts': '-mabi=64'}])

        self.add_config(arch='mips64',

                        os_name='linux-gnu',

                        variant='soft',

                        gcc_cfg=['--with-mips-plt', '--with-float=soft'],

                        glibcs=[{'variant': 'n32-soft'},

                                {'variant': 'soft',

                                 'arch': 'mips',

                                 'ccopts': '-mabi=32'},

                                {'variant': 'n64-soft',

                                 'ccopts': '-mabi=64'}])

        self.add_config(arch='mips64',

                        os_name='linux-gnu',

                        variant='nan2008',

                        gcc_cfg=['--with-mips-plt', '--with-nan=2008',

                                 '--with-arch-64=mips64r2',

                                 '--with-arch-32=mips32r2'],

                        glibcs=[{'variant': 'n32-nan2008'},

                                {'variant': 'nan2008',

                                 'arch': 'mips',

                                 'ccopts': '-mabi=32'},

                                {'variant': 'n64-nan2008',

                                 'ccopts': '-mabi=64'}])

        self.add_config(arch='mips64',

                        os_name='linux-gnu',

                        variant='nan2008-soft',

                        gcc_cfg=['--with-mips-plt', '--with-nan=2008',

                                 '--with-arch-64=mips64r2',

                                 '--with-arch-32=mips32r2',

                                 '--with-float=soft'],

                        glibcs=[{'variant': 'n32-nan2008-soft'},

                                {'variant': 'nan2008-soft',

                                 'arch': 'mips',

                                 'ccopts': '-mabi=32'},

                                {'variant': 'n64-nan2008-soft',

                                 'ccopts': '-mabi=64'}])

        self.add_config(arch='mips64el',

                        os_name='linux-gnu',

                        gcc_cfg=['--with-mips-plt'],

                        glibcs=[{'variant': 'n32'},

                                {'arch': 'mipsel',

                                 'ccopts': '-mabi=32'},

                                {'variant': 'n64',

                                 'ccopts': '-mabi=64'}])

        self.add_config(arch='mips64el',

                        os_name='linux-gnu',

                        variant='soft',

                        gcc_cfg=['--with-mips-plt', '--with-float=soft'],

                        glibcs=[{'variant': 'n32-soft'},

                                {'variant': 'soft',

                                 'arch': 'mipsel',

                                 'ccopts': '-mabi=32'},

                                {'variant': 'n64-soft',

                                 'ccopts': '-mabi=64'}])

        self.add_config(arch='mips64el',

                        os_name='linux-gnu',

                        variant='nan2008',

                        gcc_cfg=['--with-mips-plt', '--with-nan=2008',

                                 '--with-arch-64=mips64r2',

                                 '--with-arch-32=mips32r2'],

                        glibcs=[{'variant': 'n32-nan2008'},

                                {'variant': 'nan2008',

                                 'arch': 'mipsel',

                                 'ccopts': '-mabi=32'},

                                {'variant': 'n64-nan2008',

                                 'ccopts': '-mabi=64'}])

        self.add_config(arch='mips64el',

                        os_name='linux-gnu',

                        variant='nan2008-soft',

                        gcc_cfg=['--with-mips-plt', '--with-nan=2008',

                                 '--with-arch-64=mips64r2',

                                 '--with-arch-32=mips32r2',

                                 '--with-float=soft'],

                        glibcs=[{'variant': 'n32-nan2008-soft'},

                                {'variant': 'nan2008-soft',

                                 'arch': 'mipsel',

                                 'ccopts': '-mabi=32'},

                                {'variant': 'n64-nan2008-soft',

                                 'ccopts': '-mabi=64'}])

        self.add_config(arch='nios2',

                        os_name='linux-gnu')

        self.add_config(arch='powerpc',

                        os_name='linux-gnu',

                        gcc_cfg=['--disable-multilib', '--enable-secureplt'],

                        extra_glibcs=[{'variant': 'power4',

                                       'ccopts': '-mcpu=power4',

                                       'cfg': ['--with-cpu=power4']}])

        self.add_config(arch='powerpc',

                        os_name='linux-gnu',

                        variant='soft',

                        gcc_cfg=['--disable-multilib', '--with-float=soft',

                                 '--enable-secureplt'])

        self.add_config(arch='powerpc64',

                        os_name='linux-gnu',

                        gcc_cfg=['--disable-multilib', '--enable-secureplt'])

        self.add_config(arch='powerpc64le',

                        os_name='linux-gnu',

                        gcc_cfg=['--disable-multilib', '--enable-secureplt'])

        self.add_config(arch='powerpc',

                        os_name='linux-gnuspe',

                        gcc_cfg=['--disable-multilib', '--enable-secureplt',

                                 '--enable-e500-double', '--enable-obsolete'])

        self.add_config(arch='powerpc',

                        os_name='linux-gnuspe',

                        variant='e500v1',

                        gcc_cfg=['--disable-multilib', '--enable-secureplt',

                                 '--enable-obsolete'])

        self.add_config(arch='riscv64',

                        os_name='linux-gnu',

                        variant='rv64imac-lp64',

                        gcc_cfg=['--with-arch=rv64imac', '--with-abi=lp64',

                                 '--disable-multilib'])

        self.add_config(arch='riscv64',

                        os_name='linux-gnu',

                        variant='rv64imafdc-lp64',

                        gcc_cfg=['--with-arch=rv64imafdc', '--with-abi=lp64',

                                 '--disable-multilib'])

        self.add_config(arch='riscv64',

                        os_name='linux-gnu',

                        variant='rv64imafdc-lp64d',

                        gcc_cfg=['--with-arch=rv64imafdc', '--with-abi=lp64d',

                                 '--disable-multilib'])

        self.add_config(arch='s390x',

                        os_name='linux-gnu',

                        glibcs=[{},

                                {'arch': 's390', 'ccopts': '-m31'}])

        self.add_config(arch='sh3',

                        os_name='linux-gnu')

        self.add_config(arch='sh3eb',

                        os_name='linux-gnu')

        self.add_config(arch='sh4',

                        os_name='linux-gnu')

        self.add_config(arch='sh4eb',

                        os_name='linux-gnu')

        self.add_config(arch='sh4',

                        os_name='linux-gnu',

                        variant='soft',

                        gcc_cfg=['--without-fp'])

        self.add_config(arch='sh4eb',

                        os_name='linux-gnu',

                        variant='soft',

                        gcc_cfg=['--without-fp'])

        self.add_config(arch='sparc64',

                        os_name='linux-gnu',

                        glibcs=[{},

                                {'arch': 'sparcv9',

                                 'ccopts': '-m32 -mlong-double-128'}],

                        extra_glibcs=[{'variant': 'disable-multi-arch',

                                       'cfg': ['--disable-multi-arch']},

                                      {'variant': 'disable-multi-arch',

                                       'arch': 'sparcv9',

                                       'ccopts': '-m32 -mlong-double-128',

                                       'cfg': ['--disable-multi-arch']}])

        self.add_config(arch='x86_64',

                        os_name='linux-gnu',

                        gcc_cfg=['--with-multilib-list=m64,m32,mx32'],

                        glibcs=[{},

                                {'variant': 'x32', 'ccopts': '-mx32'},

                                {'arch': 'i686', 'ccopts': '-m32 -march=i686'}],

                        extra_glibcs=[{'variant': 'disable-multi-arch',

                                       'cfg': ['--disable-multi-arch']},

                                      {'variant': 'static-pie',

                                       'cfg': ['--enable-static-pie']},

                                      {'variant': 'x32-static-pie',

                                       'ccopts': '-mx32',

                                       'cfg': ['--enable-static-pie']},

                                      {'variant': 'static-pie',

                                       'arch': 'i686',

                                       'ccopts': '-m32 -march=i686',

                                       'cfg': ['--enable-static-pie']},

                                      {'variant': 'disable-multi-arch',

                                       'arch': 'i686',

                                       'ccopts': '-m32 -march=i686',

                                       'cfg': ['--disable-multi-arch']},

                                      {'arch': 'i486',

                                       'ccopts': '-m32 -march=i486'},

                                      {'arch': 'i586',

                                       'ccopts': '-m32 -march=i586'}])

    def add_config(self, **args):

        """Add an individual build configuration."""

        cfg = Config(self, **args)

        if cfg.name in self.configs:

            print('error: duplicate config %s' % cfg.name)

            exit(1)

        self.configs[cfg.name] = cfg

        for c in cfg.all_glibcs:

            if c.name in self.glibc_configs:

                print('error: duplicate glibc config %s' % c.name)

                exit(1)

            self.glibc_configs[c.name] = c

    def component_srcdir(self, component):

        """Return the source directory for a given component, e.g. gcc."""

        return os.path.join(self.srcdir, component)

    def component_builddir(self, action, config, component, subconfig=None):

        """Return the directory to use for a build."""

        if config is None:

            # Host libraries.

            assert subconfig is None

            return os.path.join(self.builddir, action, component)

        if subconfig is None:

            return os.path.join(self.builddir, action, config, component)

        else:

            # glibc build as part of compiler build.

            return os.path.join(self.builddir, action, config, component,

                                subconfig)

    def compiler_installdir(self, config):

        """Return the directory in which to install a compiler."""

        return os.path.join(self.installdir, 'compilers', config)

    def compiler_bindir(self, config):

        """Return the directory in which to find compiler binaries."""

        return os.path.join(self.compiler_installdir(config), 'bin')

    def compiler_sysroot(self, config):

        """Return the sysroot directory for a compiler."""

        return os.path.join(self.compiler_installdir(config), 'sysroot')

    def glibc_installdir(self, config):

        """Return the directory in which to install glibc."""

        return os.path.join(self.installdir, 'glibcs', config)

    def run_builds(self, action, configs):

        """Run the requested builds."""

        if action == 'checkout':

            self.checkout(configs)

            return

        if action == 'bot-cycle':

            if configs:

                print('error: configurations specified for bot-cycle')

                exit(1)

            self.bot_cycle()

            return

        if action == 'bot':

            if configs:

                print('error: configurations specified for bot')

                exit(1)

            self.bot()

            return

        if action == 'host-libraries' and configs:

            print('error: configurations specified for host-libraries')

            exit(1)

        self.clear_last_build_state(action)

        build_time = datetime.datetime.utcnow()

        if action == 'host-libraries':

            build_components = ('gmp', 'mpfr', 'mpc')

            old_components = ()

            old_versions = {}

            self.build_host_libraries()

        elif action == 'compilers':

            build_components = ('binutils', 'gcc', 'glibc', 'linux', 'mig',

                                'gnumach', 'hurd')

            old_components = ('gmp', 'mpfr', 'mpc')

            old_versions = self.build_state['host-libraries']['build-versions']

            self.build_compilers(configs)

        else:

            build_components = ('glibc',)

            old_components = ('gmp', 'mpfr', 'mpc', 'binutils', 'gcc', 'linux',

                              'mig', 'gnumach', 'hurd')

            old_versions = self.build_state['compilers']['build-versions']

            self.build_glibcs(configs)

        self.write_files()

        self.do_build()

        if configs:

            # Partial build, do not update stored state.

            return

        build_versions = {}

        for k in build_components:

            if k in self.versions:

                build_versions[k] = {'version': self.versions[k]['version'],

                                     'revision': self.versions[k]['revision']}

        for k in old_components:

            if k in old_versions:

                build_versions[k] = {'version': old_versions[k]['version'],

                                     'revision': old_versions[k]['revision']}

        self.update_build_state(action, build_time, build_versions)

    @staticmethod

    def remove_dirs(*args):

        """Remove directories and their contents if they exist."""

        for dir in args:

            shutil.rmtree(dir, ignore_errors=True)

    @staticmethod

    def remove_recreate_dirs(*args):

        """Remove directories if they exist, and create them as empty."""

        Context.remove_dirs(*args)

        for dir in args:

            os.makedirs(dir, exist_ok=True)

    def add_makefile_cmdlist(self, target, cmdlist, logsdir):

        """Add makefile text for a list of commands."""

        commands = cmdlist.makefile_commands(self.wrapper, logsdir)

        self.makefile_pieces.append('all: %s\n.PHONY: %s\n%s:\n%s\n' %

                                    (target, target, target, commands))

        self.status_log_list.extend(cmdlist.status_logs(logsdir))

    def write_files(self):

        """Write out the Makefile and wrapper script."""

        mftext = ''.join(self.makefile_pieces)

        with open(self.makefile, 'w') as f:

            f.write(mftext)

        wrapper_text = (

            '#!/bin/sh\n'

            'prev_base=$1\n'

            'this_base=$2\n'

            'desc=$3\n'

            'dir=$4\n'

            'path=$5\n'

            'shift 5\n'

            'prev_status=$prev_base-status.txt\n'

            'this_status=$this_base-status.txt\n'

            'this_log=$this_base-log.txt\n'

            'date > "$this_log"\n'

            'echo >> "$this_log"\n'

            'echo "Description: $desc" >> "$this_log"\n'

            'printf "%s" "Command:" >> "$this_log"\n'

            'for word in "$@"; do\n'

            '  if expr "$word" : "[]+,./0-9@A-Z_a-z-]\\\\{1,\\\\}\\$" > /dev/null; then\n'

            '    printf " %s" "$word"\n'

            '  else\n'

            '    printf " \'"\n'

            '    printf "%s" "$word" | sed -e "s/\'/\'\\\\\\\\\'\'/"\n'

            '    printf "\'"\n'

            '  fi\n'

            'done >> "$this_log"\n'

            'echo >> "$this_log"\n'

            'echo "Directory: $dir" >> "$this_log"\n'

            'echo "Path addition: $path" >> "$this_log"\n'

            'echo >> "$this_log"\n'

            'record_status ()\n'

            '{\n'

            '  echo >> "$this_log"\n'

            '  echo "$1: $desc" > "$this_status"\n'

            '  echo "$1: $desc" >> "$this_log"\n'

            '  echo >> "$this_log"\n'

            '  date >> "$this_log"\n'

            '  echo "$1: $desc"\n'

            '  exit 0\n'

            '}\n'

            'check_error ()\n'

            '{\n'

            '  if [ "$1" != "0" ]; then\n'

            '    record_status FAIL\n'

            '  fi\n'

            '}\n'

            'if [ "$prev_base" ] && ! grep -q "^PASS" "$prev_status"; then\n'

            '    record_status UNRESOLVED\n'

            'fi\n'

            'if [ "$dir" ]; then\n'

            '  cd "$dir"\n'

            '  check_error "$?"\n'

            'fi\n'

            'if [ "$path" ]; then\n'

            '  PATH=$path:$PATH\n'

            'fi\n'

            '"$@" < /dev/null >> "$this_log" 2>&1\n'

            'check_error "$?"\n'

            'record_status PASS\n')

        with open(self.wrapper, 'w') as f:

            f.write(wrapper_text)

        # Mode 0o755.

        mode_exec = (stat.S_IRWXU|stat.S_IRGRP|stat.S_IXGRP|

                     stat.S_IROTH|stat.S_IXOTH)

        os.chmod(self.wrapper, mode_exec)

        save_logs_text = (

            '#!/bin/sh\n'

            'if ! [ -f tests.sum ]; then\n'

            '  echo "No test summary available."\n'

            '  exit 0\n'

            'fi\n'

            'save_file ()\n'

            '{\n'

            '  echo "Contents of $1:"\n'

            '  echo\n'

            '  cat "$1"\n'

            '  echo\n'

            '  echo "End of contents of $1."\n'

            '  echo\n'

            '}\n'

            'save_file tests.sum\n'

            'non_pass_tests=$(grep -v "^PASS: " tests.sum | sed -e "s/^PASS: //")\n'

            'for t in $non_pass_tests; do\n'

            '  if [ -f "$t.out" ]; then\n'

            '    save_file "$t.out"\n'

            '  fi\n'

            'done\n')

        with open(self.save_logs, 'w') as f:

            f.write(save_logs_text)

        os.chmod(self.save_logs, mode_exec)

    def do_build(self):

        """Do the actual build."""

        cmd = ['make', '-j%d' % self.parallelism]

        subprocess.run(cmd, cwd=self.builddir, check=True)

    def build_host_libraries(self):

        """Build the host libraries."""

        installdir = self.host_libraries_installdir

        builddir = os.path.join(self.builddir, 'host-libraries')

        logsdir = os.path.join(self.logsdir, 'host-libraries')

        self.remove_recreate_dirs(installdir, builddir, logsdir)

        cmdlist = CommandList('host-libraries', self.keep)

        self.build_host_library(cmdlist, 'gmp')

        self.build_host_library(cmdlist, 'mpfr',

                                ['--with-gmp=%s' % installdir])

        self.build_host_library(cmdlist, 'mpc',

                                ['--with-gmp=%s' % installdir,

                                '--with-mpfr=%s' % installdir])

        cmdlist.add_command('done', ['touch', os.path.join(installdir, 'ok')])

        self.add_makefile_cmdlist('host-libraries', cmdlist, logsdir)

    def build_host_library(self, cmdlist, lib, extra_opts=None):

        """Build one host library."""

        srcdir = self.component_srcdir(lib)

        builddir = self.component_builddir('host-libraries', None, lib)

        installdir = self.host_libraries_installdir

        cmdlist.push_subdesc(lib)

        cmdlist.create_use_dir(builddir)

        cfg_cmd = [os.path.join(srcdir, 'configure'),

                   '--prefix=%s' % installdir,

                   '--disable-shared']

        if extra_opts:

            cfg_cmd.extend (extra_opts)

        cmdlist.add_command('configure', cfg_cmd)

        cmdlist.add_command('build', ['make'])

        cmdlist.add_command('check', ['make', 'check'])

        cmdlist.add_command('install', ['make', 'install'])

        cmdlist.cleanup_dir()

        cmdlist.pop_subdesc()

    def build_compilers(self, configs):

        """Build the compilers."""

        if not configs:

            self.remove_dirs(os.path.join(self.builddir, 'compilers'))

            self.remove_dirs(os.path.join(self.installdir, 'compilers'))

            self.remove_dirs(os.path.join(self.logsdir, 'compilers'))

            configs = sorted(self.configs.keys())

        for c in configs:

            self.configs[c].build()

    def build_glibcs(self, configs):

        """Build the glibcs."""

        if not configs:

            self.remove_dirs(os.path.join(self.builddir, 'glibcs'))

            self.remove_dirs(os.path.join(self.installdir, 'glibcs'))

            self.remove_dirs(os.path.join(self.logsdir, 'glibcs'))

            configs = sorted(self.glibc_configs.keys())

        for c in configs:

            self.glibc_configs[c].build()

    def load_versions_json(self):

        """Load information about source directory versions."""

        if not os.access(self.versions_json, os.F_OK):

            self.versions = {}

            return

        with open(self.versions_json, 'r') as f:

            self.versions = json.load(f)

    def store_json(self, data, filename):

        """Store information in a JSON file."""

        filename_tmp = filename + '.tmp'

        with open(filename_tmp, 'w') as f:

            json.dump(data, f, indent=2, sort_keys=True)

        os.rename(filename_tmp, filename)

    def store_versions_json(self):

        """Store information about source directory versions."""

        self.store_json(self.versions, self.versions_json)

    def set_component_version(self, component, version, explicit, revision):

        """Set the version information for a component."""

        self.versions[component] = {'version': version,

                                    'explicit': explicit,

                                    'revision': revision}

        self.store_versions_json()

    def checkout(self, versions):

        """Check out the desired component versions."""

        default_versions = {'binutils': 'vcs-2.31',

                            'gcc': 'vcs-8',

                            'glibc': 'vcs-mainline',

                            'gmp': '6.1.2',

                            'linux': '4.17',

                            'mpc': '1.1.0',

                            'mpfr': '4.0.1',

                            'mig': 'vcs-mainline',

                            'gnumach': 'vcs-mainline',

                            'hurd': 'vcs-mainline'}

        use_versions = {}

        explicit_versions = {}

        for v in versions:

            found_v = False

            for k in default_versions.keys():

                kx = k + '-'

                if v.startswith(kx):

                    vx = v[len(kx):]

                    if k in use_versions:

                        print('error: multiple versions for %s' % k)

                        exit(1)

                    use_versions[k] = vx

                    explicit_versions[k] = True

                    found_v = True

                    break

            if not found_v:

                print('error: unknown component in %s' % v)

                exit(1)

        for k in default_versions.keys():

            if k not in use_versions:

                if k in self.versions and self.versions[k]['explicit']:

                    use_versions[k] = self.versions[k]['version']

                    explicit_versions[k] = True

                else:

                    use_versions[k] = default_versions[k]

                    explicit_versions[k] = False

        os.makedirs(self.srcdir, exist_ok=True)

        for k in sorted(default_versions.keys()):

            update = os.access(self.component_srcdir(k), os.F_OK)

            v = use_versions[k]

            if (update and

                k in self.versions and

                v != self.versions[k]['version']):

                if not self.replace_sources:

                    print('error: version of %s has changed from %s to %s, '

                          'use --replace-sources to check out again' %

                          (k, self.versions[k]['version'], v))

                    exit(1)

                shutil.rmtree(self.component_srcdir(k))

                update = False

            if v.startswith('vcs-'):

                revision = self.checkout_vcs(k, v[4:], update)

            else:

                self.checkout_tar(k, v, update)

                revision = v

            self.set_component_version(k, v, explicit_versions[k], revision)

        if self.get_script_text() != self.script_text:

            # Rerun the checkout process in case the updated script

            # uses different default versions or new components.

            self.exec_self()

    def checkout_vcs(self, component, version, update):

        """Check out the given version of the given component from version

        control.  Return a revision identifier."""

        if component == 'binutils':

            git_url = 'git://sourceware.org/git/binutils-gdb.git'

            if version == 'mainline':

                git_branch = 'master'

            else:

                trans = str.maketrans({'.': '_'})

                git_branch = 'binutils-%s-branch' % version.translate(trans)

            return self.git_checkout(component, git_url, git_branch, update)

        elif component == 'gcc':

            if version == 'mainline':

                branch = 'trunk'

            else:

                trans = str.maketrans({'.': '_'})

                branch = 'branches/gcc-%s-branch' % version.translate(trans)

            svn_url = 'svn://gcc.gnu.org/svn/gcc/%s' % branch

            return self.gcc_checkout(svn_url, update)

        elif component == 'glibc':

            git_url = 'git://sourceware.org/git/glibc.git'

            if version == 'mainline':

                git_branch = 'master'

            else:

                git_branch = 'release/%s/master' % version

            r = self.git_checkout(component, git_url, git_branch, update)

            self.fix_glibc_timestamps()

            return r

        elif component == 'gnumach':

            git_url = 'git://git.savannah.gnu.org/hurd/gnumach.git'

            git_branch = 'master'

            r = self.git_checkout(component, git_url, git_branch, update)

            subprocess.run(['autoreconf', '-i'],

                           cwd=self.component_srcdir(component), check=True)

            return r

        elif component == 'mig':

            git_url = 'git://git.savannah.gnu.org/hurd/mig.git'

            git_branch = 'master'

            r = self.git_checkout(component, git_url, git_branch, update)

            subprocess.run(['autoreconf', '-i'],

                           cwd=self.component_srcdir(component), check=True)

            return r

        elif component == 'hurd':

            git_url = 'git://git.savannah.gnu.org/hurd/hurd.git'

            git_branch = 'master'

            r = self.git_checkout(component, git_url, git_branch, update)

            subprocess.run(['autoconf'],

                           cwd=self.component_srcdir(component), check=True)

            return r

        else:

            print('error: component %s coming from VCS' % component)

            exit(1)

    def git_checkout(self, component, git_url, git_branch, update):

        """Check out a component from git.  Return a commit identifier."""

        if update:

            subprocess.run(['git', 'remote', 'prune', 'origin'],

                           cwd=self.component_srcdir(component), check=True)

            if self.replace_sources:

                subprocess.run(['git', 'clean', '-dxfq'],

                               cwd=self.component_srcdir(component), check=True)

            subprocess.run(['git', 'pull', '-q'],

                           cwd=self.component_srcdir(component), check=True)

        else:

            subprocess.run(['git', 'clone', '-q', '-b', git_branch, git_url,

                            self.component_srcdir(component)], check=True)

        r = subprocess.run(['git', 'rev-parse', 'HEAD'],

                           cwd=self.component_srcdir(component),

                           stdout=subprocess.PIPE,

                           check=True, universal_newlines=True).stdout

        return r.rstrip()

    def fix_glibc_timestamps(self):

        """Fix timestamps in a glibc checkout."""

        # Ensure that builds do not try to regenerate generated files

        # in the source tree.

        srcdir = self.component_srcdir('glibc')

        for dirpath, dirnames, filenames in os.walk(srcdir):

            for f in filenames:

                if (f == 'configure' or

                    f == 'preconfigure' or

                    f.endswith('-kw.h')):

                    to_touch = os.path.join(dirpath, f)

                    subprocess.run(['touch', to_touch], check=True)

    def gcc_checkout(self, svn_url, update):

        """Check out GCC from SVN.  Return the revision number."""

        if not update:

            subprocess.run(['svn', 'co', '-q', svn_url,

                            self.component_srcdir('gcc')], check=True)

        subprocess.run(['contrib/gcc_update', '--silent'],

                       cwd=self.component_srcdir('gcc'), check=True)

        r = subprocess.run(['svnversion', self.component_srcdir('gcc')],

                           stdout=subprocess.PIPE,

                           check=True, universal_newlines=True).stdout

        return r.rstrip()

    def checkout_tar(self, component, version, update):

        """Check out the given version of the given component from a

        tarball."""

        if update:

            return

        url_map = {'binutils': 'https://ftp.gnu.org/gnu/binutils/binutils-%(version)s.tar.bz2',

                   'gcc': 'https://ftp.gnu.org/gnu/gcc/gcc-%(version)s/gcc-%(version)s.tar.gz',

                   'gmp': 'https://ftp.gnu.org/gnu/gmp/gmp-%(version)s.tar.xz',

                   'linux': 'https://www.kernel.org/pub/linux/kernel/v4.x/linux-%(version)s.tar.xz',

                   'mpc': 'https://ftp.gnu.org/gnu/mpc/mpc-%(version)s.tar.gz',

                   'mpfr': 'https://ftp.gnu.org/gnu/mpfr/mpfr-%(version)s.tar.xz',

                   'mig': 'https://ftp.gnu.org/gnu/mig/mig-%(version)s.tar.bz2',

                   'gnumach': 'https://ftp.gnu.org/gnu/gnumach/gnumach-%(version)s.tar.bz2',

                   'hurd': 'https://ftp.gnu.org/gnu/hurd/hurd-%(version)s.tar.bz2'}

        if component not in url_map: