"""
Highly experimental script that compiles the CPython standard library using Cython.
Execute the script either in the CPython 'Lib' directory or pass the
option '--current-python' to compile the standard library of the running
Python interpreter.
Pass '-j N' to get a parallel build with N processes.
Usage example::
$ python cystdlib.py --current-python build_ext -i
"""
import os
import sys
from distutils.core import setup
from Cython.Build import cythonize
from Cython.Compiler import Options
# improve Python compatibility by allowing some broken code
Options.error_on_unknown_names = False
Options.error_on_uninitialized = False
exclude_patterns = ['**/test/**/*.py', '**/tests/**/*.py', '**/__init__.py']
broken = [
'idlelib/MultiCall.py',
'email/utils.py',
'multiprocessing/reduction.py',
'multiprocessing/util.py',
'threading.py', # interrupt handling
'lib2to3/fixes/fix_sys_exc.py',
'traceback.py',
'types.py',
'enum.py',
'keyword.py',
'_collections_abc.py',
'importlib/_bootstrap',
]
default_directives = dict(
auto_cpdef=False, # enable when it's safe, see long list of failures below
binding=True,
set_initial_path='SOURCEFILE')
default_directives['optimize.inline_defnode_calls'] = True
special_directives = [
(['pkgutil.py',
'decimal.py',
'datetime.py',
'optparse.py',
'sndhdr.py',
'opcode.py',
'ntpath.py',
'urllib/request.py',
'plat-*/TYPES.py',
'plat-*/IN.py',
'tkinter/_fix.py',
'lib2to3/refactor.py',
'webbrowser.py',
'shutil.py',
'multiprocessing/forking.py',
'xml/sax/expatreader.py',
'xmlrpc/client.py',
'pydoc.py',
'xml/etree/ElementTree.py',
'posixpath.py',
'inspect.py',
'ctypes/util.py',
'urllib/parse.py',
'warnings.py',
'tempfile.py',
'trace.py',
'heapq.py',
'pickletools.py',
'multiprocessing/connection.py',
'hashlib.py',
'getopt.py',
'os.py',
'types.py',
], dict(auto_cpdef=False)),
]
del special_directives[:] # currently unused
def build_extensions(includes='**/*.py',
excludes=None,
special_directives=special_directives,
language_level=sys.version_info[0],
parallel=None):
if isinstance(includes, str):
includes = [includes]
excludes = list(excludes or exclude_patterns) + broken
all_groups = (special_directives or []) + [(includes, {})]
extensions = []
for modules, directives in all_groups:
exclude_now = excludes[:]
for other_modules, _ in special_directives:
if other_modules != modules:
exclude_now.extend(other_modules)
d = dict(default_directives)
d.update(directives)
extensions.extend(
cythonize(
modules,
exclude=exclude_now,
exclude_failures=True,
language_level=language_level,
compiler_directives=d,
nthreads=parallel,
))
return extensions
def build(extensions):
try:
setup(ext_modules=extensions)
result = True
except:
import traceback
print('error building extensions %s' % (
[ext.name for ext in extensions],))
traceback.print_exc()
result = False
return extensions, result
def _build(args):
sys_args, ext = args
sys.argv[1:] = sys_args
return build([ext])
def parse_args():
from optparse import OptionParser
parser = OptionParser('%prog [options] [LIB_DIR (default: ./Lib)]')
parser.add_option(
'--current-python', dest='current_python', action='store_true',
help='compile the stdlib of the running Python')
parser.add_option(
'-j', '--jobs', dest='parallel_jobs', metavar='N',
type=int, default=1,
help='run builds in N parallel jobs (default: 1)')
parser.add_option(
'-x', '--exclude', dest='excludes', metavar='PATTERN',
action="append", help='exclude modules/packages matching PATTERN')
options, args = parser.parse_args()
if not args:
args = ['./Lib']
elif len(args) > 1:
parser.error('only one argument expected, got %d' % len(args))
return options, args
if __name__ == '__main__':
options, args = parse_args()
if options.current_python:
# assume that the stdlib is where the "os" module lives
os.chdir(os.path.dirname(os.__file__))
else:
os.chdir(args[0])
pool = None
parallel_jobs = options.parallel_jobs
if options.parallel_jobs:
try:
import multiprocessing
pool = multiprocessing.Pool(parallel_jobs)
print("Building in %d parallel processes" % parallel_jobs)
except (ImportError, OSError):
print("Not building in parallel")
parallel_jobs = 0
extensions = build_extensions(
parallel=parallel_jobs,
excludes=options.excludes)
sys_args = ['build_ext', '-i']
if pool is not None:
results = pool.map(_build, [(sys_args, ext) for ext in extensions])
pool.close()
pool.join()
for ext, result in results:
if not result:
print("building extension %s failed" % (ext[0].name,))
else:
sys.argv[1:] = sys_args
build(extensions)