From 6ee5f07061e53e98b6b8a76c0d1555fdc1399397 Mon Sep 17 00:00:00 2001 From: Thomas A Caswell Date: Mon, 24 Jan 2022 18:39:55 -0500 Subject: [PATCH] WIP: Attempt to get pyyaml to build with cython 3.0 This is cribbed from h5py's setup_build.py. While this works and the tests pass, it drops a lot of the configuration that used to be possible / support for other Pythons / setting the include path / ... --- setup.py | 236 ++++++++++++++++++++----------------------------------- 1 file changed, 86 insertions(+), 150 deletions(-) diff --git a/setup.py b/setup.py index 944e7fa2..6be43400 100644 --- a/setup.py +++ b/setup.py @@ -66,23 +66,29 @@ import sys, os, os.path, pathlib, platform, shutil, tempfile, warnings - +import os.path as op # for newer setuptools, enable the embedded distutils before importing setuptools/distutils to avoid warnings -os.environ['SETUPTOOLS_USE_DISTUTILS'] = 'local' +os.environ["SETUPTOOLS_USE_DISTUTILS"] = "local" -from setuptools import setup, Command, Distribution as _Distribution, Extension as _Extension +from setuptools import setup, Command, Distribution, Extension from setuptools.command.build_ext import build_ext as _build_ext + # NB: distutils imports must remain below setuptools to ensure we use the embedded version from distutils import log -from distutils.errors import DistutilsError, CompileError, LinkError, DistutilsPlatformError +from distutils.errors import ( + DistutilsError, + CompileError, + LinkError, + DistutilsPlatformError, +) with_cython = False -if 'sdist' in sys.argv or os.environ.get('PYYAML_FORCE_CYTHON') == '1': +if "sdist" in sys.argv or os.environ.get("PYYAML_FORCE_CYTHON") == "1": # we need cython here with_cython = True try: - from Cython.Distutils.extension import Extension as _Extension - from Cython.Distutils import build_ext as _build_ext + import Cython # noqa + with_cython = True except ImportError: if with_cython: @@ -96,147 +102,82 @@ # on Windows, disable wheel generation warning noise windows_ignore_warnings = [ -"Unknown distribution option: 'python_requires'", -"Config variable 'Py_DEBUG' is unset", -"Config variable 'WITH_PYMALLOC' is unset", -"Config variable 'Py_UNICODE_SIZE' is unset", -"Cython directive 'language_level' not set" + "Unknown distribution option: 'python_requires'", + "Config variable 'Py_DEBUG' is unset", + "Config variable 'WITH_PYMALLOC' is unset", + "Config variable 'Py_UNICODE_SIZE' is unset", + "Cython directive 'language_level' not set", ] -if platform.system() == 'Windows': +if platform.system() == "Windows": for w in windows_ignore_warnings: - warnings.filterwarnings('ignore', w) - - -class Distribution(_Distribution): - def __init__(self, attrs=None): - _Distribution.__init__(self, attrs) - if not self.ext_modules: - return - for idx in range(len(self.ext_modules)-1, -1, -1): - ext = self.ext_modules[idx] - if not isinstance(ext, Extension): - continue - setattr(self, ext.attr_name, None) - self.global_options = [ - (ext.option_name, None, - "include %s (default if %s is available)" - % (ext.feature_description, ext.feature_name)), - (ext.neg_option_name, None, - "exclude %s" % ext.feature_description), - ] + self.global_options - self.negative_opt = self.negative_opt.copy() - self.negative_opt[ext.neg_option_name] = ext.option_name - - def has_ext_modules(self): - if not self.ext_modules: - return False - for ext in self.ext_modules: - with_ext = self.ext_status(ext) - if with_ext is None or with_ext: - return True - return False - - def ext_status(self, ext): - implementation = platform.python_implementation() - if implementation not in ['CPython', 'PyPy']: - return False - if isinstance(ext, Extension): - # the "build by default" behavior is implemented by this returning None - with_ext = getattr(self, ext.attr_name) or os.environ.get('PYYAML_FORCE_{0}'.format(ext.feature_name.upper())) - try: - with_ext = int(with_ext) # attempt coerce envvar to int - except TypeError: - pass - return with_ext - else: - return True - - -class Extension(_Extension): - - def __init__(self, name, sources, feature_name, feature_description, - feature_check, **kwds): - if not with_cython: - for filename in sources[:]: - base, ext = os.path.splitext(filename) - if ext == '.pyx': - sources.remove(filename) - sources.append('%s.c' % base) - _Extension.__init__(self, name, sources, **kwds) - self.feature_name = feature_name - self.feature_description = feature_description - self.feature_check = feature_check - self.attr_name = 'with_' + feature_name.replace('-', '_') - self.option_name = 'with-' + feature_name - self.neg_option_name = 'without-' + feature_name + warnings.filterwarnings("ignore", w) + + +COMPILER_SETTINGS = { + "libraries": ["yaml"], + "include_dirs": ["yaml"], + "library_dirs": [], + "define_macros": [], +} + +MODULES = ["_yaml"] +EXTRA_LIBRARIES = {} +EXTRA_SRC = {} + + +def localpath(*args): + return op.abspath(op.join(op.dirname(__file__), *args)) class build_ext(_build_ext): + @staticmethod + def _make_extensions(): + """Produce a list of Extension instances which can be passed to + cythonize(). + + This is the point at which custom directories, MPI options, etc. + enter the build process. + """ + settings = COMPILER_SETTINGS.copy() + + # TODO: should this only be done on UNIX? + if os.name != "nt": + settings["runtime_library_dirs"] = settings["library_dirs"] + + def make_extension(module): + sources = [localpath("yaml", module + ".pyx")] + EXTRA_SRC.get(module, []) + settings["libraries"] += EXTRA_LIBRARIES.get(module, []) + print(("yaml." + module, sources, settings)) + ext = Extension("yaml." + module, sources, **settings) + ext._needs_stub = False + return ext + + return [make_extension(m) for m in MODULES] def run(self): - optional = True - disabled = True - for ext in self.extensions: - with_ext = self.distribution.ext_status(ext) - if with_ext is None: - disabled = False - elif with_ext: - optional = False - disabled = False - break - if disabled: - return - try: - _build_ext.run(self) - except DistutilsPlatformError: - exc = sys.exc_info()[1] - if optional: - log.warn(str(exc)) - log.warn("skipping build_ext") - else: - raise - - def get_source_files(self): - self.check_extensions_list(self.extensions) - filenames = [] - for ext in self.extensions: - if with_cython: - self.cython_sources(ext.sources, ext) - for filename in ext.sources: - filenames.append(filename) - base = os.path.splitext(filename)[0] - for ext in ['c', 'h', 'pyx', 'pxd']: - filename = '%s.%s' % (base, ext) - if filename not in filenames and os.path.isfile(filename): - filenames.append(filename) - return filenames - - def get_outputs(self): - self.check_extensions_list(self.extensions) - outputs = [] - for ext in self.extensions: - fullname = self.get_ext_fullname(ext.name) - filename = os.path.join(self.build_lib, - self.get_ext_filename(fullname)) - if os.path.isfile(filename): - outputs.append(filename) - return outputs - - def build_extensions(self): - self.check_extensions_list(self.extensions) - for ext in self.extensions: - with_ext = self.distribution.ext_status(ext) - if with_ext is not None and not with_ext: - continue - if with_cython: - ext.sources = self.cython_sources(ext.sources, ext) - try: - self.build_extension(ext) - except (CompileError, LinkError): - if with_ext is not None: - raise - log.warn("Error compiling module, falling back to pure Python") + """Distutils calls this method to run the command""" + + from Cython.Build import cythonize + + # This allows ccache to recognise the files when pip builds in a temp + # directory. It speeds up repeatedly running tests through tox with + # ccache configured (CC="ccache gcc"). It should have no effect if + # ccache is not in use. + os.environ["CCACHE_BASEDIR"] = op.dirname(op.abspath(__file__)) + os.environ["CCACHE_NOHASHDIR"] = "1" + + # Run Cython + print("Executing cythonize()") + self.extensions = cythonize( + self._make_extensions(), force=self.force, language_level=3 + ) + + print(self.extensions) + for ex in self.extensions: + ex._needs_stub = False + # Perform the build + super().run() class test(Command): @@ -299,16 +240,11 @@ def run(self): download_url=DOWNLOAD_URL, classifiers=CLASSIFIERS, project_urls=PROJECT_URLS, - - package_dir={'': 'lib'}, - packages=['yaml', '_yaml'], - ext_modules=[ - Extension('yaml._yaml', ['yaml/_yaml.pyx'], - 'libyaml', "LibYAML bindings", LIBYAML_CHECK, - libraries=['yaml']), - ], - + package_dir={"": "lib"}, + packages=["yaml", "_yaml"], + # To trick build into running build_ext + ext_modules=[Extension("yaml.x", ["x.c"])], distclass=Distribution, cmdclass=cmdclass, - python_requires='>=3.6', + python_requires=">=3.6", )