Implement interface for calling PEP 517 hooks

colcon_core/python_project/hook_caller/__init__.py

@@ -0,0 +1,160 @@
+# Copyright 2022 Open Source Robotics Foundation, Inc.
+# Licensed under the Apache License, Version 2.0
+
+import asyncio
+from contextlib import AbstractContextManager
+import json
+import os
+import sys
+
+from colcon_core.python_project.hook_caller import _call_hook
+from colcon_core.python_project.hook_caller import _list_hooks
+from colcon_core.python_project.spec import load_and_cache_spec
+from colcon_core.subprocess import run
+
+
+class _SubprocessTransport(AbstractContextManager):
+
+    def __enter__(self):
+        self.child_in, self.parent_out = os.pipe()
+        self.parent_in, self.child_out = os.pipe()
+
+        try:
+            import msvcrt
+        except ImportError:
+            os.set_inheritable(self.child_in, True)
+            self.pass_in = self.child_in
+            os.set_inheritable(self.child_out, True)
+            self.pass_out = self.child_out
+        else:
+            self.pass_in = msvcrt.get_osfhandle(self.child_in)
+            os.set_handle_inheritable(self.pass_in, True)
+            self.pass_out = msvcrt.get_osfhandle(self.child_out)
+            os.set_handle_inheritable(self.pass_out, True)
+
+        return self
+
+    def __exit__(self, exc_type, exc_value, traceback):
+        os.close(self.parent_out)
+        os.close(self.parent_in)
+        os.close(self.child_out)
+        os.close(self.child_in)
+
+
+class AsyncHookCaller:
+    """Calls PEP 517 style hooks asynchronously in a new process."""
+
+    def __init__(
+        self, backend_name, *, project_path=None, env=None,
+        stdout_callback=None, stderr_callback=None,
+    ):
+        """
+        Initialize a new AsyncHookCaller.
+
+        :param backend_name: The name of the PEP 517 build backend.
+        :param project_path: Path to the project's root directory.
+        :param env: Environment variables to use when invoking hooks.
+        :param stdout_callback: Callback for stdout from the hook invocation.
+        :param stderr_callback: Callback for stderr from the hook invocation.
+        """
+        self._backend_name = backend_name
+        self._project_path = str(project_path) if project_path else None
+        self._env = dict(env if env is not None else os.environ)
+        self._stdout_callback = stdout_callback
+        self._stderr_callback = stderr_callback
+
+    @property
+    def backend_name(self):
+        """Get the name of the backend to call hooks on."""
+        return self._backend_name
+
+    @property
+    def env(self):
+        """Get the environment variables to use when invoking hooks."""
+        return self._env
+
+    async def list_hooks(self):
+        """
+        Call into the backend to list implemented hooks.
+
+        This function lists all callable methods on the backend, which may
+        include more than just the hook names.
+
+        :returns: List of hook names.
+        """
+        args = [
+            sys.executable, _list_hooks.__file__,
+            self._backend_name]
+        process = await run(
+            args, None, self._stderr_callback,
+            cwd=self._project_path, env=self.env,
+            capture_output=True)
+        process.check_returncode()
+        hook_names = [
+            line.strip().decode() for line in process.stdout.splitlines()]
+        return [
+            hook for hook in hook_names if hook and not hook.startswith('_')]
+
+    async def call_hook(self, hook_name, **kwargs):
+        """
+        Call the given hook with given arguments.
+
+        :param hook_name: Name of the hook to call.
+        """
+        with _SubprocessTransport() as transport:
+            args = [
+                sys.executable, _call_hook.__file__,
+                self._backend_name, hook_name,
+                str(transport.pass_in), str(transport.pass_out)]
+
+            def write_args():
+                with os.fdopen(
+                    os.dup(transport.parent_out), 'w',
+                    encoding='utf-8',
+                ) as f:
+                    f.write(json.dumps(kwargs) + '\n')
+
+            # Start writing in a background thread to prevent deadlocks
+            # when the argument payload is larger than the pipe buffer size.
+            loop = asyncio.get_event_loop()
+            write_task = loop.run_in_executor(None, write_args)
+
+            have_callbacks = self._stdout_callback or self._stderr_callback
+            process = await run(
+                args, self._stdout_callback, self._stderr_callback,
+                cwd=self._project_path, env=self.env, close_fds=False,
+                capture_output=not have_callbacks)
+            process.check_returncode()
+
+            # Ensure the write has completed and propagate any write exceptions
+            await write_task
+
+            with os.fdopen(
+                os.dup(transport.parent_in), 'r',
+                encoding='utf-8',
+            ) as f:
+                res = json.loads(f.readline())
+            return res
+
+
+def get_hook_caller(desc, **kwargs):
+    """
+    Create a new AsyncHookCaller instance for a package descriptor.
+
+    :param desc: The package descriptor
+    """
+    spec = load_and_cache_spec(desc)
+    backend_path = spec['build-system'].get('backend-path')
+    if backend_path:
+        # TODO: This isn't *technically* the beginning of sys.path
+        #       as PEP 517 calls for, but it's pretty darn close.
+        kwargs['env'] = {
+            **kwargs.get('env', os.environ),
+            'PYTHONDONTWRITEBYTECODE': '1',
+        }
+        pythonpath = kwargs['env'].get('PYTHONPATH', '')
+        kwargs['env']['PYTHONPATH'] = os.pathsep.join(
+            backend_path + ([pythonpath] if pythonpath else []))
+    return AsyncHookCaller(
+        spec['build-system']['build-backend'],
+        project_path=desc.path, **kwargs)

colcon_core/python_project/hook_caller/_call_hook.py

@@ -0,0 +1,29 @@
+# Copyright 2022 Open Source Robotics Foundation, Inc.
+# Licensed under the Apache License, Version 2.0
+
+from importlib import import_module
+import json
+import os
+import sys
+
+
+if __name__ == '__main__':
+    backend_name, hook_name, child_in, child_out = sys.argv[1:]
+    try:
+        import msvcrt
+    except ImportError:
+        pass
+    else:
+        child_in = msvcrt.open_osfhandle(int(child_in), os.O_RDONLY)
+        child_out = msvcrt.open_osfhandle(int(child_out), 0)
+    if ':' in backend_name:
+        backend_module_name, backend_object_name = backend_name.split(':', 2)
+        backend_module = import_module(backend_module_name)
+        backend = getattr(backend_module, backend_object_name)
+    else:
+        backend = import_module(backend_name)
+    with os.fdopen(int(child_in), 'r', encoding='utf-8') as f:
+        kwargs = json.loads(f.readline()) or {}
+    res = getattr(backend, hook_name)(**kwargs)
+    with os.fdopen(int(child_out), 'w', encoding='utf-8') as f:
+        f.write(json.dumps(res) + '\n')

colcon_core/python_project/hook_caller/_list_hooks.py

@@ -0,0 +1,19 @@
+# Copyright 2023 Open Source Robotics Foundation, Inc.
+# Licensed under the Apache License, Version 2.0
+
+from importlib import import_module
+import sys
+
+
+if __name__ == '__main__':
+    backend_name = sys.argv[1]
+    if ':' in backend_name:
+        backend_module_name, backend_object_name = backend_name.split(':', 2)
+        backend_module = import_module(backend_module_name)
+        backend = getattr(backend_module, backend_object_name)
+    else:
+        backend = import_module(backend_name)
+
+    for attr in dir(backend):
+        if callable(getattr(backend, attr)):
+            print(attr)

colcon_core/python_project/hook_caller_decorator/__init__.py

@@ -0,0 +1,93 @@
+# Copyright 2022 Open Source Robotics Foundation, Inc.
+# Licensed under the Apache License, Version 2.0
+
+import traceback
+
+from colcon_core.logging import colcon_logger
+from colcon_core.plugin_system import instantiate_extensions
+from colcon_core.plugin_system import order_extensions_by_priority
+from colcon_core.python_project.hook_caller import get_hook_caller
+
+logger = colcon_logger.getChild(__name__)
+
+
+class HookCallerDecoratorExtensionPoint:
+    """
+    The interface for PEP 517 hook caller decorator extensions.
+
+    For each instance the attribute `HOOK_CALLER_DECORATOR_NAME` is being
+    set to the basename of the entry point registering the extension.
+    """
+
+    """The version of the hook caller decorator extension interface."""
+    EXTENSION_POINT_VERSION = '1.0'
+
+    """The default priority of hook caller decorator extensions."""
+    PRIORITY = 100
+
+    def decorate_hook_caller(self, *, hook_caller):
+        """
+        Decorate a hook caller to perform additional functionality.
+
+        This method must be overridden in a subclass.
+
+        :param hook_caller: The hook caller
+        :returns: A decorator
+        """
+        raise NotImplementedError()
+
+
+def get_hook_caller_extensions():
+    """
+    Get the available hook caller decorator extensions.
+
+    The extensions are ordered by their priority and entry point name.
+
+    :rtype: OrderedDict
+    """
+    extensions = instantiate_extensions(__name__)
+    for name, extension in extensions.items():
+        extension.HOOK_CALLER_DECORATOR_NAME = name
+    return order_extensions_by_priority(extensions)
+
+
+def decorate_hook_caller(hook_caller):
+    """
+    Decorate the hook caller using hook caller decorator extensions.
+
+    :param hook_caller: The hook caller
+
+    :returns: The decorated parser
+    """
+    extensions = get_hook_caller_extensions()
+    for extension in extensions.values():
+        logger.log(
+            1, 'decorate_hook_caller() %s',
+            extension.HOOK_CALLER_DECORATOR_NAME)
+        try:
+            decorated_hook_caller = extension.decorate_hook_caller(
+                hook_caller=hook_caller)
+            assert hasattr(decorated_hook_caller, 'call_hook'), \
+                'decorate_hook_caller() should return something to call hooks'
+        except Exception as e:  # noqa: F841
+            # catch exceptions raised in decorator extension
+            exc = traceback.format_exc()
+            logger.error(
+                'Exception in hook caller decorator extension '
+                f"'{extension.HOOK_CALLER_DECORATOR_NAME}': {e}\n{exc}")
+            # skip failing extension, continue with next one
+        else:
+            hook_caller = decorated_hook_caller
+
+    return hook_caller
+
+
+def get_decorated_hook_caller(desc, **kwargs):
+    """
+    Create and decorate a hook caller instance for a package descriptor.
+
+    :param desc: The package descriptor
+    """
+    hook_caller = get_hook_caller(desc, **kwargs)
+    decorated_hook_caller = decorate_hook_caller(hook_caller)
+    return decorated_hook_caller

test/spell_check.words

@@ -5,6 +5,7 @@ argparse
 asyncio
 autouse
 backend
+backends
 backported
 bazqux
 blocklist
@@ -75,6 +76,7 @@ noqa
 notestscollected
 openpty
 optionxform
+osfhandle
 pathlib
 pkgname
 pkgs

test/test_flake8.py

@@ -25,7 +25,9 @@ def test_flake8():
         show_source=True,
     )
     style_guide_tests = get_style_guide(
-        extend_ignore=['D100', 'D101', 'D102', 'D103', 'D104', 'D105', 'D107'],
+        extend_ignore=[
+            'D100', 'D101', 'D102', 'D103', 'D104', 'D105', 'D106', 'D107',
+        ],
         show_source=True,
     )