diff --git a/docs/reference/cli.md b/docs/reference/cli.md index a7a47d0..4681e8e 100644 --- a/docs/reference/cli.md +++ b/docs/reference/cli.md @@ -35,6 +35,8 @@ pypeline run [OPTIONS] | `--force-run` | FLAG | `false` | Force execution ignoring dependencies | | `--dry-run` | FLAG | `false` | Show what would run | | `-i`, `--input` | TEXT | — | Input as `key=value` (repeatable) | +| `--command` | TEXT | — | Command appended as last step; pypeline waits for it | +| `--application` | TEXT | — | Application started detached once the pipeline finished | ### `pypeline --version` @@ -62,3 +64,21 @@ pypeline run -i env=prod -i debug=true # Preview without running pypeline run --print ``` + +## Running Commands and Applications + +Both options use the environment collected by the pipeline: every `install_dirs` entry is in `PATH` and every `env_vars` entry is set. This is what makes them different from running the command yourself afterwards. + +`--command` is appended to the schedule as a regular step. It runs last, pypeline waits for it and fails if it fails. Because it is a step, it is skipped by `--dry-run` and supports `${{ inputs. }}` placeholders for the inputs declared in the configuration file. + +```shell +pypeline run --step CreateVEnv --command "pytest -v" +pypeline run --command "ruff check ${{ inputs.target }}" -i target=src +``` + +`--application` adds no step. The pipeline runs, the application is started as a detached process and pypeline exits without waiting for it. Use it to launch editors, GUIs or shells that need the installed tools. + +```shell +pypeline run --step ScoopInstall --application "code ." +pypeline run --application "cmd" +``` diff --git a/src/pypeline/domain/execution_context.py b/src/pypeline/domain/execution_context.py index ef2efd8..e8075ab 100644 --- a/src/pypeline/domain/execution_context.py +++ b/src/pypeline/domain/execution_context.py @@ -30,13 +30,16 @@ def add_install_dirs(self, install_dirs: List[Path]) -> None: def add_env_vars(self, env_vars: Dict[str, Any]) -> None: self.env_vars.update(env_vars) - def create_process_executor(self, command: List[str | Path], cwd: Optional[Path] = None) -> SubprocessExecutor: + def create_process_env(self) -> Dict[str, str]: env = os.environ.copy() env.update(self.env_vars) env["PATH"] = os.pathsep.join([path.absolute().as_posix() for path in self.install_dirs] + [env["PATH"]]) + return env + + def create_process_executor(self, command: List[str | Path], cwd: Optional[Path] = None) -> SubprocessExecutor: # When started from a windows shell (e.g. cmd on Jenkins) the shell parameter must be set to True shell = True if os.name == "nt" else False - return SubprocessExecutor(command, cwd=cwd, env=env, shell=shell) + return SubprocessExecutor(command, cwd=cwd, env=self.create_process_env(), shell=shell) def create_artifacts_locator(self) -> ProjectArtifactsLocator: return ProjectArtifactsLocator(self.project_root_dir) diff --git a/src/pypeline/main.py b/src/pypeline/main.py index e646bd3..f09fe92 100644 --- a/src/pypeline/main.py +++ b/src/pypeline/main.py @@ -1,6 +1,9 @@ +import os +import shlex +import subprocess import sys from pathlib import Path -from typing import List, Optional +from typing import Dict, List, Optional import typer from py_app_dev.core.exceptions import UserNotificationException @@ -8,11 +11,11 @@ from pypeline import __version__ from pypeline.domain.execution_context import ExecutionContext -from pypeline.domain.pipeline import PipelineConfigIterator +from pypeline.domain.pipeline import PipelineConfigIterator, PipelineStepConfig, PipelineStepReference from pypeline.domain.project_slurper import ProjectSlurper from pypeline.inputs_parser import InputsParser from pypeline.kickstart.create import KickstartProject -from pypeline.pypeline import PipelineScheduler, PipelineStepsExecutor +from pypeline.pypeline import PipelineScheduler, PipelineStepsExecutor, RunCommandClassFactory package_name = "pypeline" @@ -21,6 +24,24 @@ def package_version_file() -> Path: return Path(__file__).parent / "__init__.py" +def start_detached_process(command: List[str], env: Dict[str, str], cwd: Path) -> None: + """Start a process which outlives pypeline, e.g. an IDE which needs the tools installed by the pipeline in its PATH.""" + logger.info(f"Starting detached process: {' '.join(command)}") + # A new process group (Windows) or session (POSIX) detaches the child, so it survives pypeline exiting. + if sys.platform == "win32": + creation_flags = subprocess.CREATE_NEW_PROCESS_GROUP + else: + creation_flags = 0 + subprocess.Popen( + command, + cwd=cwd, + env=env, + shell=True if os.name == "nt" else False, # noqa: S603 # The command comes from the user, same as for any other pipeline step. + creationflags=creation_flags, + start_new_session=os.name != "nt", + ) + + app = typer.Typer( name=package_name, help="Configure and execute pipelines with Python (similar to GitHub workflows or Jenkins pipelines).", no_args_is_help=True, add_completion=False ) @@ -60,6 +81,8 @@ def run( "-i", help="Provide input parameters as key=value pairs (e.g., -i name=value -i flag=true).", ), + command: Optional[str] = typer.Option(None, help="Command to run as last pipeline step. Pypeline waits for it to finish."), + application: Optional[str] = typer.Option(None, help="Application to start once the pipeline finished. It is started as detached process, pypeline does not wait for it."), ) -> None: project_dir = project_dir.absolute() project_slurper = ProjectSlurper(project_dir, config_file) @@ -75,7 +98,10 @@ def run( raise UserNotificationException("No pipeline found in the configuration.") # Schedule the steps to run steps_references = PipelineScheduler[ExecutionContext](project_slurper.pipeline, project_dir).get_steps_to_run(step, single) - if not steps_references: + if command: + command_step_class = RunCommandClassFactory().create_step_class(PipelineStepConfig(step="Command", run=command), project_dir) + steps_references.append(PipelineStepReference(None, command_step_class)) + if not steps_references and not application: logger.info("No steps to run.") return # Parse the inputs @@ -86,7 +112,14 @@ def run( inputs_dict = InputsParser.from_inputs_definitions(input_definitions).parse_inputs(inputs) else: inputs_dict = {} - PipelineStepsExecutor[ExecutionContext](ExecutionContext(project_dir, inputs=inputs_dict), steps_references, force_run, dry_run).run() + execution_context = ExecutionContext(project_dir, inputs=inputs_dict) + PipelineStepsExecutor[ExecutionContext](execution_context, steps_references, force_run, dry_run).run() + if application and not dry_run: + try: + application_command = shlex.split(application, posix=(os.name != "nt")) + except ValueError as exc: + raise UserNotificationException(f"Could not parse the application command '{application}': {exc}") from exc + start_detached_process(application_command, execution_context.create_process_env(), project_dir) def main() -> None: try: diff --git a/tests/conftest.py b/tests/conftest.py index 0a790bd..a2a6562 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -71,6 +71,9 @@ def run(self) -> None: data = self.execution_context.data_registry.find_data(MyData) if not data: raise ValueError("Data not found") + class MyEnvStep(BaseStep): + def update_execution_context(self) -> None: + self.execution_context.add_env_vars({"PYPELINE_TEST_VAR": "from_step"}) class MyInputsChecker(BaseStep): def run(self) -> None: input = self.execution_context.inputs.get("my_input") diff --git a/tests/test_main.py b/tests/test_main.py index b7c7ea0..ec1eacf 100644 --- a/tests/test_main.py +++ b/tests/test_main.py @@ -1,194 +1,264 @@ -import sys -import textwrap -from pathlib import Path -from typing import List - -import pytest -from py_app_dev.core.exceptions import UserNotificationException -from typer.testing import CliRunner - -from pypeline.domain.artifacts import ProjectArtifactsLocator -from pypeline.main import __version__, app, package_version_file - -runner = CliRunner() - - -def test_run(artifacts_locator: ProjectArtifactsLocator) -> None: - result = runner.invoke( - app, - [ - "run", - "--project-dir", - artifacts_locator.project_root_dir.as_posix(), - "--step", - "MyStep", - "--single", - ], - ) - assert result.exit_code == 0 - assert artifacts_locator.build_dir.joinpath("custom/MyStep.deps.json").exists(), "Step dependencies file shall exist" - - result = runner.invoke( - app, - ["run", "--project-dir", artifacts_locator.project_root_dir.as_posix(), "--print"], - ) - assert result.exit_code == 0 - - -def test_run_multiple_steps(artifacts_locator: ProjectArtifactsLocator) -> None: - result = runner.invoke( - app, - [ - "run", - "--project-dir", - artifacts_locator.project_root_dir.as_posix(), - "--step", - "MyStep", - "--step", - "CheckPython", - "--single", - ], - ) - assert result.exit_code == 0 - - -def test_run_custom_config_file(artifacts_locator: ProjectArtifactsLocator) -> None: - artifacts_locator.config_file.unlink() - assert not artifacts_locator.config_file.exists() - result = runner.invoke( - app, - [ - "run", - "--project-dir", - artifacts_locator.project_root_dir.as_posix(), - "--print", - "--config-file", - "my_pypeline.yaml", - ], - ) - assert result.exit_code == 0 - - -def test_run_no_step(artifacts_locator: ProjectArtifactsLocator) -> None: - result = runner.invoke( - app, - ["run", "--project-dir", artifacts_locator.project_root_dir.as_posix(), "--step", "NonExistentStep", "--single"], - ) - assert result.exit_code == 1 - assert isinstance(result.exception, UserNotificationException) - - -def test_run_no_pypeline_config(tmp_path: Path) -> None: - result = runner.invoke(app, ["run", "--project-dir", tmp_path.as_posix()], catch_exceptions=True) - assert result.exit_code == 1 - assert isinstance(result.exception, UserNotificationException) - - -def test_pipeline_read_inputs(project: Path) -> None: - config_file = project / "pypeline.yaml" - config_file.write_text( - textwrap.dedent("""\ - inputs: - my_input: - type: string - description: My input description - required: true - my_bool: - type: boolean - description: Some boolean input - pipeline: - - step: MyInputsChecker - file: my_python_file.py - """) - ) - result = runner.invoke( - app, - [ - "run", - "--project-dir", - project.as_posix(), - "--input", - "my_input=my_value", - "--input", - "my_bool", - ], - ) - assert result.exit_code == 0 - - -def test_no_inputs_are_configured(project: Path) -> None: - config_file = project / "pypeline.yaml" - config_file.write_text( - textwrap.dedent("""\ - pipeline: - - step: MyStep - file: my_python_file.py - """) - ) - result = runner.invoke( - app, - [ - "run", - "--project-dir", - project.as_posix(), - "--input", - "my_input=my_value", - ], - ) - assert result.exit_code == 1 - assert "Inputs are not accepted because there are no inputs defined" in str(result.exception) - - -@pytest.fixture -def kickstart_files() -> List[str]: - return [ - ".gitignore", - "pypeline.ps1", - "pypeline.yaml", - "pyproject.toml", - "steps/my_step.py", - ] - - -def test_init_default(kickstart_files: List[str], tmp_path: Path) -> None: - result = runner.invoke(app, ["init", "--project-dir", tmp_path.as_posix()]) - assert result.exit_code == 0 - - for file in kickstart_files: - assert tmp_path.joinpath(file).exists(), f"{file} shall exist" - - result = runner.invoke(app, ["run", "--project-dir", tmp_path.as_posix()]) - assert result.exit_code == 0 - - -def test_init_pins_python_version_to_current_interpreter(tmp_path: Path) -> None: - result = runner.invoke(app, ["init", "--project-dir", tmp_path.as_posix()]) - assert result.exit_code == 0 - - generated = tmp_path.joinpath("pypeline.yaml").read_text() - # The generated config pins python_version (stable bootstrap env) and drops python_executable. - assert f'python_version: "{sys.version_info.major}.{sys.version_info.minor}"' in generated - assert "python_executable" not in generated - - -def test_init_with_force_in_non_empty_directory(project: Path, kickstart_files: List[str]) -> None: - result = runner.invoke(app, ["init", "--force", "--project-dir", project.as_posix()]) - assert result.exit_code == 0 - - for file in kickstart_files: - assert project.joinpath(file).exists(), f"{file} shall exist" - - -def test_help() -> None: - result = runner.invoke(app, ["--help"]) - assert result.exit_code == 0 - - result = runner.invoke(app, ["--version"]) - assert result.exit_code == 0 - assert f"pypeline {__version__}" in result.output - - -def test_package_version_file(): - file = package_version_file() - assert file.exists(), "Package version file shall exist" - assert __version__ in file.read_text() +import sys +import textwrap +import time +from pathlib import Path +from typing import List + +import pytest +from py_app_dev.core.exceptions import UserNotificationException +from typer.testing import CliRunner + +from pypeline.domain.artifacts import ProjectArtifactsLocator +from pypeline.main import __version__, app, package_version_file + +runner = CliRunner() + + +def test_run(artifacts_locator: ProjectArtifactsLocator) -> None: + result = runner.invoke( + app, + [ + "run", + "--project-dir", + artifacts_locator.project_root_dir.as_posix(), + "--step", + "MyStep", + "--single", + ], + ) + assert result.exit_code == 0 + assert artifacts_locator.build_dir.joinpath("custom/MyStep.deps.json").exists(), "Step dependencies file shall exist" + + result = runner.invoke( + app, + ["run", "--project-dir", artifacts_locator.project_root_dir.as_posix(), "--print"], + ) + assert result.exit_code == 0 + + +def test_run_multiple_steps(artifacts_locator: ProjectArtifactsLocator) -> None: + result = runner.invoke( + app, + [ + "run", + "--project-dir", + artifacts_locator.project_root_dir.as_posix(), + "--step", + "MyStep", + "--step", + "CheckPython", + "--single", + ], + ) + assert result.exit_code == 0 + + +@pytest.fixture +def project_with_env_step(project: Path) -> Path: + """Project with a single step exporting PYPELINE_TEST_VAR and a script writing it to the file given as argument.""" + project.joinpath("pypeline.yaml").write_text( + textwrap.dedent("""\ + pipeline: + - step: MyEnvStep + file: my_python_file.py + """) + ) + project.joinpath("write_env_var_to_file.py").write_text("import os, pathlib, sys; pathlib.Path(sys.argv[1]).write_text(os.environ['PYPELINE_TEST_VAR'])") + return project + + +def test_run_command_runs_as_last_step_with_the_pipeline_environment(project_with_env_step: Path) -> None: + result = runner.invoke( + app, + [ + "run", + "--project-dir", + project_with_env_step.as_posix(), + "--command", + "python write_env_var_to_file.py command.txt", + ], + ) + assert result.exit_code == 0 + assert project_with_env_step.joinpath("command.txt").read_text() == "from_step" + + +@pytest.mark.parametrize("option", ["--command", "--application"]) +def test_run_dry_run_neither_runs_the_command_nor_starts_the_application(project_with_env_step: Path, option: str) -> None: + result = runner.invoke( + app, + [ + "run", + "--project-dir", + project_with_env_step.as_posix(), + "--dry-run", + option, + "python write_env_var_to_file.py dry_run.txt", + ], + ) + assert result.exit_code == 0 + + time.sleep(0.3) # A started process would have written the file long before this + assert not project_with_env_step.joinpath("dry_run.txt").exists() + + +def test_run_application_is_spawned_detached_with_the_pipeline_environment(project_with_env_step: Path) -> None: + result = runner.invoke( + app, + [ + "run", + "--project-dir", + project_with_env_step.as_posix(), + "--application", + "python write_env_var_to_file.py application.txt", + ], + ) + assert result.exit_code == 0 + + written_file = project_with_env_step.joinpath("application.txt") + for _ in range(100): + if written_file.exists(): + break + time.sleep(0.1) + assert written_file.read_text() == "from_step", "The detached application shall run with the environment collected by the pipeline" + + +def test_run_custom_config_file(artifacts_locator: ProjectArtifactsLocator) -> None: + artifacts_locator.config_file.unlink() + assert not artifacts_locator.config_file.exists() + result = runner.invoke( + app, + [ + "run", + "--project-dir", + artifacts_locator.project_root_dir.as_posix(), + "--print", + "--config-file", + "my_pypeline.yaml", + ], + ) + assert result.exit_code == 0 + + +def test_run_no_step(artifacts_locator: ProjectArtifactsLocator) -> None: + result = runner.invoke( + app, + ["run", "--project-dir", artifacts_locator.project_root_dir.as_posix(), "--step", "NonExistentStep", "--single"], + ) + assert result.exit_code == 1 + assert isinstance(result.exception, UserNotificationException) + + +def test_run_no_pypeline_config(tmp_path: Path) -> None: + result = runner.invoke(app, ["run", "--project-dir", tmp_path.as_posix()], catch_exceptions=True) + assert result.exit_code == 1 + assert isinstance(result.exception, UserNotificationException) + + +def test_pipeline_read_inputs(project: Path) -> None: + config_file = project / "pypeline.yaml" + config_file.write_text( + textwrap.dedent("""\ + inputs: + my_input: + type: string + description: My input description + required: true + my_bool: + type: boolean + description: Some boolean input + pipeline: + - step: MyInputsChecker + file: my_python_file.py + """) + ) + result = runner.invoke( + app, + [ + "run", + "--project-dir", + project.as_posix(), + "--input", + "my_input=my_value", + "--input", + "my_bool", + ], + ) + assert result.exit_code == 0 + + +def test_no_inputs_are_configured(project: Path) -> None: + config_file = project / "pypeline.yaml" + config_file.write_text( + textwrap.dedent("""\ + pipeline: + - step: MyStep + file: my_python_file.py + """) + ) + result = runner.invoke( + app, + [ + "run", + "--project-dir", + project.as_posix(), + "--input", + "my_input=my_value", + ], + ) + assert result.exit_code == 1 + assert "Inputs are not accepted because there are no inputs defined" in str(result.exception) + + +@pytest.fixture +def kickstart_files() -> List[str]: + return [ + ".gitignore", + "pypeline.ps1", + "pypeline.yaml", + "pyproject.toml", + "steps/my_step.py", + ] + + +def test_init_default(kickstart_files: List[str], tmp_path: Path) -> None: + result = runner.invoke(app, ["init", "--project-dir", tmp_path.as_posix()]) + assert result.exit_code == 0 + + for file in kickstart_files: + assert tmp_path.joinpath(file).exists(), f"{file} shall exist" + + result = runner.invoke(app, ["run", "--project-dir", tmp_path.as_posix()]) + assert result.exit_code == 0 + + +def test_init_pins_python_version_to_current_interpreter(tmp_path: Path) -> None: + result = runner.invoke(app, ["init", "--project-dir", tmp_path.as_posix()]) + assert result.exit_code == 0 + + generated = tmp_path.joinpath("pypeline.yaml").read_text() + # The generated config pins python_version (stable bootstrap env) and drops python_executable. + assert f'python_version: "{sys.version_info.major}.{sys.version_info.minor}"' in generated + assert "python_executable" not in generated + + +def test_init_with_force_in_non_empty_directory(project: Path, kickstart_files: List[str]) -> None: + result = runner.invoke(app, ["init", "--force", "--project-dir", project.as_posix()]) + assert result.exit_code == 0 + + for file in kickstart_files: + assert project.joinpath(file).exists(), f"{file} shall exist" + + +def test_help() -> None: + result = runner.invoke(app, ["--help"]) + assert result.exit_code == 0 + + result = runner.invoke(app, ["--version"]) + assert result.exit_code == 0 + assert f"pypeline {__version__}" in result.output + + +def test_package_version_file(): + file = package_version_file() + assert file.exists(), "Package version file shall exist" + assert __version__ in file.read_text()