From 6bf3746593cd3fc3029275cf8c0dcbd30522bd58 Mon Sep 17 00:00:00 2001 From: Brandon Kammerdiener Date: Thu, 27 Aug 2026 11:26:56 -0400 Subject: [PATCH 01/10] add iaprof parser --- gprofiler/utils/iaprof.py | 165 ++++++++++++++++++++++++++++++++++++++ tests/test_iaprof.py | 103 ++++++++++++++++++++++++ 2 files changed, 268 insertions(+) create mode 100644 gprofiler/utils/iaprof.py create mode 100644 tests/test_iaprof.py diff --git a/gprofiler/utils/iaprof.py b/gprofiler/utils/iaprof.py new file mode 100644 index 000000000..7478a79ca --- /dev/null +++ b/gprofiler/utils/iaprof.py @@ -0,0 +1,165 @@ +# +# Copyright (C) 2026 Intel Corporation +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +from collections import Counter, defaultdict +from dataclasses import dataclass +from typing import Dict, List, Optional, Tuple + +from gprofiler.gprofiler_types import ProcessToStackSampleCounters +from gprofiler.log import get_logger_adapter + +logger = get_logger_adapter(__name__) + +STALL_REASONS = ( + "active", + "control", + "pipestall", + "send", + "dist_acc", + "sbid", + "sync", + "inst_fetch", + "other", + "tdr", +) + + +@dataclass(frozen=True) +class _Kernel: + pid: int + frames: Tuple[str, ...] + + +class IaprofParser: + def __init__(self) -> None: + self._strings: Dict[int, str] = {} + self._kernel: Optional[_Kernel] = None + self._profiles: ProcessToStackSampleCounters = defaultdict(Counter) + self._bad_line_count = 0 + self._bad_lines: List[str] = [] + + def parse_line(self, line: str) -> None: + line = line.rstrip("\n") + if not line: + return + + record_type = line.partition("\t")[0] + if record_type == "interval": + self._kernel = None + try: + self._parse_interval(line) + except ValueError: + self._record_bad_line(line) + return + + try: + if record_type == "string": + self._parse_string(line) + elif record_type == "kernel": + self._kernel = None + self._parse_kernel(line) + elif record_type == "eustall": + self._parse_eustall(line) + elif record_type == "metric": + self._parse_metric(line) + else: + raise ValueError + except (KeyError, ValueError): + self._record_bad_line(line) + + return + + def take_snapshot(self) -> ProcessToStackSampleCounters: + profiles = self._profiles + self._profiles = defaultdict(Counter) + + if self._bad_line_count: + logger.warning( + f"Skipped {self._bad_line_count} malformed iaprof records (showing up to 8):\n" + + "\n".join(self._bad_lines) + ) + self._bad_line_count = 0 + self._bad_lines = [] + return profiles + + def _parse_string(self, line: str) -> None: + _, string_id, value = line.split("\t", maxsplit=2) + parsed_id = int(string_id) + if parsed_id <= 0: + raise ValueError + self._strings[parsed_id] = value + + def _parse_interval(self, line: str) -> None: + _, interval, timestamp = line.split("\t") + int(interval) + float(timestamp) + + def _parse_metric(self, line: str) -> None: + _, _, value = line.split("\t") + float(value) + + def _parse_kernel(self, line: str) -> None: + _, gpu_address, comm_id, pid, cpu_stack_id, source_id, symbol_id = line.split("\t") + int(gpu_address, 0) + parsed_pid = int(pid) + if parsed_pid <= 0: + raise ValueError + + cpu_frames = tuple( + self._frame(frame, "[unknown]") for frame in self._string_value(cpu_stack_id).split(";") if frame + ) + frames = ( + self._frame(self._string_value(comm_id), "[unknown]"), + *cpu_frames, + "-", + f"{self._frame(self._string_value(source_id), '[unknown]')}_[G]", + f"{self._frame(self._string_value(symbol_id), '[unknown]')}_[G]", + ) + self._kernel = _Kernel(parsed_pid, frames) + + def _parse_eustall(self, line: str) -> None: + if self._kernel is None: + raise ValueError + + fields = line.split("\t") + if len(fields) != 13: + raise ValueError + offset = fields[1] + int(offset, 0) + instruction = self._frame(self._string_value(fields[2]), "[unknown]") + counts = tuple(int(value) for value in fields[3:]) + if any(count < 0 for count in counts): + raise ValueError + + for reason, count in zip(STALL_REASONS, counts): + if count: + stack = ";".join((*self._kernel.frames, f"{instruction}_[g]", f"{reason}_[g]", f"{offset}_[g]")) + self._profiles[self._kernel.pid][stack] += count + + def _string_value(self, string_id: str) -> str: + parsed_id = int(string_id) + if parsed_id == 0: + return "" + return self._strings[parsed_id] + + @staticmethod + def _frame(value: str, fallback: str) -> str: + return value.strip().replace(";", "|") or fallback + + def _record_bad_line(self, line: str) -> None: + self._bad_line_count += 1 + if len(self._bad_lines) < 8: + self._bad_lines.append(line) diff --git a/tests/test_iaprof.py b/tests/test_iaprof.py new file mode 100644 index 000000000..9440dfe7f --- /dev/null +++ b/tests/test_iaprof.py @@ -0,0 +1,103 @@ +# +# Copyright (C) 2026 Intel Corporation +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +import logging +from collections import Counter +from typing import Iterable + +from pytest import LogCaptureFixture + +from gprofiler.utils.iaprof import STALL_REASONS, IaprofParser + + +def _parse_lines(parser: IaprofParser, lines: Iterable[str]) -> None: + for line in lines: + parser.parse_line(line) + + +def test_iaprof_eustall_stacks() -> None: + parser = IaprofParser() + _parse_lines( + parser, + [ + "string\t1\tapp", + "string\t2\t_start;main", + "string\t3\tsource;cpp", + "string\t4\tkernel", + "string\t5\tmov", + "interval\t0\t1.0", + "kernel\t0x1000\t1\t42\t2\t3\t4", + "eustall\t0x10\t5\t1\t2\t3\t4\t5\t6\t7\t8\t9\t10", + ], + ) + + prefix = "app;_start;main;-;source|cpp_[G];kernel_[G];mov_[g]" + assert parser.take_snapshot() == { + 42: Counter({f"{prefix};{reason}_[g];0x10_[g]": count for reason, count in zip(STALL_REASONS, range(1, 11))}) + } + + +def test_iaprof_snapshot_resets_counters_and_retains_strings() -> None: + parser = IaprofParser() + _parse_lines( + parser, + [ + "string\t1\tapp", + "string\t2\tkernel", + "string\t3\tmov", + "kernel\t0x1000\t1\t42\t0\t0\t2", + "eustall\t0x10\t3\t2\t0\t0\t0\t0\t0\t0\t0\t0\t0", + ], + ) + parser.parse_line("interval\t1\t2.0") + + stack = "app;-;[unknown]_[G];kernel_[G];mov_[g];active_[g];0x10_[g]" + assert parser.take_snapshot() == {42: Counter({stack: 2})} + assert parser.take_snapshot() == {} + + _parse_lines( + parser, + [ + "kernel\t0x1000\t1\t42\t0\t0\t2", + "eustall\t0x10\t3\t3\t0\t0\t0\t0\t0\t0\t0\t0\t0", + ], + ) + assert parser.take_snapshot() == {42: Counter({stack: 3})} + + +def test_iaprof_interval_clears_kernel() -> None: + parser = IaprofParser() + _parse_lines(parser, ["string\t1\tapp", "kernel\t0x1000\t1\t42\t0\t0\t0"]) + + parser.parse_line("interval\t1\t2.0") + parser.parse_line("eustall\t0x10\t0\t1\t0\t0\t0\t0\t0\t0\t0\t0\t0") + assert parser.take_snapshot() == {} + + +def test_iaprof_bad_records_are_skipped(caplog: LogCaptureFixture) -> None: + parser = IaprofParser() + caplog.set_level(logging.WARNING) + _parse_lines( + parser, + [ + "kernel\t0x1000\t0\t0\t0\t0\t0", + "eustall\t0x10\t0\t1\t0\t0\t0\t0\t0\t0\t0\t0\t0", + "invalid", + ], + ) + + assert parser.take_snapshot() == {} + assert "Skipped 3 malformed iaprof records" in caplog.text From 5f2438dbca114c761c31ca031edb3129861d99d0 Mon Sep 17 00:00:00 2001 From: Brandon Kammerdiener Date: Wed, 23 Sep 2026 15:18:25 -0400 Subject: [PATCH 02/10] PoC iaprof integration --- gprofiler/consts.py | 3 + gprofiler/main.py | 52 ++++--- gprofiler/profilers/__init__.py | 3 +- gprofiler/profilers/factory.py | 8 +- gprofiler/profilers/iaprof.py | 129 ++++++++++++++++++ gprofiler/profilers/perf.py | 8 +- gprofiler/profilers/profiler_base.py | 9 ++ gprofiler/utils/iaprof.py | 74 ++++++++-- gprofiler/utils/iaprof_process.py | 174 ++++++++++++++++++++++++ tests/test_iaprof.py | 77 ++++++++++- tests/test_iaprof_process.py | 173 +++++++++++++++++++++++ tests/test_iaprof_profiler.py | 196 +++++++++++++++++++++++++++ 12 files changed, 862 insertions(+), 44 deletions(-) create mode 100644 gprofiler/profilers/iaprof.py create mode 100644 gprofiler/utils/iaprof_process.py create mode 100644 tests/test_iaprof_process.py create mode 100644 tests/test_iaprof_profiler.py diff --git a/gprofiler/consts.py b/gprofiler/consts.py index 31c5edf09..1cb4fb2ef 100644 --- a/gprofiler/consts.py +++ b/gprofiler/consts.py @@ -1,2 +1,5 @@ CPU_PROFILING_MODE = "cpu" +GPU_PROFILING_MODE = "gpu" DEFAULT_PROFILING_MODE = CPU_PROFILING_MODE +DEFAULT_GPU_PROFILING_FREQUENCY = 100 +MAX_GPU_PROFILING_FREQUENCY = 1000 diff --git a/gprofiler/main.py b/gprofiler/main.py index 9887e2e47..60ca8a271 100644 --- a/gprofiler/main.py +++ b/gprofiler/main.py @@ -43,7 +43,12 @@ DEFAULT_UPLOAD_TIMEOUT, ProfilerAPIClient, ) -from gprofiler.consts import CPU_PROFILING_MODE +from gprofiler.consts import ( + CPU_PROFILING_MODE, + DEFAULT_GPU_PROFILING_FREQUENCY, + GPU_PROFILING_MODE, + MAX_GPU_PROFILING_FREQUENCY, +) from gprofiler.containers_client import ContainerNamesClient from gprofiler.diagnostics import log_diagnostics, set_diagnostics from gprofiler.dynamic_profiling_management.heartbeat import DynamicGProfilerManager, HeartbeatClient @@ -62,7 +67,7 @@ from gprofiler.profiler_state import ProfilerState from gprofiler.profilers.factory import get_profilers from gprofiler.profilers.perf import SystemProfiler -from gprofiler.profilers.profiler_base import NoopProfiler, ProcessProfilerBase, ProfilerInterface +from gprofiler.profilers.profiler_base import NoopProfiler, ProcessProfilerBase, ProfilerInterface, SystemProfilerBase from gprofiler.profilers.registry import get_profilers_registry from gprofiler.state import State, init_state from gprofiler.system_metrics import Metrics, NoopSystemMetricsMonitor, SystemMetricsMonitor, SystemMetricsMonitorBase @@ -312,7 +317,10 @@ def start(self) -> None: # Check if system should skip continuous profilers due to process count skip_system_profilers = False - if self._profiler_state.max_system_processes_for_system_profilers > 0: + if ( + isinstance(self.system_profiler, SystemProfiler) + and self._profiler_state.max_system_processes_for_system_profilers > 0 + ): try: total_processes = len(list(process_iter())) if total_processes > self._profiler_state.max_system_processes_for_system_profilers: @@ -333,18 +341,12 @@ def start(self) -> None: for prof in list(self.all_profilers): try: # Skip system profilers if threshold exceeded - if ( - skip_system_profilers - and hasattr(prof, "_is_system_wide_profiler") - and prof._is_system_wide_profiler() - ): + if skip_system_profilers and prof is self.system_profiler: logger.info(f"Skipping {prof.__class__.__name__} due to high system process count") continue prof.start() except Exception: - # the SystemProfiler is handled separately - let the user run with '--perf-mode none' if they - # wish so. if prof is self.system_profiler: raise @@ -380,15 +382,12 @@ def _snapshot(self) -> None: future_name = future.name # type: ignore # hack, add the profiler's name to the Future object logger.exception(f"{future_name} profiling failed") - local_end_time = local_start_time + datetime.timedelta(seconds=(time.monotonic() - monotonic_start_time)) - try: system_result = system_future.result() except Exception: - logger.critical( - "Running perf failed; consider running gProfiler with '--perf-mode disabled' to avoid using perf", - ) + logger.critical(f"Running {self.system_profiler.name} failed") raise + local_end_time = local_start_time + datetime.timedelta(seconds=(time.monotonic() - monotonic_start_time)) metadata = ( get_current_metadata(cast(ProfileMetadata, self._static_metadata)) if self._collect_metadata @@ -431,6 +430,8 @@ def _snapshot(self) -> None: "sampling_frequency": self.system_profiler._frequency, } ) + elif isinstance(self.system_profiler, SystemProfilerBase): + metadata.update(self.system_profiler.get_profile_metadata()) else: # NoopProfiler - use default values metadata.update( @@ -669,8 +670,10 @@ def parse_cmd_args() -> configargparse.Namespace: "--profiling-frequency", type=positive_integer, dest="frequency", - help=f"Profiler frequency in Hz (default: {DEFAULT_SAMPLING_FREQUENCY}), to be used only in CPU profiling " - f"(--mode=cpu, also the default mode)", + help=( + f"CPU sampling or GPU output frequency in Hz (defaults: CPU {DEFAULT_SAMPLING_FREQUENCY}, " + f"GPU {DEFAULT_GPU_PROFILING_FREQUENCY})" + ), ) parser.add_argument( "-d", @@ -710,10 +713,10 @@ def parse_cmd_args() -> configargparse.Namespace: parser.add_argument( "--mode", dest="profiling_mode", - choices=["cpu", "allocation", "none"], - default="cpu", + choices=[CPU_PROFILING_MODE, GPU_PROFILING_MODE, "allocation", "none"], + default=CPU_PROFILING_MODE, help="Select gProfiler's profiling mode, default is %(default)s, available options are " - "%(choices)s; allocation will profile only Java processes", + "%(choices)s; gpu will profile Intel GPUs and allocation will profile only Java processes", ) parser.add_argument( "--alloc-interval", @@ -1114,6 +1117,15 @@ def parse_cmd_args() -> configargparse.Namespace: if not args.alloc_interval: args.alloc_interval = DEFAULT_ALLOC_INTERVAL args.frequency = humanfriendly.parse_size(args.alloc_interval, binary=True) + elif args.profiling_mode == GPU_PROFILING_MODE: + if args.alloc_interval is not None: + parser.error("--alloc-interval is only allowed in allocation profiling (--mode=allocation)") + if args.rootless: + parser.error("--rootless is not supported in gpu profiling") + if args.frequency is None: + args.frequency = DEFAULT_GPU_PROFILING_FREQUENCY + elif args.frequency > MAX_GPU_PROFILING_FREQUENCY: + parser.error(f"-f|--frequency cannot exceed {MAX_GPU_PROFILING_FREQUENCY} in gpu profiling") if args.subcommand == UPLOAD_FILE_SUBCOMMAND: args.upload_results = True diff --git a/gprofiler/profilers/__init__.py b/gprofiler/profilers/__init__.py index 295a83f19..261be972f 100644 --- a/gprofiler/profilers/__init__.py +++ b/gprofiler/profilers/__init__.py @@ -4,6 +4,7 @@ from gprofiler.profilers.python import PythonProfiler if is_linux(): + from gprofiler.profilers.iaprof import IaprofProfiler from gprofiler.profilers.java import JavaProfiler from gprofiler.profilers.perf import SystemProfiler from gprofiler.profilers.php import PHPSpyProfiler @@ -12,6 +13,6 @@ __all__ = ["PythonProfiler", "DotnetProfiler"] if is_linux(): - __all__ += ["JavaProfiler", "PHPSpyProfiler", "RbSpyProfiler", "SystemProfiler"] + __all__ += ["IaprofProfiler", "JavaProfiler", "PHPSpyProfiler", "RbSpyProfiler", "SystemProfiler"] del is_linux diff --git a/gprofiler/profilers/factory.py b/gprofiler/profilers/factory.py index 3e128dd7e..c2137530a 100644 --- a/gprofiler/profilers/factory.py +++ b/gprofiler/profilers/factory.py @@ -5,7 +5,7 @@ from gprofiler.metadata.system_metadata import get_arch from gprofiler.platform import is_windows from gprofiler.profilers.perf import SystemProfiler -from gprofiler.profilers.profiler_base import NoopProfiler +from gprofiler.profilers.profiler_base import NoopProfiler, SystemProfilerBase from gprofiler.profilers.registry import get_profilers_registry from gprofiler.utils import is_profiler_disabled @@ -20,10 +20,10 @@ def get_profilers( user_args: "UserArgs", **profiler_init_kwargs: Any -) -> Tuple[Union["SystemProfiler", "NoopProfiler"], List["ProcessProfilerBase"]]: +) -> Tuple[Union["SystemProfilerBase", "NoopProfiler"], List["ProcessProfilerBase"]]: profiling_mode = user_args.get("profiling_mode") process_profilers_instances: List["ProcessProfilerBase"] = [] - system_profiler: Union["SystemProfiler", "NoopProfiler"] = NoopProfiler() + system_profiler: Union["SystemProfilerBase", "NoopProfiler"] = NoopProfiler() # When custom event is specified, only use perf (SystemProfiler), disable all language profilers custom_event_mode = user_args.get("perf_event") is not None @@ -81,7 +81,7 @@ def get_profilers( ) sys.exit(1) else: - if isinstance(profiler_instance, SystemProfiler): + if isinstance(profiler_instance, SystemProfilerBase): system_profiler = profiler_instance else: # In custom event mode, skip all process profilers diff --git a/gprofiler/profilers/iaprof.py b/gprofiler/profilers/iaprof.py new file mode 100644 index 000000000..c02dc2108 --- /dev/null +++ b/gprofiler/profilers/iaprof.py @@ -0,0 +1,129 @@ +# +# Copyright (C) 2026 Intel Corporation +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +from typing import Optional + +from packaging.version import Version + +from gprofiler.consts import GPU_PROFILING_MODE, MAX_GPU_PROFILING_FREQUENCY +from gprofiler.exceptions import StopEventSetException +from gprofiler.gprofiler_types import ProcessToProfileData, ProfileData, positive_integer +from gprofiler.metadata import ProfileMetadata +from gprofiler.profiler_state import ProfilerState +from gprofiler.profilers.profiler_base import SystemProfilerBase +from gprofiler.profilers.registry import ProfilerArgument, register_profiler +from gprofiler.utils import run_process +from gprofiler.utils.iaprof_process import IaprofProcess + +IAPROF_INTERVAL_MS = 10 +IAPROF_SNAPSHOT_TIMEOUT = 5 + + +def frequency_to_iaprof_interval(frequency: int) -> int: + if not 1 <= frequency <= MAX_GPU_PROFILING_FREQUENCY: + raise ValueError(f"iaprof frequency must be between 1 and {MAX_GPU_PROFILING_FREQUENCY}") + return round(1000 / frequency) + + +@register_profiler( + "Iaprof", + possible_modes=["enabled", "disabled"], + default_mode="enabled", + supported_archs=["x86_64"], # not confirmed working on ARM, but should in theory + supported_profiling_modes=[GPU_PROFILING_MODE], + profiler_mode_argument_help="Enable or disable Intel GPU profiling with iaprof", + disablement_help="Disable Intel GPU profiling with iaprof", + profiler_arguments=[ + ProfilerArgument( + "--iaprof-path", + dest="iaprof_path", + default="iaprof", + help="Path to the iaprof executable. Default: %(default)s", + ), + ProfilerArgument( + "--iaprof-eu-stall-subsample", + dest="iaprof_eu_stall_subsample", + type=positive_integer, + default=100, + help="Process one out of every N EU stall samples. Default: %(default)s", + ), + ], +) +class IaprofProfiler(SystemProfilerBase): + MINIMUM_SUPPORTED_VERSION = Version("2026.3") + _VERSION_TIMEOUT = 5 + + def __init__( + self, + frequency: int, + duration: int, + profiler_state: ProfilerState, + iaprof_mode: str, + iaprof_path: str, + iaprof_eu_stall_subsample: int, + min_duration: int = 0, + ) -> None: + assert iaprof_mode == "enabled" + super().__init__(frequency, duration, profiler_state, min_duration) + self._iaprof_path = iaprof_path + self._iaprof_version: Optional[str] = None + self._output_interval_ms = frequency_to_iaprof_interval(frequency) + self._eu_stall_subsample = iaprof_eu_stall_subsample + self._iaprof = IaprofProcess( + self._iaprof_path, + self._output_interval_ms, + self._eu_stall_subsample, + ) + + def get_profile_metadata(self) -> ProfileMetadata: + assert self._iaprof_version is not None + return { + "iaprof_version": self._iaprof_version, + "iaprof_output_interval_ms": self._output_interval_ms, + "iaprof_eu_stall_subsample": self._eu_stall_subsample, + } + + def _get_iaprof_version(self) -> str: + result = run_process( + [self._iaprof_path, "--version"], + stop_event=self._profiler_state.stop_event, + timeout=self._VERSION_TIMEOUT, + pdeathsigger=False, + ) + version_output = result.stdout.decode().strip() or result.stderr.decode().strip() + version = Version(version_output) + if version < self.MINIMUM_SUPPORTED_VERSION: + raise RuntimeError( + f"iaprof {version} is unsupported; minimum version is {self.MINIMUM_SUPPORTED_VERSION}" + ) + return str(version) + + def start(self) -> None: + self._iaprof_version = self._get_iaprof_version() + self._iaprof.start() + + def stop(self) -> None: + self._iaprof.stop() + + def snapshot(self) -> ProcessToProfileData: + if self._profiler_state.stop_event.wait(self._duration): + raise StopEventSetException + + stacks_by_pid = self._iaprof.snapshot(IAPROF_SNAPSHOT_TIMEOUT) + return { + pid: ProfileData(stacks, None, None, self._profiler_state.get_container_name(pid)) + for pid, stacks in stacks_by_pid.items() + } diff --git a/gprofiler/profilers/perf.py b/gprofiler/profilers/perf.py index 7d6f4edd5..afaa33604 100644 --- a/gprofiler/profilers/perf.py +++ b/gprofiler/profilers/perf.py @@ -39,7 +39,7 @@ from gprofiler.metadata.application_metadata import ApplicationMetadata from gprofiler.profiler_state import ProfilerState from gprofiler.profilers.node import clean_up_node_maps, generate_map_for_node_processes, get_node_processes -from gprofiler.profilers.profiler_base import ProfilerBase +from gprofiler.profilers.profiler_base import SystemProfilerBase from gprofiler.profilers.registry import ProfilerArgument, register_profiler from gprofiler.utils.perf import discover_appropriate_perf_event, parse_perf_script_from_iterator, valid_perf_pid from gprofiler.utils.perf_process import PerfProcess @@ -154,7 +154,7 @@ def add_highest_avg_depth_stacks_per_process( " and instead only concatenate runtime-specific profilers results", supported_profiling_modes=["cpu"], ) -class SystemProfiler(ProfilerBase): +class SystemProfiler(SystemProfilerBase): """ We are running 2 perfs in parallel - one with DWARF and one with FP, and then we merge their results. This improves the results from software that is compiled without frame pointers, @@ -162,10 +162,6 @@ class SystemProfiler(ProfilerBase): versions of Go processes. """ - def _is_system_wide_profiler(self) -> bool: - """Perf is a system-wide profiler that can be disabled on busy systems.""" - return True - def __init__( self, frequency: int, diff --git a/gprofiler/profilers/profiler_base.py b/gprofiler/profilers/profiler_base.py index f2285d8dc..f3b535ac6 100644 --- a/gprofiler/profilers/profiler_base.py +++ b/gprofiler/profilers/profiler_base.py @@ -34,6 +34,7 @@ from gprofiler.exceptions import StopEventSetException from gprofiler.gprofiler_types import ProcessToProfileData, ProfileData, ProfilingErrorStack, StackToSampleCount from gprofiler.log import get_logger_adapter +from gprofiler.metadata import ProfileMetadata from gprofiler.profiler_state import ProfilerState from gprofiler.utils import limit_frequency from gprofiler.utils.process import process_comm @@ -118,6 +119,14 @@ def __init__( ) +class SystemProfilerBase(ProfilerBase): + def _is_system_wide_profiler(self) -> bool: + return True + + def get_profile_metadata(self) -> ProfileMetadata: + return {} + + class NoopProfiler(ProfilerInterface): """ No-op profiler - used as a drop-in replacement for runtime profilers, when they are disabled. diff --git a/gprofiler/utils/iaprof.py b/gprofiler/utils/iaprof.py index 7478a79ca..02536f325 100644 --- a/gprofiler/utils/iaprof.py +++ b/gprofiler/utils/iaprof.py @@ -15,8 +15,10 @@ # from collections import Counter, defaultdict +from concurrent.futures import Future from dataclasses import dataclass -from typing import Dict, List, Optional, Tuple +from threading import Lock +from typing import Dict, Iterable, List, Optional, Tuple from gprofiler.gprofiler_types import ProcessToStackSampleCounters from gprofiler.log import get_logger_adapter @@ -51,19 +53,19 @@ def __init__(self) -> None: self._bad_line_count = 0 self._bad_lines: List[str] = [] - def parse_line(self, line: str) -> None: + def parse_line(self, line: str) -> Optional[int]: line = line.rstrip("\n") if not line: - return + return None record_type = line.partition("\t")[0] if record_type == "interval": self._kernel = None try: - self._parse_interval(line) + return self._parse_interval(line) except ValueError: self._record_bad_line(line) - return + return None try: if record_type == "string": @@ -80,7 +82,7 @@ def parse_line(self, line: str) -> None: except (KeyError, ValueError): self._record_bad_line(line) - return + return None def take_snapshot(self) -> ProcessToStackSampleCounters: profiles = self._profiles @@ -102,10 +104,11 @@ def _parse_string(self, line: str) -> None: raise ValueError self._strings[parsed_id] = value - def _parse_interval(self, line: str) -> None: + def _parse_interval(self, line: str) -> int: _, interval, timestamp = line.split("\t") - int(interval) + parsed_interval = int(interval) float(timestamp) + return parsed_interval def _parse_metric(self, line: str) -> None: _, _, value = line.split("\t") @@ -163,3 +166,58 @@ def _record_bad_line(self, line: str) -> None: self._bad_line_count += 1 if len(self._bad_lines) < 8: self._bad_lines.append(line) + + +class IaprofOutputReader: + def __init__(self) -> None: + self._parser = IaprofParser() + self._lock = Lock() + self._ready: Future[int] = Future() + self._snapshot: Optional[Future[ProcessToStackSampleCounters]] = None + self._error: Optional[BaseException] = None + + def read(self, lines: Iterable[str]) -> None: + try: + for line in lines: + interval = self._parser.parse_line(line) + if interval is not None: + if not self._ready.done(): + self._ready.set_result(interval) + self._publish_snapshot() + except BaseException as error: + self._finish(error) + else: + self._finish(EOFError("iaprof output closed")) + + def wait_until_ready(self, timeout: float) -> int: + return self._ready.result(timeout) + + def request_snapshot(self) -> Future[ProcessToStackSampleCounters]: + snapshot: Future[ProcessToStackSampleCounters] = Future() + with self._lock: + if self._snapshot is not None: + raise RuntimeError("iaprof snapshot already requested") + if self._error is None: + self._snapshot = snapshot + else: + snapshot.set_exception(self._error) + return snapshot + + def _publish_snapshot(self) -> None: + with self._lock: + if self._snapshot is None: + return + profiles = self._parser.take_snapshot() + snapshot = self._snapshot + self._snapshot = None + snapshot.set_result(profiles) + + def _finish(self, error: BaseException) -> None: + with self._lock: + self._error = error + snapshot = self._snapshot + self._snapshot = None + if not self._ready.done(): + self._ready.set_exception(error) + if snapshot is not None and not snapshot.done(): + snapshot.set_exception(error) diff --git a/gprofiler/utils/iaprof_process.py b/gprofiler/utils/iaprof_process.py new file mode 100644 index 000000000..d8d1d9103 --- /dev/null +++ b/gprofiler/utils/iaprof_process.py @@ -0,0 +1,174 @@ +# +# Copyright (C) 2026 Intel Corporation +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +import signal +from collections import deque +from subprocess import Popen, TimeoutExpired +from threading import Lock, Thread +from typing import Deque, List, Optional, TextIO + +from gprofiler.gprofiler_types import ProcessToStackSampleCounters +from gprofiler.log import get_logger_adapter +from gprofiler.utils import cleanup_process_reference, start_process +from gprofiler.utils.iaprof import IaprofOutputReader + +logger = get_logger_adapter(__name__) + + +class IaprofProcessError(RuntimeError): + pass + + +class IaprofProcess: + _STDERR_LINES = 200 + + def __init__( + self, + path: str, + interval_ms: int, + eu_stall_subsample: int, + startup_timeout: float = 10, + stop_timeout: float = 5, + ) -> None: + if interval_ms <= 0 or eu_stall_subsample <= 0: + raise ValueError("iaprof interval and EU stall subsample must be positive") + + self._path = path + self._interval_ms = interval_ms + self._eu_stall_subsample = eu_stall_subsample + self._startup_timeout = startup_timeout + self._stop_timeout = stop_timeout + self._process: Optional[Popen[str]] = None + self._reader: Optional[IaprofOutputReader] = None + self._stdout_thread: Optional[Thread] = None + self._stderr_thread: Optional[Thread] = None + self._stderr_lines: Deque[str] = deque(maxlen=self._STDERR_LINES) + self._stderr_lock = Lock() + + def start(self) -> None: + if self._process is not None: + raise RuntimeError("iaprof is already started") + + self._reader = IaprofOutputReader() + self._stderr_lines.clear() + process = start_process( + self._command(), + text=True, + encoding="utf-8", + errors="replace", + bufsize=1, + ) + assert process.stdout is not None and process.stderr is not None + self._process = process + + try: + self._stdout_thread = Thread( + target=self._reader.read, + args=(process.stdout,), + name="iaprof-stdout", + ) + self._stderr_thread = Thread( + target=self._read_stderr, + args=(process.stderr,), + name="iaprof-stderr", + ) + self._stdout_thread.start() + self._stderr_thread.start() + self._reader.wait_until_ready(self._startup_timeout) + except Exception as error: + self._stop_after_failure() + raise IaprofProcessError(self._error_message("iaprof failed to start")) from error + except BaseException: + self._stop_after_failure() + raise + + def snapshot(self, timeout: float) -> ProcessToStackSampleCounters: + if self._process is None or self._reader is None: + raise RuntimeError("iaprof is not started") + try: + return self._reader.request_snapshot().result(timeout) + except Exception as error: + self._stop_after_failure() + raise IaprofProcessError(self._error_message("iaprof snapshot failed")) from error + + def stop(self) -> None: + process = self._process + if process is None: + return + + if process.poll() is None: + process.send_signal(signal.SIGINT) + try: + process.wait(timeout=self._stop_timeout) + except TimeoutExpired: + process.kill() + try: + process.wait(timeout=self._stop_timeout) + except TimeoutExpired as error: + raise IaprofProcessError("iaprof failed to stop after SIGKILL") from error + + try: + self._join_threads() + finally: + for stream in (process.stdin, process.stdout, process.stderr): + if stream is not None: + stream.close() + cleanup_process_reference(process) + self._process = None + self._reader = None + self._stdout_thread = None + self._stderr_thread = None + + logger.info("Stopped iaprof", exit_code=process.returncode, stderr=self.stderr) + + def _stop_after_failure(self) -> None: + try: + self.stop() + except Exception: + logger.exception("Failed to stop iaprof after failure") + + def is_running(self) -> bool: + return self._process is not None and self._process.poll() is None + + @property + def stderr(self) -> str: + with self._stderr_lock: + return "".join(self._stderr_lines) + + def _command(self) -> List[str]: + return [ + self._path, + f"--interval={self._interval_ms}", + f"--eu-stall-subsample={self._eu_stall_subsample}", + ] + + def _read_stderr(self, lines: TextIO) -> None: + for line in lines: + with self._stderr_lock: + self._stderr_lines.append(line) + + def _join_threads(self) -> None: + threads = (self._stdout_thread, self._stderr_thread) + for thread in threads: + if thread is not None and thread.ident is not None: + thread.join(timeout=self._stop_timeout) + + if any(thread is not None and thread.is_alive() for thread in threads): + raise IaprofProcessError("iaprof output readers failed to stop") + + def _error_message(self, message: str) -> str: + stderr = self.stderr.strip() + return f"{message}: {stderr}" if stderr else message diff --git a/tests/test_iaprof.py b/tests/test_iaprof.py index 9440dfe7f..f8bb77ffb 100644 --- a/tests/test_iaprof.py +++ b/tests/test_iaprof.py @@ -15,12 +15,16 @@ # import logging +import os from collections import Counter -from typing import Iterable +from contextlib import contextmanager +from threading import Thread +from typing import Iterable, Iterator, TextIO, Tuple +import pytest from pytest import LogCaptureFixture -from gprofiler.utils.iaprof import STALL_REASONS, IaprofParser +from gprofiler.utils.iaprof import STALL_REASONS, IaprofOutputReader, IaprofParser def _parse_lines(parser: IaprofParser, lines: Iterable[str]) -> None: @@ -28,6 +32,28 @@ def _parse_lines(parser: IaprofParser, lines: Iterable[str]) -> None: parser.parse_line(line) +@contextmanager +def _reader_pipe() -> Iterator[Tuple[IaprofOutputReader, TextIO]]: + read_fd, write_fd = os.pipe() + read_stream = os.fdopen(read_fd) + output = os.fdopen(write_fd, "w") + reader = IaprofOutputReader() + thread = Thread(target=reader.read, args=(read_stream,)) + thread.start() + try: + yield reader, output + finally: + output.close() + thread.join(timeout=5) + read_stream.close() + assert not thread.is_alive() + + +def _write_lines(output: TextIO, *lines: str) -> None: + output.write("\n".join(lines) + "\n") + output.flush() + + def test_iaprof_eustall_stacks() -> None: parser = IaprofParser() _parse_lines( @@ -62,7 +88,7 @@ def test_iaprof_snapshot_resets_counters_and_retains_strings() -> None: "eustall\t0x10\t3\t2\t0\t0\t0\t0\t0\t0\t0\t0\t0", ], ) - parser.parse_line("interval\t1\t2.0") + assert parser.parse_line("interval\t1\t2.0") == 1 stack = "app;-;[unknown]_[G];kernel_[G];mov_[g];active_[g];0x10_[g]" assert parser.take_snapshot() == {42: Counter({stack: 2})} @@ -82,8 +108,8 @@ def test_iaprof_interval_clears_kernel() -> None: parser = IaprofParser() _parse_lines(parser, ["string\t1\tapp", "kernel\t0x1000\t1\t42\t0\t0\t0"]) - parser.parse_line("interval\t1\t2.0") - parser.parse_line("eustall\t0x10\t0\t1\t0\t0\t0\t0\t0\t0\t0\t0\t0") + assert parser.parse_line("interval\t1\t2.0") == 1 + assert parser.parse_line("eustall\t0x10\t0\t1\t0\t0\t0\t0\t0\t0\t0\t0\t0") is None assert parser.take_snapshot() == {} @@ -101,3 +127,44 @@ def test_iaprof_bad_records_are_skipped(caplog: LogCaptureFixture) -> None: assert parser.take_snapshot() == {} assert "Skipped 3 malformed iaprof records" in caplog.text + + +def test_iaprof_output_reader_snapshots_at_interval() -> None: + with _reader_pipe() as (reader, output): + snapshot = reader.request_snapshot() + _write_lines( + output, + "string\t1\tapp", + "string\t2\tkernel", + "string\t3\tmov", + "kernel\t0x1000\t1\t42\t0\t0\t2", + "eustall\t0x10\t3\t2\t0\t0\t0\t0\t0\t0\t0\t0\t0", + "interval\t1\t2.0", + ) + + stack = "app;-;[unknown]_[G];kernel_[G];mov_[g];active_[g];0x10_[g]" + assert snapshot.result(timeout=5) == {42: Counter({stack: 2})} + + snapshot = reader.request_snapshot() + _write_lines( + output, + "kernel\t0x1000\t1\t42\t0\t0\t2", + "eustall\t0x10\t3\t3\t0\t0\t0\t0\t0\t0\t0\t0\t0", + "interval\t2\t3.0", + ) + assert snapshot.result(timeout=5) == {42: Counter({stack: 3})} + + +def test_iaprof_output_reader_empty_snapshot() -> None: + with _reader_pipe() as (reader, output): + snapshot = reader.request_snapshot() + _write_lines(output, "interval\t1\t2.0") + assert snapshot.result(timeout=5) == {} + + +def test_iaprof_output_reader_wakes_snapshot_on_eof() -> None: + with _reader_pipe() as (reader, output): + snapshot = reader.request_snapshot() + output.close() + with pytest.raises(EOFError, match="iaprof output closed"): + snapshot.result(timeout=5) diff --git a/tests/test_iaprof_process.py b/tests/test_iaprof_process.py new file mode 100644 index 000000000..ac9391606 --- /dev/null +++ b/tests/test_iaprof_process.py @@ -0,0 +1,173 @@ +# +# Copyright (C) 2026 Intel Corporation +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +import subprocess +import sys +from pathlib import Path +from time import monotonic +from typing import Any, List + +import pytest +from pytest import MonkeyPatch + +import gprofiler.utils.iaprof_process as iaprof_process +from gprofiler.utils import start_process +from gprofiler.utils.iaprof_process import IaprofProcess, IaprofProcessError + + +def _write_executable(path: Path, source: str) -> None: + path.write_text(f"#!{sys.executable}\n{source}") + path.chmod(0o755) + + +def _disable_pdeathsigger(monkeypatch: MonkeyPatch) -> None: + def start_without_pdeathsigger(command: List[str], **kwargs: Any) -> subprocess.Popen[str]: + return start_process(command, pdeathsigger=False, **kwargs) + + monkeypatch.setattr(iaprof_process, "start_process", start_without_pdeathsigger) + + +def test_iaprof_process_snapshot_and_stop(tmp_path: Path, monkeypatch: MonkeyPatch) -> None: + _disable_pdeathsigger(monkeypatch) + executable = tmp_path / "fake-iaprof" + _write_executable( + executable, + """ +import signal +import sys +import time + +running = True + +def stop(*args): + global running + running = False + +signal.signal(signal.SIGINT, stop) +print("interval\\t0\\t1.0", flush=True) +print("string\\t1\\tapp") +print("string\\t2\\tkernel") +print("string\\t3\\tmov") +interval = 1 +while running: + print("kernel\\t0x1000\\t1\\t42\\t0\\t0\\t2") + print("eustall\\t0x10\\t3\\t1\\t0\\t0\\t0\\t0\\t0\\t0\\t0\\t0\\t0") + print(f"interval\\t{interval}\\t2.0", flush=True) + interval += 1 + time.sleep(0.01) +print("stopped", file=sys.stderr, flush=True) +""", + ) + process = IaprofProcess(str(executable), 10, 100, startup_timeout=2, stop_timeout=1) + + try: + process.start() + profiles = process.snapshot(timeout=2) + stack = "app;-;[unknown]_[G];kernel_[G];mov_[g];active_[g];0x10_[g]" + assert profiles[42][stack] >= 1 + finally: + process.stop() + + assert not process.is_running() + assert "stopped" in process.stderr + + +def test_iaprof_process_start_failure(tmp_path: Path, monkeypatch: MonkeyPatch) -> None: + _disable_pdeathsigger(monkeypatch) + executable = tmp_path / "fake-iaprof" + _write_executable( + executable, + """ +import sys + +print("GPU unavailable", file=sys.stderr, flush=True) +raise SystemExit(2) +""", + ) + process = IaprofProcess(str(executable), 10, 100, startup_timeout=2, stop_timeout=1) + + with pytest.raises(IaprofProcessError, match="GPU unavailable"): + process.start() + assert not process.is_running() + + +def test_iaprof_process_kills_after_stop_timeout(tmp_path: Path, monkeypatch: MonkeyPatch) -> None: + _disable_pdeathsigger(monkeypatch) + executable = tmp_path / "fake-iaprof" + _write_executable( + executable, + """ +import signal +import time + +signal.signal(signal.SIGINT, signal.SIG_IGN) +print("interval\\t0\\t1.0", flush=True) +while True: + time.sleep(1) +""", + ) + process = IaprofProcess(str(executable), 10, 100, startup_timeout=2, stop_timeout=0.1) + process.start() + + start = monotonic() + process.stop() + + assert monotonic() - start < 1 + assert not process.is_running() + + +def test_iaprof_process_snapshot_timeout_stops_child( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + _disable_pdeathsigger(monkeypatch) + executable = tmp_path / "iaprof" + _write_executable( + executable, + """ +import signal +import sys +import time + +running = True + + +def stop(*args): + global running + running = False + + +signal.signal(signal.SIGINT, stop) +print("interval\\t0\\t1.0", flush=True) +while running: + time.sleep(0.01) +print("stopped", file=sys.stderr, flush=True) +""", + ) + + process = IaprofProcess( + str(executable), + 10, + 100, + startup_timeout=2, + stop_timeout=1, + ) + process.start() + + with pytest.raises(IaprofProcessError, match="iaprof snapshot failed"): + process.snapshot(timeout=0.5) + + assert not process.is_running() + assert "stopped" in process.stderr diff --git a/tests/test_iaprof_profiler.py b/tests/test_iaprof_profiler.py new file mode 100644 index 000000000..63b77337a --- /dev/null +++ b/tests/test_iaprof_profiler.py @@ -0,0 +1,196 @@ +# +# Copyright (C) 2026 Intel Corporation +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +import sys +from collections import Counter +from pathlib import Path +from subprocess import CompletedProcess +from typing import Optional, Tuple +from unittest.mock import Mock, patch + +import pytest +from packaging.version import InvalidVersion + +import gprofiler.profilers.iaprof as iaprof +from gprofiler.exceptions import StopEventSetException +from gprofiler.gprofiler_types import UserArgs +from gprofiler.main import parse_cmd_args +from gprofiler.profiler_state import ProfilerState +from gprofiler.profilers.factory import get_profilers +from gprofiler.profilers.iaprof import IAPROF_SNAPSHOT_TIMEOUT, IaprofProfiler, frequency_to_iaprof_interval + + +def _make_profiler(profiler_state: ProfilerState) -> Tuple[IaprofProfiler, Mock]: + profiler_state.profiling_mode = "gpu" + with patch.object(iaprof, "IaprofProcess") as process_class: + profiler = IaprofProfiler(100, 0, profiler_state, "enabled", "/opt/iaprof", 100) + process_class.assert_called_once_with("/opt/iaprof", 10, 100) + return profiler, process_class.return_value + + +def test_iaprof_profiler_lifecycle(profiler_state: ProfilerState) -> None: + profiler, process = _make_profiler(profiler_state) + + with patch.object(profiler, "_get_iaprof_version", return_value="2026.3") as get_version: + profiler.start() + profiler.stop() + + get_version.assert_called_once_with() + + process.start.assert_called_once_with() + process.stop.assert_called_once_with() + + +def test_iaprof_profiler_metadata(profiler_state: ProfilerState) -> None: + profiler, process = _make_profiler(profiler_state) + + with patch.object(profiler, "_get_iaprof_version", return_value="2026.3"): + profiler.start() + + assert profiler.get_profile_metadata() == { + "iaprof_version": "2026.3", + "iaprof_output_interval_ms": 10, + "iaprof_eu_stall_subsample": 100, + } + process.start.assert_called_once_with() + + +def test_iaprof_profiler_version(profiler_state: ProfilerState) -> None: + profiler, _ = _make_profiler(profiler_state) + result = CompletedProcess(["/opt/iaprof", "--version"], 0, b"", b"2026.3\n") + + with patch.object(iaprof, "run_process", return_value=result) as run_process: + version = profiler._get_iaprof_version() + + assert version == "2026.3" + run_process.assert_called_once_with( + ["/opt/iaprof", "--version"], + stop_event=profiler_state.stop_event, + timeout=5, + pdeathsigger=False, + ) + + +def test_iaprof_profiler_rejects_old_version(profiler_state: ProfilerState) -> None: + profiler, _ = _make_profiler(profiler_state) + result = CompletedProcess(["/opt/iaprof", "--version"], 0, b"", b"2026.2\n") + + with patch.object(iaprof, "run_process", return_value=result), pytest.raises( + RuntimeError, match="iaprof 2026.2 is unsupported; minimum version is 2026.3" + ): + profiler._get_iaprof_version() + + +def test_iaprof_profiler_rejects_invalid_version(profiler_state: ProfilerState) -> None: + profiler, _ = _make_profiler(profiler_state) + result = CompletedProcess(["/opt/iaprof", "--version"], 0, b"", b"invalid\n") + + with patch.object(iaprof, "run_process", return_value=result), pytest.raises(InvalidVersion): + profiler._get_iaprof_version() + + +def test_iaprof_gpu_mode_rejects_rootless(monkeypatch: pytest.MonkeyPatch, tmp_path: Path) -> None: + monkeypatch.setattr(sys, "argv", ["gprofiler", "--mode=gpu", "--rootless", "-o", str(tmp_path)]) + + with pytest.raises(SystemExit): + parse_cmd_args() + + +def test_iaprof_profiler_snapshot(profiler_state: ProfilerState) -> None: + profiler, process = _make_profiler(profiler_state) + stacks = Counter({"app;-;source.cpp_[G];kernel_[G];mov_[g];active_[g];0x10_[g]": 7}) + process.snapshot.return_value = {42: stacks} + + profiles = profiler.snapshot() + + process.snapshot.assert_called_once_with(IAPROF_SNAPSHOT_TIMEOUT) + assert profiles[42].stacks is stacks + assert profiles[42].appid is None + assert profiles[42].app_metadata is None + assert profiles[42].container_name == "" + + +def test_iaprof_profiler_snapshot_stops_with_gprofiler(profiler_state: ProfilerState) -> None: + profiler, process = _make_profiler(profiler_state) + profiler_state.stop_event.set() + + with pytest.raises(StopEventSetException): + profiler.snapshot() + + process.snapshot.assert_not_called() + + +def test_iaprof_profiler_selected_for_gpu_mode(profiler_state: ProfilerState) -> None: + profiler_state.profiling_mode = "gpu" + user_args: UserArgs = { + "profiling_mode": "gpu", + "frequency": 100, + "duration": 0, + "min_duration": 0, + "iaprof_mode": "enabled", + "iaprof_path": "/opt/iaprof", + "iaprof_eu_stall_subsample": 100, + } + + system_profiler, process_profilers = get_profilers(user_args, profiler_state=profiler_state) + + assert isinstance(system_profiler, IaprofProfiler) + assert process_profilers == [] + + +@pytest.mark.parametrize( + "frequency,interval_ms", + [(1, 1000), (11, 91), (100, 10), (1000, 1)], +) +def test_frequency_to_iaprof_interval(frequency: int, interval_ms: int) -> None: + assert frequency_to_iaprof_interval(frequency) == interval_ms + + +@pytest.mark.parametrize("frequency", [0, 1001]) +def test_frequency_to_iaprof_interval_rejects_out_of_range(frequency: int) -> None: + with pytest.raises(ValueError): + frequency_to_iaprof_interval(frequency) + + +@pytest.mark.parametrize("frequency", [None, 20]) +def test_iaprof_gpu_mode_args( + monkeypatch: pytest.MonkeyPatch, + tmp_path: Path, + frequency: Optional[int], +) -> None: + frequency_args = [] if frequency is None else ["-f", str(frequency)] + monkeypatch.setattr(sys, "argv", ["gprofiler", "--mode=gpu", "-o", str(tmp_path), *frequency_args]) + + args = parse_cmd_args() + + assert args.profiling_mode == "gpu" + assert args.frequency == (100 if frequency is None else frequency) + assert args.iaprof_mode == "enabled" + assert args.iaprof_path == "iaprof" + assert args.iaprof_eu_stall_subsample == 100 + + +@pytest.mark.parametrize( + "option", + [("--alloc-interval", "2mb"), ("--perf-event", "cycles"), ("-f", "1001")], +) +def test_iaprof_gpu_mode_rejects_invalid_options( + monkeypatch: pytest.MonkeyPatch, tmp_path: Path, option: Tuple[str, str] +) -> None: + monkeypatch.setattr(sys, "argv", ["gprofiler", "--mode=gpu", "-o", str(tmp_path), *option]) + + with pytest.raises(SystemExit): + parse_cmd_args() From d8f6e7831646b1e9bf1d1888c73a6ca1023184dc Mon Sep 17 00:00:00 2001 From: Brandon Kammerdiener Date: Thu, 1 Oct 2026 16:46:15 -0400 Subject: [PATCH 03/10] restart iaprof on snapshot failure just like perf --- gprofiler/profilers/iaprof.py | 2 ++ gprofiler/utils/iaprof_process.py | 10 ++++++++++ 2 files changed, 12 insertions(+) diff --git a/gprofiler/profilers/iaprof.py b/gprofiler/profilers/iaprof.py index c02dc2108..5ab0d880a 100644 --- a/gprofiler/profilers/iaprof.py +++ b/gprofiler/profilers/iaprof.py @@ -119,6 +119,8 @@ def stop(self) -> None: self._iaprof.stop() def snapshot(self) -> ProcessToProfileData: + self._iaprof.restart_if_not_running() + if self._profiler_state.stop_event.wait(self._duration): raise StopEventSetException diff --git a/gprofiler/utils/iaprof_process.py b/gprofiler/utils/iaprof_process.py index d8d1d9103..3fec21856 100644 --- a/gprofiler/utils/iaprof_process.py +++ b/gprofiler/utils/iaprof_process.py @@ -143,6 +143,16 @@ def _stop_after_failure(self) -> None: def is_running(self) -> bool: return self._process is not None and self._process.poll() is None + def restart(self) -> None: + self.stop() + self.start() + + def restart_if_not_running(self) -> None: + if not self.is_running(): + logger.warning(f"iaprof not running (unexpectedly), restarting... " + "kernels loaded before the restart may not be attributed") + self.restart() + @property def stderr(self) -> str: with self._stderr_lock: From af5551c1df92c208c58a5ac6df9dc9ce611a7fba Mon Sep 17 00:00:00 2001 From: Brandon Kammerdiener Date: Thu, 1 Oct 2026 16:54:26 -0400 Subject: [PATCH 04/10] fix formatting --- gprofiler/profilers/iaprof.py | 4 +--- gprofiler/utils/iaprof_process.py | 6 ++++-- tests/test_iaprof_process.py | 4 +--- 3 files changed, 6 insertions(+), 8 deletions(-) diff --git a/gprofiler/profilers/iaprof.py b/gprofiler/profilers/iaprof.py index 5ab0d880a..92728bae7 100644 --- a/gprofiler/profilers/iaprof.py +++ b/gprofiler/profilers/iaprof.py @@ -106,9 +106,7 @@ def _get_iaprof_version(self) -> str: version_output = result.stdout.decode().strip() or result.stderr.decode().strip() version = Version(version_output) if version < self.MINIMUM_SUPPORTED_VERSION: - raise RuntimeError( - f"iaprof {version} is unsupported; minimum version is {self.MINIMUM_SUPPORTED_VERSION}" - ) + raise RuntimeError(f"iaprof {version} is unsupported; minimum version is {self.MINIMUM_SUPPORTED_VERSION}") return str(version) def start(self) -> None: diff --git a/gprofiler/utils/iaprof_process.py b/gprofiler/utils/iaprof_process.py index 3fec21856..4e99c10ca 100644 --- a/gprofiler/utils/iaprof_process.py +++ b/gprofiler/utils/iaprof_process.py @@ -149,8 +149,10 @@ def restart(self) -> None: def restart_if_not_running(self) -> None: if not self.is_running(): - logger.warning(f"iaprof not running (unexpectedly), restarting... " - "kernels loaded before the restart may not be attributed") + logger.warning( + f"iaprof not running (unexpectedly), restarting... " + "kernels loaded before the restart may not be attributed" + ) self.restart() @property diff --git a/tests/test_iaprof_process.py b/tests/test_iaprof_process.py index ac9391606..91088df42 100644 --- a/tests/test_iaprof_process.py +++ b/tests/test_iaprof_process.py @@ -129,9 +129,7 @@ def test_iaprof_process_kills_after_stop_timeout(tmp_path: Path, monkeypatch: Mo assert not process.is_running() -def test_iaprof_process_snapshot_timeout_stops_child( - tmp_path: Path, monkeypatch: pytest.MonkeyPatch -) -> None: +def test_iaprof_process_snapshot_timeout_stops_child(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None: _disable_pdeathsigger(monkeypatch) executable = tmp_path / "iaprof" _write_executable( From 4fffd9601ec9a052ea9a1d7d820c097a6a271c31 Mon Sep 17 00:00:00 2001 From: Brandon Kammerdiener Date: Thu, 1 Oct 2026 17:22:30 -0400 Subject: [PATCH 05/10] warning->debug to reduce noise when iaprof is disabled in cpu mode --- gprofiler/profilers/factory.py | 10 ++++------ 1 file changed, 4 insertions(+), 6 deletions(-) diff --git a/gprofiler/profilers/factory.py b/gprofiler/profilers/factory.py index c2137530a..f61620f18 100644 --- a/gprofiler/profilers/factory.py +++ b/gprofiler/profilers/factory.py @@ -44,6 +44,10 @@ def get_profilers( logger.warning("--java-collect-thread-names is ignored because Java profiling is disabled") continue + if profiling_mode not in profiler_config.supported_profiling_modes: + logger.debug(f"Disabling {profiler_name} because it doesn't support profiling mode {profiling_mode!r}") + continue + supported_archs = ( profiler_config.supported_windows_archs if is_windows() else profiler_config.supported_archs ) @@ -51,12 +55,6 @@ def get_profilers( logger.warning(f"Disabling {profiler_name} because it doesn't support this architecture ({arch})") continue - if profiling_mode not in profiler_config.supported_profiling_modes: - logger.warning( - f"Disabling {profiler_name} because it doesn't support profiling mode {profiling_mode!r}" - ) - continue - profiler_kwargs = profiler_init_kwargs.copy() for key, value in user_args.items(): # Skip perf_event and perf_event_period as they're handled separately for custom events From 392dc8c8feb46b6ddf9788e24a9d02ef64539ddb Mon Sep 17 00:00:00 2001 From: Brandon Kammerdiener Date: Thu, 1 Oct 2026 17:51:15 -0400 Subject: [PATCH 06/10] iaprof: cleanup snapshot failure on next restart --- gprofiler/profilers/iaprof.py | 10 ++++++++-- gprofiler/utils/iaprof_process.py | 10 +++++++--- 2 files changed, 15 insertions(+), 5 deletions(-) diff --git a/gprofiler/profilers/iaprof.py b/gprofiler/profilers/iaprof.py index 92728bae7..988c69527 100644 --- a/gprofiler/profilers/iaprof.py +++ b/gprofiler/profilers/iaprof.py @@ -26,7 +26,7 @@ from gprofiler.profilers.profiler_base import SystemProfilerBase from gprofiler.profilers.registry import ProfilerArgument, register_profiler from gprofiler.utils import run_process -from gprofiler.utils.iaprof_process import IaprofProcess +from gprofiler.utils.iaprof_process import IaprofProcess, IaprofProcessError IAPROF_INTERVAL_MS = 10 IAPROF_SNAPSHOT_TIMEOUT = 5 @@ -122,7 +122,13 @@ def snapshot(self) -> ProcessToProfileData: if self._profiler_state.stop_event.wait(self._duration): raise StopEventSetException - stacks_by_pid = self._iaprof.snapshot(IAPROF_SNAPSHOT_TIMEOUT) + try: + stacks_by_pid = self._iaprof.snapshot(IAPROF_SNAPSHOT_TIMEOUT) + except IaprofProcessError: + if self._profiler_state.stop_event.is_set(): + raise StopEventSetException + raise + return { pid: ProfileData(stacks, None, None, self._profiler_state.get_container_name(pid)) for pid, stacks in stacks_by_pid.items() diff --git a/gprofiler/utils/iaprof_process.py b/gprofiler/utils/iaprof_process.py index 4e99c10ca..6c03a1a0b 100644 --- a/gprofiler/utils/iaprof_process.py +++ b/gprofiler/utils/iaprof_process.py @@ -14,6 +14,7 @@ # limitations under the License. # +import concurrent.futures import signal from collections import deque from subprocess import Popen, TimeoutExpired @@ -96,12 +97,15 @@ def start(self) -> None: raise def snapshot(self, timeout: float) -> ProcessToStackSampleCounters: - if self._process is None or self._reader is None: + process = self._process + if process is None or self._reader is None: raise RuntimeError("iaprof is not started") try: return self._reader.request_snapshot().result(timeout) + except concurrent.futures.TimeoutError as error: + process.kill() + raise IaprofProcessError(self._error_message("iaprof snapshot timed out")) from error except Exception as error: - self._stop_after_failure() raise IaprofProcessError(self._error_message("iaprof snapshot failed")) from error def stop(self) -> None: @@ -150,7 +154,7 @@ def restart(self) -> None: def restart_if_not_running(self) -> None: if not self.is_running(): logger.warning( - f"iaprof not running (unexpectedly), restarting... " + "iaprof not running (unexpectedly), restarting... " "kernels loaded before the restart may not be attributed" ) self.restart() From 9e0b1f7e5a5501c78422bccc6bf65ff9c01f98f2 Mon Sep 17 00:00:00 2001 From: Brandon Kammerdiener Date: Fri, 2 Oct 2026 10:09:23 -0400 Subject: [PATCH 07/10] iaprof: remove unused interval time constant, which is computed from frequency arg instead --- gprofiler/profilers/iaprof.py | 1 - 1 file changed, 1 deletion(-) diff --git a/gprofiler/profilers/iaprof.py b/gprofiler/profilers/iaprof.py index 988c69527..6d2f81731 100644 --- a/gprofiler/profilers/iaprof.py +++ b/gprofiler/profilers/iaprof.py @@ -28,7 +28,6 @@ from gprofiler.utils import run_process from gprofiler.utils.iaprof_process import IaprofProcess, IaprofProcessError -IAPROF_INTERVAL_MS = 10 IAPROF_SNAPSHOT_TIMEOUT = 5 From 2e65315374ab93490b41a87dd2561eb0b6ad4b49 Mon Sep 17 00:00:00 2001 From: Brandon Kammerdiener Date: Fri, 2 Oct 2026 10:35:21 -0400 Subject: [PATCH 08/10] iaprof: fix snapshot failure restart test --- tests/test_iaprof_process.py | 64 +++++++++++++++++++++++++---------- tests/test_iaprof_profiler.py | 14 ++++++++ 2 files changed, 60 insertions(+), 18 deletions(-) diff --git a/tests/test_iaprof_process.py b/tests/test_iaprof_process.py index 91088df42..5c0fc0dbe 100644 --- a/tests/test_iaprof_process.py +++ b/tests/test_iaprof_process.py @@ -17,7 +17,7 @@ import subprocess import sys from pathlib import Path -from time import monotonic +from time import monotonic, sleep from typing import Any, List import pytest @@ -129,29 +129,17 @@ def test_iaprof_process_kills_after_stop_timeout(tmp_path: Path, monkeypatch: Mo assert not process.is_running() -def test_iaprof_process_snapshot_timeout_stops_child(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None: +def test_iaprof_process_snapshot_timeout_kills_child(tmp_path: Path, monkeypatch: MonkeyPatch) -> None: _disable_pdeathsigger(monkeypatch) executable = tmp_path / "iaprof" _write_executable( executable, """ -import signal -import sys import time -running = True - - -def stop(*args): - global running - running = False - - -signal.signal(signal.SIGINT, stop) print("interval\\t0\\t1.0", flush=True) -while running: - time.sleep(0.01) -print("stopped", file=sys.stderr, flush=True) +while True: + time.sleep(1) """, ) @@ -164,8 +152,48 @@ def stop(*args): ) process.start() - with pytest.raises(IaprofProcessError, match="iaprof snapshot failed"): + with pytest.raises(IaprofProcessError, match="iaprof snapshot timed out"): process.snapshot(timeout=0.5) + deadline = monotonic() + 2 + while process.is_running() and monotonic() < deadline: + sleep(0.01) + assert not process.is_running() + process.stop() + + +def test_iaprof_process_restarts_after_exit(tmp_path: Path, monkeypatch: MonkeyPatch) -> None: + _disable_pdeathsigger(monkeypatch) + executable = tmp_path / "fake-iaprof" + marker = tmp_path / "started-once" + _write_executable( + executable, + f""" +import pathlib +import signal +import sys +import time + +marker = pathlib.Path({str(marker)!r}) +print("interval\\t0\\t1.0", flush=True) +if not marker.exists(): + marker.touch() + print("device lost", file=sys.stderr, flush=True) + sys.exit(3) +signal.signal(signal.SIGINT, lambda *args: sys.exit(0)) +while True: + time.sleep(0.01) + print("interval\\t0\\t1.0", flush=True) +""", + ) + process = IaprofProcess(str(executable), 10, 100, startup_timeout=2, stop_timeout=1) + process.start() + + with pytest.raises(IaprofProcessError, match="iaprof snapshot failed"): + process.snapshot(timeout=2) + + process.restart_if_not_running() + assert process.is_running() + assert process.snapshot(timeout=2) == {} + process.stop() assert not process.is_running() - assert "stopped" in process.stderr diff --git a/tests/test_iaprof_profiler.py b/tests/test_iaprof_profiler.py index 63b77337a..caad71dda 100644 --- a/tests/test_iaprof_profiler.py +++ b/tests/test_iaprof_profiler.py @@ -31,6 +31,7 @@ from gprofiler.profiler_state import ProfilerState from gprofiler.profilers.factory import get_profilers from gprofiler.profilers.iaprof import IAPROF_SNAPSHOT_TIMEOUT, IaprofProfiler, frequency_to_iaprof_interval +from gprofiler.utils.iaprof_process import IaprofProcessError def _make_profiler(profiler_state: ProfilerState) -> Tuple[IaprofProfiler, Mock]: @@ -133,6 +134,19 @@ def test_iaprof_profiler_snapshot_stops_with_gprofiler(profiler_state: ProfilerS process.snapshot.assert_not_called() +def test_iaprof_profiler_snapshot_failure_during_stop(profiler_state: ProfilerState) -> None: + profiler, process = _make_profiler(profiler_state) + + def interrupted_snapshot(timeout: float) -> None: + profiler_state.stop_event.set() + raise IaprofProcessError("iaprof snapshot failed") + + process.snapshot.side_effect = interrupted_snapshot + + with pytest.raises(StopEventSetException): + profiler.snapshot() + + def test_iaprof_profiler_selected_for_gpu_mode(profiler_state: ProfilerState) -> None: profiler_state.profiling_mode = "gpu" user_args: UserArgs = { From 9621df731fc37245723089e16503da22921ac65f Mon Sep 17 00:00:00 2001 From: Brandon Kammerdiener Date: Tue, 6 Oct 2026 12:26:40 -0400 Subject: [PATCH 09/10] rerun tests From 6dd30af0ac54d50025c3bcdf3596a7ed57e6db9a Mon Sep 17 00:00:00 2001 From: Brandon Kammerdiener Date: Fri, 16 Oct 2026 10:26:24 -0400 Subject: [PATCH 10/10] iaprof: wait for killed child after snapshot timeout so the next cycle restarts it --- gprofiler/utils/iaprof_process.py | 4 ++++ tests/test_iaprof_process.py | 5 +---- 2 files changed, 5 insertions(+), 4 deletions(-) diff --git a/gprofiler/utils/iaprof_process.py b/gprofiler/utils/iaprof_process.py index 6c03a1a0b..9328f7bf5 100644 --- a/gprofiler/utils/iaprof_process.py +++ b/gprofiler/utils/iaprof_process.py @@ -104,6 +104,10 @@ def snapshot(self, timeout: float) -> ProcessToStackSampleCounters: return self._reader.request_snapshot().result(timeout) except concurrent.futures.TimeoutError as error: process.kill() + try: + process.wait(timeout=self._stop_timeout) + except TimeoutExpired as wait_error: + raise IaprofProcessError("iaprof failed to stop after SIGKILL") from wait_error raise IaprofProcessError(self._error_message("iaprof snapshot timed out")) from error except Exception as error: raise IaprofProcessError(self._error_message("iaprof snapshot failed")) from error diff --git a/tests/test_iaprof_process.py b/tests/test_iaprof_process.py index 5c0fc0dbe..60f8a6633 100644 --- a/tests/test_iaprof_process.py +++ b/tests/test_iaprof_process.py @@ -17,7 +17,7 @@ import subprocess import sys from pathlib import Path -from time import monotonic, sleep +from time import monotonic from typing import Any, List import pytest @@ -155,9 +155,6 @@ def test_iaprof_process_snapshot_timeout_kills_child(tmp_path: Path, monkeypatch with pytest.raises(IaprofProcessError, match="iaprof snapshot timed out"): process.snapshot(timeout=0.5) - deadline = monotonic() + 2 - while process.is_running() and monotonic() < deadline: - sleep(0.01) assert not process.is_running() process.stop()