Add Intel GPU profiling mode via iaprof

gprofiler/consts.py

@@ -1,2 +1,5 @@
 CPU_PROFILING_MODE = "cpu"
+GPU_PROFILING_MODE = "gpu"
 DEFAULT_PROFILING_MODE = CPU_PROFILING_MODE
+DEFAULT_GPU_PROFILING_FREQUENCY = 100
+MAX_GPU_PROFILING_FREQUENCY = 1000

gprofiler/main.py

@@ -43,7 +43,12 @@
     DEFAULT_UPLOAD_TIMEOUT,
     ProfilerAPIClient,
 )
-from gprofiler.consts import CPU_PROFILING_MODE
+from gprofiler.consts import (
+    CPU_PROFILING_MODE,
+    DEFAULT_GPU_PROFILING_FREQUENCY,
+    GPU_PROFILING_MODE,
+    MAX_GPU_PROFILING_FREQUENCY,
+)
 from gprofiler.containers_client import ContainerNamesClient
 from gprofiler.diagnostics import log_diagnostics, set_diagnostics
 from gprofiler.dynamic_profiling_management.heartbeat import DynamicGProfilerManager, HeartbeatClient
@@ -62,7 +67,7 @@
 from gprofiler.profiler_state import ProfilerState
 from gprofiler.profilers.factory import get_profilers
 from gprofiler.profilers.perf import SystemProfiler
-from gprofiler.profilers.profiler_base import NoopProfiler, ProcessProfilerBase, ProfilerInterface
+from gprofiler.profilers.profiler_base import NoopProfiler, ProcessProfilerBase, ProfilerInterface, SystemProfilerBase
 from gprofiler.profilers.registry import get_profilers_registry
 from gprofiler.state import State, init_state
 from gprofiler.system_metrics import Metrics, NoopSystemMetricsMonitor, SystemMetricsMonitor, SystemMetricsMonitorBase
@@ -312,7 +317,10 @@ def start(self) -> None:
 
         # Check if system should skip continuous profilers due to process count
         skip_system_profilers = False
-        if self._profiler_state.max_system_processes_for_system_profilers > 0:
+        if (
+            isinstance(self.system_profiler, SystemProfiler)
+            and self._profiler_state.max_system_processes_for_system_profilers > 0
+        ):
             try:
                 total_processes = len(list(process_iter()))
                 if total_processes > self._profiler_state.max_system_processes_for_system_profilers:
@@ -333,18 +341,12 @@ def start(self) -> None:
         for prof in list(self.all_profilers):
             try:
                 # Skip system profilers if threshold exceeded
-                if (
-                    skip_system_profilers
-                    and hasattr(prof, "_is_system_wide_profiler")
-                    and prof._is_system_wide_profiler()
-                ):
+                if skip_system_profilers and prof is self.system_profiler:
                     logger.info(f"Skipping {prof.__class__.__name__} due to high system process count")
                     continue
 
                 prof.start()
             except Exception:
-                # the SystemProfiler is handled separately - let the user run with '--perf-mode none' if they
-                # wish so.
                 if prof is self.system_profiler:
                     raise
 
@@ -380,15 +382,12 @@ def _snapshot(self) -> None:
                 future_name = future.name  # type: ignore # hack, add the profiler's name to the Future object
                 logger.exception(f"{future_name} profiling failed")
 
-        local_end_time = local_start_time + datetime.timedelta(seconds=(time.monotonic() - monotonic_start_time))
-
         try:
             system_result = system_future.result()
         except Exception:
-            logger.critical(
-                "Running perf failed; consider running gProfiler with '--perf-mode disabled' to avoid using perf",
-            )
+            logger.critical(f"Running {self.system_profiler.name} failed")
             raise
+        local_end_time = local_start_time + datetime.timedelta(seconds=(time.monotonic() - monotonic_start_time))
         metadata = (
             get_current_metadata(cast(ProfileMetadata, self._static_metadata))
             if self._collect_metadata
@@ -431,6 +430,8 @@ def _snapshot(self) -> None:
                     "sampling_frequency": self.system_profiler._frequency,
                 }
             )
+        elif isinstance(self.system_profiler, SystemProfilerBase):
+            metadata.update(self.system_profiler.get_profile_metadata())
         else:
             # NoopProfiler - use default values
             metadata.update(
@@ -669,8 +670,10 @@ def parse_cmd_args() -> configargparse.Namespace:
         "--profiling-frequency",
         type=positive_integer,
         dest="frequency",
-        help=f"Profiler frequency in Hz (default: {DEFAULT_SAMPLING_FREQUENCY}), to be used only in CPU profiling "
-        f"(--mode=cpu, also the default mode)",
+        help=(
+            f"CPU sampling or GPU output frequency in Hz (defaults: CPU {DEFAULT_SAMPLING_FREQUENCY}, "
+            f"GPU {DEFAULT_GPU_PROFILING_FREQUENCY})"
+        ),
     )
     parser.add_argument(
         "-d",
@@ -710,10 +713,10 @@ def parse_cmd_args() -> configargparse.Namespace:
     parser.add_argument(
         "--mode",
         dest="profiling_mode",
-        choices=["cpu", "allocation", "none"],
-        default="cpu",
+        choices=[CPU_PROFILING_MODE, GPU_PROFILING_MODE, "allocation", "none"],
+        default=CPU_PROFILING_MODE,
         help="Select gProfiler's profiling mode, default is %(default)s, available options are "
-        "%(choices)s; allocation will profile only Java processes",
+        "%(choices)s; gpu will profile Intel GPUs and allocation will profile only Java processes",
     )
     parser.add_argument(
         "--alloc-interval",
@@ -1114,6 +1117,15 @@ def parse_cmd_args() -> configargparse.Namespace:
         if not args.alloc_interval:
             args.alloc_interval = DEFAULT_ALLOC_INTERVAL
         args.frequency = humanfriendly.parse_size(args.alloc_interval, binary=True)
+    elif args.profiling_mode == GPU_PROFILING_MODE:
+        if args.alloc_interval is not None:
+            parser.error("--alloc-interval is only allowed in allocation profiling (--mode=allocation)")
+        if args.rootless:
+            parser.error("--rootless is not supported in gpu profiling")
+        if args.frequency is None:
+            args.frequency = DEFAULT_GPU_PROFILING_FREQUENCY
+        elif args.frequency > MAX_GPU_PROFILING_FREQUENCY:
+            parser.error(f"-f|--frequency cannot exceed {MAX_GPU_PROFILING_FREQUENCY} in gpu profiling")
 
     if args.subcommand == UPLOAD_FILE_SUBCOMMAND:
         args.upload_results = True

gprofiler/profilers/__init__.py

@@ -4,6 +4,7 @@
 from gprofiler.profilers.python import PythonProfiler
 
 if is_linux():
+    from gprofiler.profilers.iaprof import IaprofProfiler
     from gprofiler.profilers.java import JavaProfiler
     from gprofiler.profilers.perf import SystemProfiler
     from gprofiler.profilers.php import PHPSpyProfiler
@@ -12,6 +13,6 @@
 __all__ = ["PythonProfiler", "DotnetProfiler"]
 
 if is_linux():
-    __all__ += ["JavaProfiler", "PHPSpyProfiler", "RbSpyProfiler", "SystemProfiler"]
+    __all__ += ["IaprofProfiler", "JavaProfiler", "PHPSpyProfiler", "RbSpyProfiler", "SystemProfiler"]
 
 del is_linux

gprofiler/profilers/factory.py

@@ -5,7 +5,7 @@
 from gprofiler.metadata.system_metadata import get_arch
 from gprofiler.platform import is_windows
 from gprofiler.profilers.perf import SystemProfiler
-from gprofiler.profilers.profiler_base import NoopProfiler
+from gprofiler.profilers.profiler_base import NoopProfiler, SystemProfilerBase
 from gprofiler.profilers.registry import get_profilers_registry
 from gprofiler.utils import is_profiler_disabled
 
@@ -20,10 +20,10 @@
 
 def get_profilers(
     user_args: "UserArgs", **profiler_init_kwargs: Any
-) -> Tuple[Union["SystemProfiler", "NoopProfiler"], List["ProcessProfilerBase"]]:
+) -> Tuple[Union["SystemProfilerBase", "NoopProfiler"], List["ProcessProfilerBase"]]:
     profiling_mode = user_args.get("profiling_mode")
     process_profilers_instances: List["ProcessProfilerBase"] = []
-    system_profiler: Union["SystemProfiler", "NoopProfiler"] = NoopProfiler()
+    system_profiler: Union["SystemProfilerBase", "NoopProfiler"] = NoopProfiler()
 
     # When custom event is specified, only use perf (SystemProfiler), disable all language profilers
     custom_event_mode = user_args.get("perf_event") is not None
@@ -44,19 +44,17 @@ def get_profilers(
                     logger.warning("--java-collect-thread-names is ignored because Java profiling is disabled")
                 continue
 
+            if profiling_mode not in profiler_config.supported_profiling_modes:
+                logger.debug(f"Disabling {profiler_name} because it doesn't support profiling mode {profiling_mode!r}")
+                continue
+
             supported_archs = (
                 profiler_config.supported_windows_archs if is_windows() else profiler_config.supported_archs
             )
             if arch not in supported_archs:
                 logger.warning(f"Disabling {profiler_name} because it doesn't support this architecture ({arch})")
                 continue
 
-            if profiling_mode not in profiler_config.supported_profiling_modes:
-                logger.warning(
-                    f"Disabling {profiler_name} because it doesn't support profiling mode {profiling_mode!r}"
-                )
-                continue
-
             profiler_kwargs = profiler_init_kwargs.copy()
             for key, value in user_args.items():
                 # Skip perf_event and perf_event_period as they're handled separately for custom events
@@ -81,7 +79,7 @@ def get_profilers(
                 )
                 sys.exit(1)
             else:
-                if isinstance(profiler_instance, SystemProfiler):
+                if isinstance(profiler_instance, SystemProfilerBase):
                     system_profiler = profiler_instance
                 else:
                     # In custom event mode, skip all process profilers

gprofiler/profilers/iaprof.py

@@ -0,0 +1,134 @@
+#
+# Copyright (C) 2026 Intel Corporation
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+from typing import Optional
+
+from packaging.version import Version
+
+from gprofiler.consts import GPU_PROFILING_MODE, MAX_GPU_PROFILING_FREQUENCY
+from gprofiler.exceptions import StopEventSetException
+from gprofiler.gprofiler_types import ProcessToProfileData, ProfileData, positive_integer
+from gprofiler.metadata import ProfileMetadata
+from gprofiler.profiler_state import ProfilerState
+from gprofiler.profilers.profiler_base import SystemProfilerBase
+from gprofiler.profilers.registry import ProfilerArgument, register_profiler
+from gprofiler.utils import run_process
+from gprofiler.utils.iaprof_process import IaprofProcess, IaprofProcessError
+
+IAPROF_SNAPSHOT_TIMEOUT = 5
+
+
+def frequency_to_iaprof_interval(frequency: int) -> int:
+    if not 1 <= frequency <= MAX_GPU_PROFILING_FREQUENCY:
+        raise ValueError(f"iaprof frequency must be between 1 and {MAX_GPU_PROFILING_FREQUENCY}")
+    return round(1000 / frequency)
+
+
+@register_profiler(
+    "Iaprof",
+    possible_modes=["enabled", "disabled"],
+    default_mode="enabled",
+    supported_archs=["x86_64"],  # not confirmed working on ARM, but should in theory
+    supported_profiling_modes=[GPU_PROFILING_MODE],
+    profiler_mode_argument_help="Enable or disable Intel GPU profiling with iaprof",
+    disablement_help="Disable Intel GPU profiling with iaprof",
+    profiler_arguments=[
+        ProfilerArgument(
+            "--iaprof-path",
+            dest="iaprof_path",
+            default="iaprof",
+            help="Path to the iaprof executable. Default: %(default)s",
+        ),
+        ProfilerArgument(
+            "--iaprof-eu-stall-subsample",
+            dest="iaprof_eu_stall_subsample",
+            type=positive_integer,
+            default=100,
+            help="Process one out of every N EU stall samples. Default: %(default)s",
+        ),
+    ],
+)
+class IaprofProfiler(SystemProfilerBase):
+    MINIMUM_SUPPORTED_VERSION = Version("2026.3")
+    _VERSION_TIMEOUT = 5
+
+    def __init__(
+        self,
+        frequency: int,
+        duration: int,
+        profiler_state: ProfilerState,
+        iaprof_mode: str,
+        iaprof_path: str,
+        iaprof_eu_stall_subsample: int,
+        min_duration: int = 0,
+    ) -> None:
+        assert iaprof_mode == "enabled"
+        super().__init__(frequency, duration, profiler_state, min_duration)
+        self._iaprof_path = iaprof_path
+        self._iaprof_version: Optional[str] = None
+        self._output_interval_ms = frequency_to_iaprof_interval(frequency)
+        self._eu_stall_subsample = iaprof_eu_stall_subsample
+        self._iaprof = IaprofProcess(
+            self._iaprof_path,
+            self._output_interval_ms,
+            self._eu_stall_subsample,
+        )
+
+    def get_profile_metadata(self) -> ProfileMetadata:
+        assert self._iaprof_version is not None
+        return {
+            "iaprof_version": self._iaprof_version,
+            "iaprof_output_interval_ms": self._output_interval_ms,
+            "iaprof_eu_stall_subsample": self._eu_stall_subsample,
+        }
+
+    def _get_iaprof_version(self) -> str:
+        result = run_process(
+            [self._iaprof_path, "--version"],
+            stop_event=self._profiler_state.stop_event,
+            timeout=self._VERSION_TIMEOUT,
+            pdeathsigger=False,
+        )
+        version_output = result.stdout.decode().strip() or result.stderr.decode().strip()
+        version = Version(version_output)
+        if version < self.MINIMUM_SUPPORTED_VERSION:
+            raise RuntimeError(f"iaprof {version} is unsupported; minimum version is {self.MINIMUM_SUPPORTED_VERSION}")
+        return str(version)
+
+    def start(self) -> None:
+        self._iaprof_version = self._get_iaprof_version()
+        self._iaprof.start()
+
+    def stop(self) -> None:
+        self._iaprof.stop()
+
+    def snapshot(self) -> ProcessToProfileData:
+        self._iaprof.restart_if_not_running()
+
+        if self._profiler_state.stop_event.wait(self._duration):
+            raise StopEventSetException
+
+        try:
+            stacks_by_pid = self._iaprof.snapshot(IAPROF_SNAPSHOT_TIMEOUT)
+        except IaprofProcessError:
+            if self._profiler_state.stop_event.is_set():
+                raise StopEventSetException
+            raise
+
+        return {
+            pid: ProfileData(stacks, None, None, self._profiler_state.get_container_name(pid))
+            for pid, stacks in stacks_by_pid.items()
+        }

gprofiler/profilers/perf.py

@@ -39,7 +39,7 @@
 from gprofiler.metadata.application_metadata import ApplicationMetadata
 from gprofiler.profiler_state import ProfilerState
 from gprofiler.profilers.node import clean_up_node_maps, generate_map_for_node_processes, get_node_processes
-from gprofiler.profilers.profiler_base import ProfilerBase
+from gprofiler.profilers.profiler_base import SystemProfilerBase
 from gprofiler.profilers.registry import ProfilerArgument, register_profiler
 from gprofiler.utils.perf import discover_appropriate_perf_event, parse_perf_script_from_iterator, valid_perf_pid
 from gprofiler.utils.perf_process import PerfProcess
@@ -154,18 +154,14 @@ def add_highest_avg_depth_stacks_per_process(
     " and instead only concatenate runtime-specific profilers results",
     supported_profiling_modes=["cpu"],
 )
-class SystemProfiler(ProfilerBase):
+class SystemProfiler(SystemProfilerBase):
     """
     We are running 2 perfs in parallel - one with DWARF and one with FP, and then we merge their results.
     This improves the results from software that is compiled without frame pointers,
     like some native software. DWARF by itself is not good enough, as it has issues with unwinding some
     versions of Go processes.
     """
 
-    def _is_system_wide_profiler(self) -> bool:
-        """Perf is a system-wide profiler that can be disabled on busy systems."""
-        return True
-
     def __init__(
         self,
         frequency: int,

gprofiler/profilers/profiler_base.py

@@ -34,6 +34,7 @@
 from gprofiler.exceptions import StopEventSetException
 from gprofiler.gprofiler_types import ProcessToProfileData, ProfileData, ProfilingErrorStack, StackToSampleCount
 from gprofiler.log import get_logger_adapter
+from gprofiler.metadata import ProfileMetadata
 from gprofiler.profiler_state import ProfilerState
 from gprofiler.utils import limit_frequency
 from gprofiler.utils.process import process_comm
@@ -118,6 +119,14 @@ def __init__(
         )
 
 
+class SystemProfilerBase(ProfilerBase):
+    def _is_system_wide_profiler(self) -> bool:
+        return True
+
+    def get_profile_metadata(self) -> ProfileMetadata:
+        return {}
+
+
 class NoopProfiler(ProfilerInterface):
     """
     No-op profiler - used as a drop-in replacement for runtime profilers, when they are disabled.