diff --git a/logfire/_internal/config.py b/logfire/_internal/config.py index 05c55488c..fc4acee55 100644 --- a/logfire/_internal/config.py +++ b/logfire/_internal/config.py @@ -109,6 +109,9 @@ from .integrations.executors import instrument_executors from .logs import ProxyLoggerProvider from .metrics import ProxyMeterProvider +from .profiling.exporter import PROFILES_PATH, ProfilesExporter +from .profiling.otlp import resource_from_attributes +from .profiling.supervisor import ProfilingSupervisor from .scrubbing import NOOP_SCRUBBER, BaseScrubber, Scrubber, ScrubbingOptions from .server_response import ServerResponseCallback, install_logfire_response_hook from .stack_info import warn_at_user_stacklevel @@ -438,6 +441,7 @@ def configure( variables: VariablesOptions | LocalVariablesOptions | None = None, distributed_tracing: bool | None = None, advanced: AdvancedOptions | None = None, + profiling: bool = False, **deprecated_kwargs: Unpack[DeprecatedKwargs], ) -> Logfire: """Configure the logfire SDK. @@ -513,6 +517,9 @@ def configure( for more information. This setting always applies globally, and the last value set is used, including the default value. advanced: Advanced options primarily used for testing by Logfire developers. + profiling: **Experimental.** Set to `True` to continuously profile this process with the + Python 3.15+ sampling profiler and send CPU profiles to Logfire. A no-op with a + warning on older Pythons or unsupported platforms. """ from .. import DEFAULT_LOGFIRE_INSTANCE, Logfire @@ -646,6 +653,7 @@ def configure( variables=variables, distributed_tracing=distributed_tracing, advanced=advanced, + profiling=profiling, ) if local: @@ -731,6 +739,9 @@ class _LogfireConfigData: advanced: AdvancedOptions """Advanced options primarily used for testing by Logfire developers.""" + profiling: bool + """Whether to continuously profile this process (experimental).""" + def _load_configuration( self, # note that there are no defaults here so that the only place @@ -756,6 +767,7 @@ def _load_configuration( variables: VariablesOptions | LocalVariablesOptions | None, distributed_tracing: bool | None, advanced: AdvancedOptions | None, + profiling: bool, ) -> None: """Merge the given parameters with the environment variables file configurations.""" self.param_manager = param_manager = ParamManager.create(config_dir) @@ -861,6 +873,8 @@ def _load_configuration( metrics = MetricsOptions() self.metrics = metrics + self.profiling = profiling + if self.service_version is None: try: self.service_version = get_git_revision_hash() @@ -893,6 +907,7 @@ def __init__( code_source: CodeSource | None = None, distributed_tracing: bool | None = None, advanced: AdvancedOptions | None = None, + profiling: bool = False, ) -> None: """Create a new LogfireConfig. @@ -923,6 +938,7 @@ def __init__( variables=variables, distributed_tracing=distributed_tracing, advanced=advanced, + profiling=profiling, ) # initialize with no-ops so that we don't impact OTEL's global config just because logfire is installed # that is, we defer setting logfire as the otel global config until `configure` is called @@ -935,6 +951,7 @@ def __init__( # This ensures that we only call OTEL's global set_tracer_provider once to avoid warnings. self._has_set_providers = False self._initialized = False + self._profiling_supervisor: ProfilingSupervisor | None = None self._lock = RLock() def configure( @@ -959,6 +976,7 @@ def configure( variables: VariablesOptions | LocalVariablesOptions | None, distributed_tracing: bool | None, advanced: AdvancedOptions | None, + profiling: bool, ) -> None: with self._lock: self._initialized = False @@ -983,6 +1001,7 @@ def configure( variables, distributed_tracing, advanced, + profiling, ) self.initialize() @@ -995,6 +1014,10 @@ def _initialize(self) -> None: if self._initialized: # pragma: no cover return + if self._profiling_supervisor is not None: # pragma: no cover + self._profiling_supervisor.shutdown() + self._profiling_supervisor = None + emscripten = platform_is_emscripten() with suppress_instrumentation(): @@ -1042,6 +1065,7 @@ def _initialize(self) -> None: # https://github.com/open-telemetry/semantic-conventions/blob/e44693245eef815071402b88c3a44a8f7f8f24c8/docs/resource/README.md#service-experimental # Both recommend generating a UUID. resource = Resource({'service.instance.id': uuid4().hex}).merge(resource) + profiles_exporter: ProfilesExporter | None = None head = self.sampling.head sampler: Sampler | None = None @@ -1176,6 +1200,8 @@ def check_tokens(): headers = {'User-Agent': f'logfire/{VERSION}', 'Authorization': token} session = OTLPExporterHttpSession() install_logfire_response_hook(session, self.advanced.server_response_hook) + if self.profiling and profiles_exporter is None: # pragma: no cover + profiles_exporter = ProfilesExporter(session, urljoin(base_url, PROFILES_PATH)) span_exporter = BodySizeCheckingOTLPSpanExporter( endpoint=urljoin(base_url, '/v1/traces'), session=session, @@ -1400,6 +1426,14 @@ def fix_pid(): # pragma: no cover self._ensure_flush_after_aws_lambda() + if profiles_exporter is not None: # pragma: no cover + self._profiling_supervisor = ProfilingSupervisor( + profiles_exporter, + resource=resource_from_attributes(resource.attributes), + scope_version=VERSION, + ) + self._profiling_supervisor.start() + def force_flush(self, timeout_millis: int = 30_000) -> bool: """Force flush all spans and metrics. diff --git a/logfire/_internal/main.py b/logfire/_internal/main.py index cc674c0fd..1f0bb1a2d 100644 --- a/logfire/_internal/main.py +++ b/logfire/_internal/main.py @@ -2491,6 +2491,10 @@ def remaining_ms() -> int: if not remaining: # pragma: no cover return False + profiling_supervisor = self.config._profiling_supervisor # pyright: ignore[reportPrivateUsage] + if profiling_supervisor is not None: + profiling_supervisor.shutdown(timeout=remaining_ms() / 1000.0) + if flush: # pragma: no branch self._tracer_provider.force_flush(remaining) remaining = remaining_ms() diff --git a/logfire/_internal/profiling/__init__.py b/logfire/_internal/profiling/__init__.py new file mode 100644 index 000000000..8b4e835f4 --- /dev/null +++ b/logfire/_internal/profiling/__init__.py @@ -0,0 +1,10 @@ +"""Experimental: ship Python sampling-profiler data to the OpenTelemetry profiles signal. + +This package converts the output of the Python 3.15 `profiling.sampling` +profiler (Tachyon) into OTLP profiles and exports them over HTTP. + +Status: work in progress. Implemented here are the conversion (`collapsed` + +`otlp`) and the HTTP `exporter`. Still to come: the supervisor that actually +runs the profiler subprocess, wiring into `logfire.configure()`, and per-span +correlation. +""" diff --git a/logfire/_internal/profiling/_proto/README.md b/logfire/_internal/profiling/_proto/README.md new file mode 100644 index 000000000..fb511398a --- /dev/null +++ b/logfire/_internal/profiling/_proto/README.md @@ -0,0 +1,48 @@ +# Vendored OTLP profiles protobuf bindings + +`profiles_pb2.py` / `profiles_service_pb2.py` (and their `.pyi` stubs) are +**generated** code, vendored here rather than imported from the +`opentelemetry-proto` PyPI package. + +## Why vendored + +The OpenTelemetry profiles signal is alpha (`v1development`). The +`opentelemetry-proto` PyPI package (≤ 1.41.x) is generated from the +`opentelemetry-proto` repo pinned at `v1.9.0`, which carries an **outdated** +`Sample` message: its field numbers (`values`, `attribute_indices`, +`link_index`) were renumbered in proto `v1.10.0`. A `Sample` serialized with +the stale numbering is silently misread by current consumers (e.g. Grafana +Pyroscope), so we cannot use the PyPI bindings. + +These files are generated from `opentelemetry-proto` **v1.10.0** instead. + +## When this directory can be deleted + +Once the `opentelemetry-proto` PyPI package ships bindings generated from +proto ≥ `v1.10.0`, delete this directory and import from +`opentelemetry.proto.profiles...` directly. The upstream pin bump is tracked +in . + +## Regeneration + +The proto `package` / file paths are renamed to `lf_otlp.*` so the descriptors +do not collide with the (stale) copy registered by the installed +`opentelemetry-proto` package — protobuf's descriptor pool rejects two files +with the same name. This rename is purely about the descriptor registry; it +does not affect the wire format (only field numbers do). + +To regenerate, fetch `profiles.proto` and `profiles_service.proto` from +`opentelemetry-proto` at the desired tag, place them under an `lf_otlp/` +directory with `opentelemetry.proto.{profiles.v1development,collector.profiles.v1development}` +rewritten to `lf_otlp.{profiles,collector}`, then: + +```sh +uvx --from grpcio-tools --with mypy-protobuf python -m grpc_tools.protoc \ + --proto_path= --python_out= --mypy_out= \ + lf_otlp/profiles.proto lf_otlp/profiles_service.proto +``` + +Finally rewrite `from lf_otlp import profiles_pb2` to +`from logfire._internal.profiling._proto import profiles_pb2` in the generated +`profiles_service_pb2.{py,pyi}`. `common`/`resource` come from the installed +`opentelemetry-proto` package (those signals are stable). diff --git a/logfire/_internal/profiling/_proto/__init__.py b/logfire/_internal/profiling/_proto/__init__.py new file mode 100644 index 000000000..41fe5ae2c --- /dev/null +++ b/logfire/_internal/profiling/_proto/__init__.py @@ -0,0 +1,11 @@ +"""Vendored OTLP profiles protobuf bindings. + +Generated from the canonical `open-telemetry/opentelemetry-proto` repo (tag +`v1.10.0`, the `v1development` profiles signal). Vendored because the +`opentelemetry-proto` PyPI package ships an outdated snapshot of this +still-alpha proto: its `Sample` field numbers predate a renumbering and are +wire-incompatible with current consumers (e.g. Grafana Pyroscope). + +See `README.md` in this directory for the regeneration recipe and the +condition under which this directory can be deleted. +""" diff --git a/logfire/_internal/profiling/_proto/profiles_pb2.py b/logfire/_internal/profiling/_proto/profiles_pb2.py new file mode 100644 index 000000000..ad2f6ede8 --- /dev/null +++ b/logfire/_internal/profiling/_proto/profiles_pb2.py @@ -0,0 +1,65 @@ +# -*- coding: utf-8 -*- +# Generated by the protocol buffer compiler. DO NOT EDIT! +# NO CHECKED-IN PROTOBUF GENCODE +# source: lf_otlp/profiles.proto +# Protobuf Python Version: 6.31.1 +"""Generated protocol buffer code.""" +from google.protobuf import descriptor as _descriptor +from google.protobuf import descriptor_pool as _descriptor_pool +from google.protobuf import runtime_version as _runtime_version +from google.protobuf import symbol_database as _symbol_database +from google.protobuf.internal import builder as _builder +_runtime_version.ValidateProtobufRuntimeVersion( + _runtime_version.Domain.PUBLIC, + 6, + 31, + 1, + '', + 'lf_otlp/profiles.proto' +) +# @@protoc_insertion_point(imports) + +_sym_db = _symbol_database.Default() + + +from opentelemetry.proto.common.v1 import common_pb2 as opentelemetry_dot_proto_dot_common_dot_v1_dot_common__pb2 +from opentelemetry.proto.resource.v1 import resource_pb2 as opentelemetry_dot_proto_dot_resource_dot_v1_dot_resource__pb2 + + +DESCRIPTOR = _descriptor_pool.Default().AddSerializedFile(b'\n\x16lf_otlp/profiles.proto\x12\x10lf_otlp.profiles\x1a*opentelemetry/proto/common/v1/common.proto\x1a.opentelemetry/proto/resource/v1/resource.proto\"\xda\x02\n\x12ProfilesDictionary\x12\x30\n\rmapping_table\x18\x01 \x03(\x0b\x32\x19.lf_otlp.profiles.Mapping\x12\x32\n\x0elocation_table\x18\x02 \x03(\x0b\x32\x1a.lf_otlp.profiles.Location\x12\x32\n\x0e\x66unction_table\x18\x03 \x03(\x0b\x32\x1a.lf_otlp.profiles.Function\x12*\n\nlink_table\x18\x04 \x03(\x0b\x32\x16.lf_otlp.profiles.Link\x12\x14\n\x0cstring_table\x18\x05 \x03(\t\x12:\n\x0f\x61ttribute_table\x18\x06 \x03(\x0b\x32!.lf_otlp.profiles.KeyValueAndUnit\x12,\n\x0bstack_table\x18\x07 \x03(\x0b\x32\x17.lf_otlp.profiles.Stack\"\x87\x01\n\x0cProfilesData\x12=\n\x11resource_profiles\x18\x01 \x03(\x0b\x32\".lf_otlp.profiles.ResourceProfiles\x12\x38\n\ndictionary\x18\x02 \x01(\x0b\x32$.lf_otlp.profiles.ProfilesDictionary\"\xa4\x01\n\x10ResourceProfiles\x12;\n\x08resource\x18\x01 \x01(\x0b\x32).opentelemetry.proto.resource.v1.Resource\x12\x37\n\x0escope_profiles\x18\x02 \x03(\x0b\x32\x1f.lf_otlp.profiles.ScopeProfiles\x12\x12\n\nschema_url\x18\x03 \x01(\tJ\x06\x08\xe8\x07\x10\xe9\x07\"\x94\x01\n\rScopeProfiles\x12\x42\n\x05scope\x18\x01 \x01(\x0b\x32\x33.opentelemetry.proto.common.v1.InstrumentationScope\x12+\n\x08profiles\x18\x02 \x03(\x0b\x32\x19.lf_otlp.profiles.Profile\x12\x12\n\nschema_url\x18\x03 \x01(\t\"\xe3\x02\n\x07Profile\x12\x30\n\x0bsample_type\x18\x01 \x01(\x0b\x32\x1b.lf_otlp.profiles.ValueType\x12)\n\x07samples\x18\x02 \x03(\x0b\x32\x18.lf_otlp.profiles.Sample\x12\x16\n\x0etime_unix_nano\x18\x03 \x01(\x06\x12\x15\n\rduration_nano\x18\x04 \x01(\x04\x12\x30\n\x0bperiod_type\x18\x05 \x01(\x0b\x32\x1b.lf_otlp.profiles.ValueType\x12\x0e\n\x06period\x18\x06 \x01(\x03\x12\x12\n\nprofile_id\x18\x07 \x01(\x0c\x12 \n\x18\x64ropped_attributes_count\x18\x08 \x01(\r\x12\x1f\n\x17original_payload_format\x18\t \x01(\t\x12\x18\n\x10original_payload\x18\n \x01(\x0c\x12\x19\n\x11\x61ttribute_indices\x18\x0b \x03(\x05\")\n\x04Link\x12\x10\n\x08trace_id\x18\x01 \x01(\x0c\x12\x0f\n\x07span_id\x18\x02 \x01(\x0c\"9\n\tValueType\x12\x15\n\rtype_strindex\x18\x01 \x01(\x05\x12\x15\n\runit_strindex\x18\x02 \x01(\x05\"z\n\x06Sample\x12\x13\n\x0bstack_index\x18\x01 \x01(\x05\x12\x19\n\x11\x61ttribute_indices\x18\x02 \x03(\x05\x12\x12\n\nlink_index\x18\x03 \x01(\x05\x12\x0e\n\x06values\x18\x04 \x03(\x03\x12\x1c\n\x14timestamps_unix_nano\x18\x05 \x03(\x06\"\x80\x01\n\x07Mapping\x12\x14\n\x0cmemory_start\x18\x01 \x01(\x04\x12\x14\n\x0cmemory_limit\x18\x02 \x01(\x04\x12\x13\n\x0b\x66ile_offset\x18\x03 \x01(\x04\x12\x19\n\x11\x66ilename_strindex\x18\x04 \x01(\x05\x12\x19\n\x11\x61ttribute_indices\x18\x05 \x03(\x05\"!\n\x05Stack\x12\x18\n\x10location_indices\x18\x01 \x03(\x05\"t\n\x08Location\x12\x15\n\rmapping_index\x18\x01 \x01(\x05\x12\x0f\n\x07\x61\x64\x64ress\x18\x02 \x01(\x04\x12%\n\x05lines\x18\x03 \x03(\x0b\x32\x16.lf_otlp.profiles.Line\x12\x19\n\x11\x61ttribute_indices\x18\x04 \x03(\x05\"<\n\x04Line\x12\x16\n\x0e\x66unction_index\x18\x01 \x01(\x05\x12\x0c\n\x04line\x18\x02 \x01(\x03\x12\x0e\n\x06\x63olumn\x18\x03 \x01(\x03\"n\n\x08\x46unction\x12\x15\n\rname_strindex\x18\x01 \x01(\x05\x12\x1c\n\x14system_name_strindex\x18\x02 \x01(\x05\x12\x19\n\x11\x66ilename_strindex\x18\x03 \x01(\x05\x12\x12\n\nstart_line\x18\x04 \x01(\x03\"v\n\x0fKeyValueAndUnit\x12\x14\n\x0ckey_strindex\x18\x01 \x01(\x05\x12\x36\n\x05value\x18\x02 \x01(\x0b\x32\'.opentelemetry.proto.common.v1.AnyValue\x12\x15\n\runit_strindex\x18\x03 \x01(\x05\x42\x8a\x01\n\x13io.lf_otlp.profilesB\rProfilesProtoP\x01Z5go.opentelemetry.io/proto/otlp/profiles/v1development\xaa\x02*OpenTelemetry.Proto.Profiles.V1Developmentb\x06proto3') + +_globals = globals() +_builder.BuildMessageAndEnumDescriptors(DESCRIPTOR, _globals) +_builder.BuildTopDescriptorsAndMessages(DESCRIPTOR, 'lf_otlp.profiles_pb2', _globals) +if not _descriptor._USE_C_DESCRIPTORS: + _globals['DESCRIPTOR']._loaded_options = None + _globals['DESCRIPTOR']._serialized_options = b'\n\023io.lf_otlp.profilesB\rProfilesProtoP\001Z5go.opentelemetry.io/proto/otlp/profiles/v1development\252\002*OpenTelemetry.Proto.Profiles.V1Development' + _globals['_PROFILESDICTIONARY']._serialized_start=137 + _globals['_PROFILESDICTIONARY']._serialized_end=483 + _globals['_PROFILESDATA']._serialized_start=486 + _globals['_PROFILESDATA']._serialized_end=621 + _globals['_RESOURCEPROFILES']._serialized_start=624 + _globals['_RESOURCEPROFILES']._serialized_end=788 + _globals['_SCOPEPROFILES']._serialized_start=791 + _globals['_SCOPEPROFILES']._serialized_end=939 + _globals['_PROFILE']._serialized_start=942 + _globals['_PROFILE']._serialized_end=1297 + _globals['_LINK']._serialized_start=1299 + _globals['_LINK']._serialized_end=1340 + _globals['_VALUETYPE']._serialized_start=1342 + _globals['_VALUETYPE']._serialized_end=1399 + _globals['_SAMPLE']._serialized_start=1401 + _globals['_SAMPLE']._serialized_end=1523 + _globals['_MAPPING']._serialized_start=1526 + _globals['_MAPPING']._serialized_end=1654 + _globals['_STACK']._serialized_start=1656 + _globals['_STACK']._serialized_end=1689 + _globals['_LOCATION']._serialized_start=1691 + _globals['_LOCATION']._serialized_end=1807 + _globals['_LINE']._serialized_start=1809 + _globals['_LINE']._serialized_end=1869 + _globals['_FUNCTION']._serialized_start=1871 + _globals['_FUNCTION']._serialized_end=1981 + _globals['_KEYVALUEANDUNIT']._serialized_start=1983 + _globals['_KEYVALUEANDUNIT']._serialized_end=2101 +# @@protoc_insertion_point(module_scope) diff --git a/logfire/_internal/profiling/_proto/profiles_pb2.pyi b/logfire/_internal/profiling/_proto/profiles_pb2.pyi new file mode 100644 index 000000000..38951194d --- /dev/null +++ b/logfire/_internal/profiling/_proto/profiles_pb2.pyi @@ -0,0 +1,912 @@ +""" +@generated by mypy-protobuf. Do not edit manually! +isort:skip_file +Copyright 2023, OpenTelemetry Authors + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. + +This file includes work covered by the following copyright and permission notices: + +Copyright 2016 Google Inc. All Rights Reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +""" + +from collections import abc as _abc +from google.protobuf import descriptor as _descriptor +from google.protobuf import message as _message +from google.protobuf.internal import containers as _containers +from opentelemetry.proto.common.v1 import common_pb2 as _common_pb2 +from opentelemetry.proto.resource.v1 import resource_pb2 as _resource_pb2 +import builtins as _builtins +import sys +import typing as _typing + +if sys.version_info >= (3, 11): + from typing import TypeAlias as _TypeAlias, Never as _Never +else: + from typing_extensions import TypeAlias as _TypeAlias, Never as _Never + +DESCRIPTOR: _descriptor.FileDescriptor + +@_typing.final +class ProfilesDictionary(_message.Message): + """ Relationships Diagram + + ┌──────────────────┐ LEGEND + │ ProfilesData │ ─────┐ + └──────────────────┘ │ ─────▶ embedded + │ │ + │ 1-n │ ─────▷ referenced by index + ▼ ▼ + ┌──────────────────┐ ┌────────────────────┐ + │ ResourceProfiles │ │ ProfilesDictionary │ + └──────────────────┘ └────────────────────┘ + │ + │ 1-n + ▼ + ┌──────────────────┐ + │ ScopeProfiles │ + └──────────────────┘ + │ + │ 1-n + ▼ + ┌──────────────────┐ + │ Profile │ + └──────────────────┘ + │ n-1 + │ 1-n ┌───────────────────────────────────────┐ + ▼ │ ▽ + ┌──────────────────┐ 1-n ┌─────────────────┐ ┌──────────┐ + │ Sample │ ──────▷ │ KeyValueAndUnit │ │ Link │ + └──────────────────┘ └─────────────────┘ └──────────┘ + │ △ △ + │ n-1 │ │ 1-n + ▽ │ │ + ┌──────────────────┐ │ │ + │ Stack │ │ │ + └──────────────────┘ │ │ + │ 1-n │ │ + │ 1-n ┌────────────────┘ │ + ▽ │ │ + ┌──────────────────┐ n-1 ┌─────────────┐ + │ Location │ ──────▷ │ Mapping │ + └──────────────────┘ └─────────────┘ + │ + │ 1-n + ▼ + ┌──────────────────┐ + │ Line │ + └──────────────────┘ + │ + │ 1-1 + ▽ + ┌──────────────────┐ + │ Function │ + └──────────────────┘ + + ProfilesDictionary represents the profiles data shared across the + entire message being sent. The following applies to all fields in this + message: + + - A dictionary is an array of dictionary items. Users of the dictionary + compactly reference the items using the index within the array. + + - A dictionary MUST have a zero value encoded as the first element. This + allows for _index fields pointing into the dictionary to use a 0 pointer + value to indicate 'null' / 'not set'. Unless otherwise defined, a 'zero + value' message value is one with all default field values, so as to + minimize wire encoded size. + + - There SHOULD NOT be dupes in a dictionary. The identity of dictionary + items is based on their value, recursively as needed. If a particular + implementation does emit duplicated items, it MUST NOT attempt to give them + meaning based on the index or order. A profile processor may remove + duplicate items and this MUST NOT have any observable effects for + consumers. + + - There SHOULD NOT be orphaned (unreferenced) items in a dictionary. A + profile processor may remove ("garbage-collect") orphaned items and this + MUST NOT have any observable effects for consumers. + + Status: [Alpha] + """ + + DESCRIPTOR: _descriptor.Descriptor + + MAPPING_TABLE_FIELD_NUMBER: _builtins.int + LOCATION_TABLE_FIELD_NUMBER: _builtins.int + FUNCTION_TABLE_FIELD_NUMBER: _builtins.int + LINK_TABLE_FIELD_NUMBER: _builtins.int + STRING_TABLE_FIELD_NUMBER: _builtins.int + ATTRIBUTE_TABLE_FIELD_NUMBER: _builtins.int + STACK_TABLE_FIELD_NUMBER: _builtins.int + @_builtins.property + def mapping_table(self) -> _containers.RepeatedCompositeFieldContainer[Global___Mapping]: + """Mappings from address ranges to the image/binary/library mapped + into that address range referenced by locations via Location.mapping_index. + + mapping_table[0] must always be zero value (Mapping{}) and present. + """ + + @_builtins.property + def location_table(self) -> _containers.RepeatedCompositeFieldContainer[Global___Location]: + """Locations referenced by samples via Stack.location_indices. + + location_table[0] must always be zero value (Location{}) and present. + """ + + @_builtins.property + def function_table(self) -> _containers.RepeatedCompositeFieldContainer[Global___Function]: + """Functions referenced by locations via Line.function_index. + + function_table[0] must always be zero value (Function{}) and present. + """ + + @_builtins.property + def link_table(self) -> _containers.RepeatedCompositeFieldContainer[Global___Link]: + """Links referenced by samples via Sample.link_index. + + link_table[0] must always be zero value (Link{}) and present. + """ + + @_builtins.property + def string_table(self) -> _containers.RepeatedScalarFieldContainer[_builtins.str]: + """A common table for strings referenced by various messages. + + string_table[0] must always be "" and present. + """ + + @_builtins.property + def attribute_table(self) -> _containers.RepeatedCompositeFieldContainer[Global___KeyValueAndUnit]: + """A common table for attributes referenced by the Profile, Sample, Mapping + and Location messages below through attribute_indices field. Each entry is + a key/value pair with an optional unit. Since this is a dictionary table, + multiple entries with the same key may be present, unlike direct attribute + tables like Resource.attributes. The referencing attribute_indices fields, + though, do maintain the key uniqueness requirement. + + It's recommended to use attributes for variables with bounded cardinality, + such as categorical variables + (https://en.wikipedia.org/wiki/Categorical_variable). Using an attribute of + a floating point type (e.g., CPU time) in a sample can quickly make every + attribute value unique, defeating the purpose of the dictionary and + impractically increasing the profile size. + + Examples of attributes: + "/http/user_agent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_14_2) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/71.0.3578.98 Safari/537.36" + "abc.com/myattribute": true + "allocation_size": 128 bytes + + attribute_table[0] must always be zero value (KeyValueAndUnit{}) and present. + """ + + @_builtins.property + def stack_table(self) -> _containers.RepeatedCompositeFieldContainer[Global___Stack]: + """Stacks referenced by samples via Sample.stack_index. + + stack_table[0] must always be zero value (Stack{}) and present. + """ + + def __init__( + self, + *, + mapping_table: _abc.Iterable[Global___Mapping] | None = ..., + location_table: _abc.Iterable[Global___Location] | None = ..., + function_table: _abc.Iterable[Global___Function] | None = ..., + link_table: _abc.Iterable[Global___Link] | None = ..., + string_table: _abc.Iterable[_builtins.str] | None = ..., + attribute_table: _abc.Iterable[Global___KeyValueAndUnit] | None = ..., + stack_table: _abc.Iterable[Global___Stack] | None = ..., + ) -> None: ... + _HasFieldArgType: _TypeAlias = _Never # noqa: Y015 + def HasField(self, field_name: _HasFieldArgType) -> _builtins.bool: ... + _ClearFieldArgType: _TypeAlias = _typing.Literal["attribute_table", b"attribute_table", "function_table", b"function_table", "link_table", b"link_table", "location_table", b"location_table", "mapping_table", b"mapping_table", "stack_table", b"stack_table", "string_table", b"string_table"] # noqa: Y015 + def ClearField(self, field_name: _ClearFieldArgType) -> None: ... + def WhichOneof(self, oneof_group: _Never) -> None: ... + +Global___ProfilesDictionary: _TypeAlias = ProfilesDictionary # noqa: Y015 + +@_typing.final +class ProfilesData(_message.Message): + """ProfilesData represents the profiles data that can be stored in persistent storage, + OR can be embedded by other protocols that transfer OTLP profiles data but do not + implement the OTLP protocol. + + The main difference between this message and collector protocol is that + in this message there will not be any "control" or "metadata" specific to + OTLP protocol. + + When new fields are added into this message, the OTLP request MUST be updated + as well. + + Status: [Alpha] + """ + + DESCRIPTOR: _descriptor.Descriptor + + RESOURCE_PROFILES_FIELD_NUMBER: _builtins.int + DICTIONARY_FIELD_NUMBER: _builtins.int + @_builtins.property + def resource_profiles(self) -> _containers.RepeatedCompositeFieldContainer[Global___ResourceProfiles]: + """An array of ResourceProfiles. + For data coming from an SDK profiler, this array will typically contain one + element. Host-level profilers will usually create one ResourceProfile per + container, as well as one additional ResourceProfile grouping all samples + from non-containerized processes. + Other resource groupings are possible as well and clarified via + Resource.attributes and semantic conventions. + Tools that visualize profiles should prefer displaying + resources_profiles[0].scope_profiles[0].profiles[0] by default. + """ + + @_builtins.property + def dictionary(self) -> Global___ProfilesDictionary: + """One instance of ProfilesDictionary""" + + def __init__( + self, + *, + resource_profiles: _abc.Iterable[Global___ResourceProfiles] | None = ..., + dictionary: Global___ProfilesDictionary | None = ..., + ) -> None: ... + _HasFieldArgType: _TypeAlias = _typing.Literal["dictionary", b"dictionary"] # noqa: Y015 + def HasField(self, field_name: _HasFieldArgType) -> _builtins.bool: ... + _ClearFieldArgType: _TypeAlias = _typing.Literal["dictionary", b"dictionary", "resource_profiles", b"resource_profiles"] # noqa: Y015 + def ClearField(self, field_name: _ClearFieldArgType) -> None: ... + def WhichOneof(self, oneof_group: _Never) -> None: ... + +Global___ProfilesData: _TypeAlias = ProfilesData # noqa: Y015 + +@_typing.final +class ResourceProfiles(_message.Message): + """A collection of ScopeProfiles from a Resource. + + Status: [Alpha] + """ + + DESCRIPTOR: _descriptor.Descriptor + + RESOURCE_FIELD_NUMBER: _builtins.int + SCOPE_PROFILES_FIELD_NUMBER: _builtins.int + SCHEMA_URL_FIELD_NUMBER: _builtins.int + schema_url: _builtins.str + """The Schema URL, if known. This is the identifier of the Schema that the resource data + is recorded in. Notably, the last part of the URL path is the version number of the + schema: http[s]://server[:port]/path/. To learn more about Schema URL see + https://opentelemetry.io/docs/specs/otel/schemas/#schema-url + This schema_url applies to the data in the "resource" field. It does not apply + to the data in the "scope_profiles" field which have their own schema_url field. + """ + @_builtins.property + def resource(self) -> _resource_pb2.Resource: + """The resource for the profiles in this message. + If this field is not set then no resource info is known. + """ + + @_builtins.property + def scope_profiles(self) -> _containers.RepeatedCompositeFieldContainer[Global___ScopeProfiles]: + """A list of ScopeProfiles that originate from a resource.""" + + def __init__( + self, + *, + resource: _resource_pb2.Resource | None = ..., + scope_profiles: _abc.Iterable[Global___ScopeProfiles] | None = ..., + schema_url: _builtins.str = ..., + ) -> None: ... + _HasFieldArgType: _TypeAlias = _typing.Literal["resource", b"resource"] # noqa: Y015 + def HasField(self, field_name: _HasFieldArgType) -> _builtins.bool: ... + _ClearFieldArgType: _TypeAlias = _typing.Literal["resource", b"resource", "schema_url", b"schema_url", "scope_profiles", b"scope_profiles"] # noqa: Y015 + def ClearField(self, field_name: _ClearFieldArgType) -> None: ... + def WhichOneof(self, oneof_group: _Never) -> None: ... + +Global___ResourceProfiles: _TypeAlias = ResourceProfiles # noqa: Y015 + +@_typing.final +class ScopeProfiles(_message.Message): + """A collection of Profiles produced by an InstrumentationScope. + + Status: [Alpha] + """ + + DESCRIPTOR: _descriptor.Descriptor + + SCOPE_FIELD_NUMBER: _builtins.int + PROFILES_FIELD_NUMBER: _builtins.int + SCHEMA_URL_FIELD_NUMBER: _builtins.int + schema_url: _builtins.str + """The Schema URL, if known. This is the identifier of the Schema that the profile data + is recorded in. Notably, the last part of the URL path is the version number of the + schema: http[s]://server[:port]/path/. To learn more about Schema URL see + https://opentelemetry.io/docs/specs/otel/schemas/#schema-url + This schema_url applies to the data in the "scope" field and all profiles in the + "profiles" field. + """ + @_builtins.property + def scope(self) -> _common_pb2.InstrumentationScope: + """The instrumentation scope information for the profiles in this message. + Semantically when InstrumentationScope isn't set, it is equivalent with + an empty instrumentation scope name (unknown). + """ + + @_builtins.property + def profiles(self) -> _containers.RepeatedCompositeFieldContainer[Global___Profile]: + """A list of Profiles that originate from an instrumentation scope.""" + + def __init__( + self, + *, + scope: _common_pb2.InstrumentationScope | None = ..., + profiles: _abc.Iterable[Global___Profile] | None = ..., + schema_url: _builtins.str = ..., + ) -> None: ... + _HasFieldArgType: _TypeAlias = _typing.Literal["scope", b"scope"] # noqa: Y015 + def HasField(self, field_name: _HasFieldArgType) -> _builtins.bool: ... + _ClearFieldArgType: _TypeAlias = _typing.Literal["profiles", b"profiles", "schema_url", b"schema_url", "scope", b"scope"] # noqa: Y015 + def ClearField(self, field_name: _ClearFieldArgType) -> None: ... + def WhichOneof(self, oneof_group: _Never) -> None: ... + +Global___ScopeProfiles: _TypeAlias = ScopeProfiles # noqa: Y015 + +@_typing.final +class Profile(_message.Message): + """Profile is a common stacktrace profile format. + + Measurements represented with this format should follow the + following conventions: + + - Consumers should treat unset optional fields as if they had been + set with their default value. + + - When possible, measurements should be stored in "unsampled" form + that is most useful to humans. There should be enough + information present to determine the original sampled values. + + - The profile is represented as a set of samples, where each sample + references a stack trace which is a list of locations, each belonging + to a mapping. + - There is a N->1 relationship from Stack.location_indices entries to + locations. For every Stack.location_indices entry there must be a + unique Location with that index. + - There is an optional N->1 relationship from locations to + mappings. For every nonzero Location.mapping_id there must be a + unique Mapping with that index. + + Represents a complete profile, including sample types, samples, mappings to + binaries, stacks, locations, functions, string table, and additional + metadata. It modifies and annotates pprof Profile with OpenTelemetry + specific fields. + + Note that whilst fields in this message retain the name and field id from pprof in most cases + for ease of understanding data migration, it is not intended that pprof:Profile and + OpenTelemetry:Profile encoding be wire compatible. + + Status: [Alpha] + """ + + DESCRIPTOR: _descriptor.Descriptor + + SAMPLE_TYPE_FIELD_NUMBER: _builtins.int + SAMPLES_FIELD_NUMBER: _builtins.int + TIME_UNIX_NANO_FIELD_NUMBER: _builtins.int + DURATION_NANO_FIELD_NUMBER: _builtins.int + PERIOD_TYPE_FIELD_NUMBER: _builtins.int + PERIOD_FIELD_NUMBER: _builtins.int + PROFILE_ID_FIELD_NUMBER: _builtins.int + DROPPED_ATTRIBUTES_COUNT_FIELD_NUMBER: _builtins.int + ORIGINAL_PAYLOAD_FORMAT_FIELD_NUMBER: _builtins.int + ORIGINAL_PAYLOAD_FIELD_NUMBER: _builtins.int + ATTRIBUTE_INDICES_FIELD_NUMBER: _builtins.int + time_unix_nano: _builtins.int + """The following fields 3-12 are informational, do not affect + interpretation of results. + + Time of collection. Value is UNIX Epoch time in nanoseconds since 00:00:00 + UTC on 1 January 1970. + """ + duration_nano: _builtins.int + """Duration of the profile. For instant profiles like live heap snapshot, the + duration can be zero but it may be preferable to set time_unix_nano to the + process start time and duration_nano to the relative time when the profile + was gathered. This ensures Sample.timestamps_unix_nano values such as + allocation timestamp fall into the profile time range. + """ + period: _builtins.int + """The number of events between sampled occurrences.""" + profile_id: _builtins.bytes + """A globally unique identifier for a profile. The ID is a 16-byte array. An ID with + all zeroes is considered invalid. It may be used for deduplication and signal + correlation purposes. It is acceptable to treat two profiles with different values + in this field as not equal, even if they represented the same object at an earlier + time. + This field is optional; an ID may be assigned to an ID-less profile in a later step. + """ + dropped_attributes_count: _builtins.int + """The number of attributes that were discarded. Attributes + can be discarded because their keys are too long or because there are too many + attributes. If this value is 0, then no attributes were dropped. + """ + original_payload_format: _builtins.str + """The original payload format. See also original_payload. Optional, but the + format and the bytes must be set or unset together. + + The allowed values for the format string are defined by the OpenTelemetry + specification. Some examples are "jfr", "pprof", "linux_perf". + + The original payload may be optionally provided when the conversion to the + OLTP format was done from a different format with some loss of the fidelity + and the receiver may want to store the original payload to allow future + lossless export or reinterpretation. Some examples of the original format + are JFR (Java Flight Recorder), pprof, Linux perf. + + Even when the original payload is in a format that is semantically close to + OTLP, such as pprof, a conversion may still be lossy in some cases (e.g. if + the pprof file contains custom extensions or conventions). + + The original payload can be large in size, so including the original + payload should be configurable by the profiler or collector options. The + default behavior should be to not include the original payload. + """ + original_payload: _builtins.bytes + """The original payload bytes. See also original_payload_format. Optional, but + format and the bytes must be set or unset together. + """ + @_builtins.property + def sample_type(self) -> Global___ValueType: + """The type and unit of all Sample.values in this profile. + For a cpu or off-cpu profile this might be: + ["cpu","nanoseconds"] or ["off_cpu","nanoseconds"] + For a heap profile, this might be: + ["allocated_objects","count"] or ["allocated_space","bytes"], + """ + + @_builtins.property + def samples(self) -> _containers.RepeatedCompositeFieldContainer[Global___Sample]: + """The set of samples recorded in this profile.""" + + @_builtins.property + def period_type(self) -> Global___ValueType: + """The kind of events between sampled occurrences. + e.g [ "cpu","cycles" ] or [ "heap","bytes" ] + """ + + @_builtins.property + def attribute_indices(self) -> _containers.RepeatedScalarFieldContainer[_builtins.int]: + """References to attributes in attribute_table. [optional]""" + + def __init__( + self, + *, + sample_type: Global___ValueType | None = ..., + samples: _abc.Iterable[Global___Sample] | None = ..., + time_unix_nano: _builtins.int = ..., + duration_nano: _builtins.int = ..., + period_type: Global___ValueType | None = ..., + period: _builtins.int = ..., + profile_id: _builtins.bytes = ..., + dropped_attributes_count: _builtins.int = ..., + original_payload_format: _builtins.str = ..., + original_payload: _builtins.bytes = ..., + attribute_indices: _abc.Iterable[_builtins.int] | None = ..., + ) -> None: ... + _HasFieldArgType: _TypeAlias = _typing.Literal["period_type", b"period_type", "sample_type", b"sample_type"] # noqa: Y015 + def HasField(self, field_name: _HasFieldArgType) -> _builtins.bool: ... + _ClearFieldArgType: _TypeAlias = _typing.Literal["attribute_indices", b"attribute_indices", "dropped_attributes_count", b"dropped_attributes_count", "duration_nano", b"duration_nano", "original_payload", b"original_payload", "original_payload_format", b"original_payload_format", "period", b"period", "period_type", b"period_type", "profile_id", b"profile_id", "sample_type", b"sample_type", "samples", b"samples", "time_unix_nano", b"time_unix_nano"] # noqa: Y015 + def ClearField(self, field_name: _ClearFieldArgType) -> None: ... + def WhichOneof(self, oneof_group: _Never) -> None: ... + +Global___Profile: _TypeAlias = Profile # noqa: Y015 + +@_typing.final +class Link(_message.Message): + """A pointer from a profile Sample to a trace Span. + Connects a profile sample to a trace span, identified by unique trace and span IDs. + + Status: [Alpha] + """ + + DESCRIPTOR: _descriptor.Descriptor + + TRACE_ID_FIELD_NUMBER: _builtins.int + SPAN_ID_FIELD_NUMBER: _builtins.int + trace_id: _builtins.bytes + """A unique identifier of a trace that this linked span is part of. The ID is a + 16-byte array. + """ + span_id: _builtins.bytes + """A unique identifier for the linked span. The ID is an 8-byte array.""" + def __init__( + self, + *, + trace_id: _builtins.bytes = ..., + span_id: _builtins.bytes = ..., + ) -> None: ... + _HasFieldArgType: _TypeAlias = _Never # noqa: Y015 + def HasField(self, field_name: _HasFieldArgType) -> _builtins.bool: ... + _ClearFieldArgType: _TypeAlias = _typing.Literal["span_id", b"span_id", "trace_id", b"trace_id"] # noqa: Y015 + def ClearField(self, field_name: _ClearFieldArgType) -> None: ... + def WhichOneof(self, oneof_group: _Never) -> None: ... + +Global___Link: _TypeAlias = Link # noqa: Y015 + +@_typing.final +class ValueType(_message.Message): + """ValueType describes the type and units of a value. + + Status: [Alpha] + """ + + DESCRIPTOR: _descriptor.Descriptor + + TYPE_STRINDEX_FIELD_NUMBER: _builtins.int + UNIT_STRINDEX_FIELD_NUMBER: _builtins.int + type_strindex: _builtins.int + """Index into ProfilesDictionary.string_table.""" + unit_strindex: _builtins.int + """Index into ProfilesDictionary.string_table.""" + def __init__( + self, + *, + type_strindex: _builtins.int = ..., + unit_strindex: _builtins.int = ..., + ) -> None: ... + _HasFieldArgType: _TypeAlias = _Never # noqa: Y015 + def HasField(self, field_name: _HasFieldArgType) -> _builtins.bool: ... + _ClearFieldArgType: _TypeAlias = _typing.Literal["type_strindex", b"type_strindex", "unit_strindex", b"unit_strindex"] # noqa: Y015 + def ClearField(self, field_name: _ClearFieldArgType) -> None: ... + def WhichOneof(self, oneof_group: _Never) -> None: ... + +Global___ValueType: _TypeAlias = ValueType # noqa: Y015 + +@_typing.final +class Sample(_message.Message): + """Each Sample records values encountered in some program context. The program + context is typically a stack trace, perhaps augmented with auxiliary + information like the thread-id, some indicator of a higher level request + being handled etc. + + A Sample MUST have have at least one values or timestamps_unix_nano entry. If + both fields are populated, they MUST contain the same number of elements, and + the elements at the same index MUST refer to the same event. + + For the purposes of efficiently representing aggregated data observations, a Sample is regarded + as having a shared identity and an associated collection of per-observation data points. + Samples having the same identity SHOULD be combined by inserting timestamps and values to the data arrays. + + Examples of different ways ('shapes') of representing a sample with the total value of 10: + + Report of a stacktrace at 10 timestamps (consumers must assume the value is 1 for each point): + values: [] + timestamps_unix_nano: [1, 2, 3, 4, 5, 6, 7, 8, 9, 10] + + Report of a stacktrace with an aggregated value without timestamps: + values: [10] + timestamps_unix_nano: [] + + Report of a stacktrace at 4 timestamps where each point records a specific value: + values: [2, 2, 3, 3] + timestamps_unix_nano: [1, 2, 3, 4] + + All Samples for a Profile SHOULD have the same shape, i.e. all data observation series should consistently + adopt the same data recording style. + + Status: [Alpha] + """ + + DESCRIPTOR: _descriptor.Descriptor + + STACK_INDEX_FIELD_NUMBER: _builtins.int + ATTRIBUTE_INDICES_FIELD_NUMBER: _builtins.int + LINK_INDEX_FIELD_NUMBER: _builtins.int + VALUES_FIELD_NUMBER: _builtins.int + TIMESTAMPS_UNIX_NANO_FIELD_NUMBER: _builtins.int + stack_index: _builtins.int + """A Sample's identity (i.e. 'primary key') is the tuple of {stack_index, set_of(attribute_indices), link_index} + + Reference to stack in ProfilesDictionary.stack_table. + """ + link_index: _builtins.int + """Reference to link in ProfilesDictionary.link_table. [optional] + It can be unset / set to 0 if no link exists, as link_table[0] is always a 'null' default value. + """ + @_builtins.property + def attribute_indices(self) -> _containers.RepeatedScalarFieldContainer[_builtins.int]: + """References to attributes in ProfilesDictionary.attribute_table. [optional]""" + + @_builtins.property + def values(self) -> _containers.RepeatedScalarFieldContainer[_builtins.int]: + """The following fields may contain per-observation data and do not form part of the Sample's identity. + + The type and unit of each value is defined by Profile.sample_type. + """ + + @_builtins.property + def timestamps_unix_nano(self) -> _containers.RepeatedScalarFieldContainer[_builtins.int]: + """Timestamps associated with Sample. Value is UNIX Epoch time in nanoseconds + since 00:00:00 UTC on 1 January 1970. The timestamps should fall within the + [Profile.time_unix_nano, Profile.time_unix_nano + Profile.duration_nano) + time range. + """ + + def __init__( + self, + *, + stack_index: _builtins.int = ..., + attribute_indices: _abc.Iterable[_builtins.int] | None = ..., + link_index: _builtins.int = ..., + values: _abc.Iterable[_builtins.int] | None = ..., + timestamps_unix_nano: _abc.Iterable[_builtins.int] | None = ..., + ) -> None: ... + _HasFieldArgType: _TypeAlias = _Never # noqa: Y015 + def HasField(self, field_name: _HasFieldArgType) -> _builtins.bool: ... + _ClearFieldArgType: _TypeAlias = _typing.Literal["attribute_indices", b"attribute_indices", "link_index", b"link_index", "stack_index", b"stack_index", "timestamps_unix_nano", b"timestamps_unix_nano", "values", b"values"] # noqa: Y015 + def ClearField(self, field_name: _ClearFieldArgType) -> None: ... + def WhichOneof(self, oneof_group: _Never) -> None: ... + +Global___Sample: _TypeAlias = Sample # noqa: Y015 + +@_typing.final +class Mapping(_message.Message): + """Describes the mapping of a binary in memory, including its address range, + file offset, and metadata like build ID + + Status: [Alpha] + """ + + DESCRIPTOR: _descriptor.Descriptor + + MEMORY_START_FIELD_NUMBER: _builtins.int + MEMORY_LIMIT_FIELD_NUMBER: _builtins.int + FILE_OFFSET_FIELD_NUMBER: _builtins.int + FILENAME_STRINDEX_FIELD_NUMBER: _builtins.int + ATTRIBUTE_INDICES_FIELD_NUMBER: _builtins.int + memory_start: _builtins.int + """Address at which the binary (or DLL) is loaded into memory.""" + memory_limit: _builtins.int + """The limit of the address range occupied by this mapping.""" + file_offset: _builtins.int + """Offset in the binary that corresponds to the first mapped address.""" + filename_strindex: _builtins.int + """The object this entry is loaded from. This can be a filename on + disk for the main binary and shared libraries, or virtual + abstractions like "[vdso]". + Index into ProfilesDictionary.string_table. + """ + @_builtins.property + def attribute_indices(self) -> _containers.RepeatedScalarFieldContainer[_builtins.int]: + """References to attributes in ProfilesDictionary.attribute_table. [optional]""" + + def __init__( + self, + *, + memory_start: _builtins.int = ..., + memory_limit: _builtins.int = ..., + file_offset: _builtins.int = ..., + filename_strindex: _builtins.int = ..., + attribute_indices: _abc.Iterable[_builtins.int] | None = ..., + ) -> None: ... + _HasFieldArgType: _TypeAlias = _Never # noqa: Y015 + def HasField(self, field_name: _HasFieldArgType) -> _builtins.bool: ... + _ClearFieldArgType: _TypeAlias = _typing.Literal["attribute_indices", b"attribute_indices", "file_offset", b"file_offset", "filename_strindex", b"filename_strindex", "memory_limit", b"memory_limit", "memory_start", b"memory_start"] # noqa: Y015 + def ClearField(self, field_name: _ClearFieldArgType) -> None: ... + def WhichOneof(self, oneof_group: _Never) -> None: ... + +Global___Mapping: _TypeAlias = Mapping # noqa: Y015 + +@_typing.final +class Stack(_message.Message): + """A Stack represents a stack trace as a list of locations. + + Status: [Alpha] + """ + + DESCRIPTOR: _descriptor.Descriptor + + LOCATION_INDICES_FIELD_NUMBER: _builtins.int + @_builtins.property + def location_indices(self) -> _containers.RepeatedScalarFieldContainer[_builtins.int]: + """References to locations in ProfilesDictionary.location_table. + The first location is the leaf frame. + """ + + def __init__( + self, + *, + location_indices: _abc.Iterable[_builtins.int] | None = ..., + ) -> None: ... + _HasFieldArgType: _TypeAlias = _Never # noqa: Y015 + def HasField(self, field_name: _HasFieldArgType) -> _builtins.bool: ... + _ClearFieldArgType: _TypeAlias = _typing.Literal["location_indices", b"location_indices"] # noqa: Y015 + def ClearField(self, field_name: _ClearFieldArgType) -> None: ... + def WhichOneof(self, oneof_group: _Never) -> None: ... + +Global___Stack: _TypeAlias = Stack # noqa: Y015 + +@_typing.final +class Location(_message.Message): + """Describes function and line table debug information. + + Status: [Alpha] + """ + + DESCRIPTOR: _descriptor.Descriptor + + MAPPING_INDEX_FIELD_NUMBER: _builtins.int + ADDRESS_FIELD_NUMBER: _builtins.int + LINES_FIELD_NUMBER: _builtins.int + ATTRIBUTE_INDICES_FIELD_NUMBER: _builtins.int + mapping_index: _builtins.int + """Reference to mapping in ProfilesDictionary.mapping_table. + It can be unset / set to 0 if the mapping is unknown or not applicable for + this profile type, as mapping_table[0] is always a 'null' default mapping. + """ + address: _builtins.int + """The instruction address for this location, if available. It + should be within [Mapping.memory_start...Mapping.memory_limit] + for the corresponding mapping. A non-leaf address may be in the + middle of a call instruction. It is up to display tools to find + the beginning of the instruction if necessary. + """ + @_builtins.property + def lines(self) -> _containers.RepeatedCompositeFieldContainer[Global___Line]: + """Multiple line indicates this location has inlined functions, + where the last entry represents the caller into which the + preceding entries were inlined. + + E.g., if memcpy() is inlined into printf: + lines[0].function_name == "memcpy" + lines[1].function_name == "printf" + """ + + @_builtins.property + def attribute_indices(self) -> _containers.RepeatedScalarFieldContainer[_builtins.int]: + """References to attributes in ProfilesDictionary.attribute_table. [optional]""" + + def __init__( + self, + *, + mapping_index: _builtins.int = ..., + address: _builtins.int = ..., + lines: _abc.Iterable[Global___Line] | None = ..., + attribute_indices: _abc.Iterable[_builtins.int] | None = ..., + ) -> None: ... + _HasFieldArgType: _TypeAlias = _Never # noqa: Y015 + def HasField(self, field_name: _HasFieldArgType) -> _builtins.bool: ... + _ClearFieldArgType: _TypeAlias = _typing.Literal["address", b"address", "attribute_indices", b"attribute_indices", "lines", b"lines", "mapping_index", b"mapping_index"] # noqa: Y015 + def ClearField(self, field_name: _ClearFieldArgType) -> None: ... + def WhichOneof(self, oneof_group: _Never) -> None: ... + +Global___Location: _TypeAlias = Location # noqa: Y015 + +@_typing.final +class Line(_message.Message): + """Details a specific line in a source code, linked to a function. + + Status: [Alpha] + """ + + DESCRIPTOR: _descriptor.Descriptor + + FUNCTION_INDEX_FIELD_NUMBER: _builtins.int + LINE_FIELD_NUMBER: _builtins.int + COLUMN_FIELD_NUMBER: _builtins.int + function_index: _builtins.int + """Reference to function in ProfilesDictionary.function_table.""" + line: _builtins.int + """Line number in source code. 0 means unset.""" + column: _builtins.int + """Column number in source code. 0 means unset.""" + def __init__( + self, + *, + function_index: _builtins.int = ..., + line: _builtins.int = ..., + column: _builtins.int = ..., + ) -> None: ... + _HasFieldArgType: _TypeAlias = _Never # noqa: Y015 + def HasField(self, field_name: _HasFieldArgType) -> _builtins.bool: ... + _ClearFieldArgType: _TypeAlias = _typing.Literal["column", b"column", "function_index", b"function_index", "line", b"line"] # noqa: Y015 + def ClearField(self, field_name: _ClearFieldArgType) -> None: ... + def WhichOneof(self, oneof_group: _Never) -> None: ... + +Global___Line: _TypeAlias = Line # noqa: Y015 + +@_typing.final +class Function(_message.Message): + """Describes a function, including its human-readable name, system name, + source file, and starting line number in the source. + + Status: [Alpha] + """ + + DESCRIPTOR: _descriptor.Descriptor + + NAME_STRINDEX_FIELD_NUMBER: _builtins.int + SYSTEM_NAME_STRINDEX_FIELD_NUMBER: _builtins.int + FILENAME_STRINDEX_FIELD_NUMBER: _builtins.int + START_LINE_FIELD_NUMBER: _builtins.int + name_strindex: _builtins.int + """The function name. Empty string if not available.""" + system_name_strindex: _builtins.int + """Function name, as identified by the system. For instance, + it can be a C++ mangled name. Empty string if not available. + """ + filename_strindex: _builtins.int + """Source file containing the function. Empty string if not available.""" + start_line: _builtins.int + """Line number in source file. 0 means unset.""" + def __init__( + self, + *, + name_strindex: _builtins.int = ..., + system_name_strindex: _builtins.int = ..., + filename_strindex: _builtins.int = ..., + start_line: _builtins.int = ..., + ) -> None: ... + _HasFieldArgType: _TypeAlias = _Never # noqa: Y015 + def HasField(self, field_name: _HasFieldArgType) -> _builtins.bool: ... + _ClearFieldArgType: _TypeAlias = _typing.Literal["filename_strindex", b"filename_strindex", "name_strindex", b"name_strindex", "start_line", b"start_line", "system_name_strindex", b"system_name_strindex"] # noqa: Y015 + def ClearField(self, field_name: _ClearFieldArgType) -> None: ... + def WhichOneof(self, oneof_group: _Never) -> None: ... + +Global___Function: _TypeAlias = Function # noqa: Y015 + +@_typing.final +class KeyValueAndUnit(_message.Message): + """A custom 'dictionary native' style of encoding attributes which is more convenient + for profiles than opentelemetry.proto.common.v1.KeyValue + Specifically, uses the string table for keys and allows optional unit information. + + Status: [Alpha] + """ + + DESCRIPTOR: _descriptor.Descriptor + + KEY_STRINDEX_FIELD_NUMBER: _builtins.int + VALUE_FIELD_NUMBER: _builtins.int + UNIT_STRINDEX_FIELD_NUMBER: _builtins.int + key_strindex: _builtins.int + """The index into the string table for the attribute's key.""" + unit_strindex: _builtins.int + """The index into the string table for the attribute's unit. + zero indicates implicit (by semconv) or non-defined unit. + """ + @_builtins.property + def value(self) -> _common_pb2.AnyValue: + """The value of the attribute.""" + + def __init__( + self, + *, + key_strindex: _builtins.int = ..., + value: _common_pb2.AnyValue | None = ..., + unit_strindex: _builtins.int = ..., + ) -> None: ... + _HasFieldArgType: _TypeAlias = _typing.Literal["value", b"value"] # noqa: Y015 + def HasField(self, field_name: _HasFieldArgType) -> _builtins.bool: ... + _ClearFieldArgType: _TypeAlias = _typing.Literal["key_strindex", b"key_strindex", "unit_strindex", b"unit_strindex", "value", b"value"] # noqa: Y015 + def ClearField(self, field_name: _ClearFieldArgType) -> None: ... + def WhichOneof(self, oneof_group: _Never) -> None: ... + +Global___KeyValueAndUnit: _TypeAlias = KeyValueAndUnit # noqa: Y015 diff --git a/logfire/_internal/profiling/_proto/profiles_service_pb2.py b/logfire/_internal/profiling/_proto/profiles_service_pb2.py new file mode 100644 index 000000000..2eaf81488 --- /dev/null +++ b/logfire/_internal/profiling/_proto/profiles_service_pb2.py @@ -0,0 +1,44 @@ +# -*- coding: utf-8 -*- +# Generated by the protocol buffer compiler. DO NOT EDIT! +# NO CHECKED-IN PROTOBUF GENCODE +# source: lf_otlp/profiles_service.proto +# Protobuf Python Version: 6.31.1 +"""Generated protocol buffer code.""" +from google.protobuf import descriptor as _descriptor +from google.protobuf import descriptor_pool as _descriptor_pool +from google.protobuf import runtime_version as _runtime_version +from google.protobuf import symbol_database as _symbol_database +from google.protobuf.internal import builder as _builder +_runtime_version.ValidateProtobufRuntimeVersion( + _runtime_version.Domain.PUBLIC, + 6, + 31, + 1, + '', + 'lf_otlp/profiles_service.proto' +) +# @@protoc_insertion_point(imports) + +_sym_db = _symbol_database.Default() + + +from logfire._internal.profiling._proto import profiles_pb2 as lf__otlp_dot_profiles__pb2 + + +DESCRIPTOR = _descriptor_pool.Default().AddSerializedFile(b'\n\x1elf_otlp/profiles_service.proto\x12\x11lf_otlp.collector\x1a\x16lf_otlp/profiles.proto\"\x97\x01\n\x1c\x45xportProfilesServiceRequest\x12=\n\x11resource_profiles\x18\x01 \x03(\x0b\x32\".lf_otlp.profiles.ResourceProfiles\x12\x38\n\ndictionary\x18\x02 \x01(\x0b\x32$.lf_otlp.profiles.ProfilesDictionary\"i\n\x1d\x45xportProfilesServiceResponse\x12H\n\x0fpartial_success\x18\x01 \x01(\x0b\x32/.lf_otlp.collector.ExportProfilesPartialSuccess\"P\n\x1c\x45xportProfilesPartialSuccess\x12\x19\n\x11rejected_profiles\x18\x01 \x01(\x03\x12\x15\n\rerror_message\x18\x02 \x01(\t2\x80\x01\n\x0fProfilesService\x12m\n\x06\x45xport\x12/.lf_otlp.collector.ExportProfilesServiceRequest\x1a\x30.lf_otlp.collector.ExportProfilesServiceResponse\"\x00\x42\xa6\x01\n\x14io.lf_otlp.collectorB\x14ProfilesServiceProtoP\x01Z?go.opentelemetry.io/proto/otlp/collector/profiles/v1development\xaa\x02\x34OpenTelemetry.Proto.Collector.Profiles.V1Developmentb\x06proto3') + +_globals = globals() +_builder.BuildMessageAndEnumDescriptors(DESCRIPTOR, _globals) +_builder.BuildTopDescriptorsAndMessages(DESCRIPTOR, 'lf_otlp.profiles_service_pb2', _globals) +if not _descriptor._USE_C_DESCRIPTORS: + _globals['DESCRIPTOR']._loaded_options = None + _globals['DESCRIPTOR']._serialized_options = b'\n\024io.lf_otlp.collectorB\024ProfilesServiceProtoP\001Z?go.opentelemetry.io/proto/otlp/collector/profiles/v1development\252\0024OpenTelemetry.Proto.Collector.Profiles.V1Development' + _globals['_EXPORTPROFILESSERVICEREQUEST']._serialized_start=78 + _globals['_EXPORTPROFILESSERVICEREQUEST']._serialized_end=229 + _globals['_EXPORTPROFILESSERVICERESPONSE']._serialized_start=231 + _globals['_EXPORTPROFILESSERVICERESPONSE']._serialized_end=336 + _globals['_EXPORTPROFILESPARTIALSUCCESS']._serialized_start=338 + _globals['_EXPORTPROFILESPARTIALSUCCESS']._serialized_end=418 + _globals['_PROFILESSERVICE']._serialized_start=421 + _globals['_PROFILESSERVICE']._serialized_end=549 +# @@protoc_insertion_point(module_scope) diff --git a/logfire/_internal/profiling/_proto/profiles_service_pb2.pyi b/logfire/_internal/profiling/_proto/profiles_service_pb2.pyi new file mode 100644 index 000000000..c9c218868 --- /dev/null +++ b/logfire/_internal/profiling/_proto/profiles_service_pb2.pyi @@ -0,0 +1,144 @@ +""" +@generated by mypy-protobuf. Do not edit manually! +isort:skip_file +Copyright 2023, OpenTelemetry Authors + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +""" + +from collections import abc as _abc +from google.protobuf import descriptor as _descriptor +from google.protobuf import message as _message +from google.protobuf.internal import containers as _containers +from logfire._internal.profiling._proto import profiles_pb2 as _profiles_pb2 +import builtins as _builtins +import sys +import typing as _typing + +if sys.version_info >= (3, 11): + from typing import TypeAlias as _TypeAlias, Never as _Never +else: + from typing_extensions import TypeAlias as _TypeAlias, Never as _Never + +DESCRIPTOR: _descriptor.FileDescriptor + +@_typing.final +class ExportProfilesServiceRequest(_message.Message): + """Status: [Alpha]""" + + DESCRIPTOR: _descriptor.Descriptor + + RESOURCE_PROFILES_FIELD_NUMBER: _builtins.int + DICTIONARY_FIELD_NUMBER: _builtins.int + @_builtins.property + def resource_profiles(self) -> _containers.RepeatedCompositeFieldContainer[_profiles_pb2.ResourceProfiles]: + """An array of ResourceProfiles. + For data coming from a single resource this array will typically contain one + element. Intermediary nodes (such as OpenTelemetry Collector) that receive + data from multiple origins typically batch the data before forwarding further and + in that case this array will contain multiple elements. + """ + + @_builtins.property + def dictionary(self) -> _profiles_pb2.ProfilesDictionary: + """The reference table containing all data shared by profiles across the message being sent.""" + + def __init__( + self, + *, + resource_profiles: _abc.Iterable[_profiles_pb2.ResourceProfiles] | None = ..., + dictionary: _profiles_pb2.ProfilesDictionary | None = ..., + ) -> None: ... + _HasFieldArgType: _TypeAlias = _typing.Literal["dictionary", b"dictionary"] # noqa: Y015 + def HasField(self, field_name: _HasFieldArgType) -> _builtins.bool: ... + _ClearFieldArgType: _TypeAlias = _typing.Literal["dictionary", b"dictionary", "resource_profiles", b"resource_profiles"] # noqa: Y015 + def ClearField(self, field_name: _ClearFieldArgType) -> None: ... + def WhichOneof(self, oneof_group: _Never) -> None: ... + +Global___ExportProfilesServiceRequest: _TypeAlias = ExportProfilesServiceRequest # noqa: Y015 + +@_typing.final +class ExportProfilesServiceResponse(_message.Message): + """Status: [Alpha]""" + + DESCRIPTOR: _descriptor.Descriptor + + PARTIAL_SUCCESS_FIELD_NUMBER: _builtins.int + @_builtins.property + def partial_success(self) -> Global___ExportProfilesPartialSuccess: + """The details of a partially successful export request. + + If the request is only partially accepted + (i.e. when the server accepts only parts of the data and rejects the rest) + the server MUST initialize the `partial_success` field and MUST + set the `rejected_` with the number of items it rejected. + + Servers MAY also make use of the `partial_success` field to convey + warnings/suggestions to senders even when the request was fully accepted. + In such cases, the `rejected_` MUST have a value of `0` and + the `error_message` MUST be non-empty. + + A `partial_success` message with an empty value (rejected_ = 0 and + `error_message` = "") is equivalent to it not being set/present. Senders + SHOULD interpret it the same way as in the full success case. + """ + + def __init__( + self, + *, + partial_success: Global___ExportProfilesPartialSuccess | None = ..., + ) -> None: ... + _HasFieldArgType: _TypeAlias = _typing.Literal["partial_success", b"partial_success"] # noqa: Y015 + def HasField(self, field_name: _HasFieldArgType) -> _builtins.bool: ... + _ClearFieldArgType: _TypeAlias = _typing.Literal["partial_success", b"partial_success"] # noqa: Y015 + def ClearField(self, field_name: _ClearFieldArgType) -> None: ... + def WhichOneof(self, oneof_group: _Never) -> None: ... + +Global___ExportProfilesServiceResponse: _TypeAlias = ExportProfilesServiceResponse # noqa: Y015 + +@_typing.final +class ExportProfilesPartialSuccess(_message.Message): + """Status: [Alpha]""" + + DESCRIPTOR: _descriptor.Descriptor + + REJECTED_PROFILES_FIELD_NUMBER: _builtins.int + ERROR_MESSAGE_FIELD_NUMBER: _builtins.int + rejected_profiles: _builtins.int + """The number of rejected profiles. + + A `rejected_` field holding a `0` value indicates that the + request was fully accepted. + """ + error_message: _builtins.str + """A developer-facing human-readable message in English. It should be used + either to explain why the server rejected parts of the data during a partial + success or to convey warnings/suggestions during a full success. The message + should offer guidance on how users can address such issues. + + error_message is an optional field. An error_message with an empty value + is equivalent to it not being set. + """ + def __init__( + self, + *, + rejected_profiles: _builtins.int = ..., + error_message: _builtins.str = ..., + ) -> None: ... + _HasFieldArgType: _TypeAlias = _Never # noqa: Y015 + def HasField(self, field_name: _HasFieldArgType) -> _builtins.bool: ... + _ClearFieldArgType: _TypeAlias = _typing.Literal["error_message", b"error_message", "rejected_profiles", b"rejected_profiles"] # noqa: Y015 + def ClearField(self, field_name: _ClearFieldArgType) -> None: ... + def WhichOneof(self, oneof_group: _Never) -> None: ... + +Global___ExportProfilesPartialSuccess: _TypeAlias = ExportProfilesPartialSuccess # noqa: Y015 diff --git a/logfire/_internal/profiling/collapsed.py b/logfire/_internal/profiling/collapsed.py new file mode 100644 index 000000000..fdffb1f51 --- /dev/null +++ b/logfire/_internal/profiling/collapsed.py @@ -0,0 +1,77 @@ +"""Parse Tachyon (`python -m profiling.sampling ... --collapsed`) output. + +The collapsed-stack format emitted by the Python 3.15 sampling profiler is one +folded stack per line:: + + tid:;;;...; + +Frames are ordered outermost-first (the leaf/innermost frame is last). Each +frame is ``::``. ```` is the number of +samples that landed on that exact stack. The format carries a thread id per +line but no per-sample timestamps and no sample rate. +""" + +from __future__ import annotations + +from collections.abc import Iterator +from dataclasses import dataclass + + +@dataclass(frozen=True) +class Frame: + """A single resolved stack frame.""" + + filename: str + function: str + lineno: int + + +@dataclass +class FoldedStack: + """One folded stack: a unique call stack and how many samples hit it.""" + + thread_id: int + # Ordered leaf-first (innermost frame at index 0) to match the + # pprof/OTLP `Sample` location ordering. + frames: list[Frame] + count: int + + +def _parse_frame(token: str) -> Frame: + # rsplit so a Windows drive letter (or any ':' in the path) survives: + # only the final two ':' separate function name and line number. + filename, function, lineno = token.rsplit(':', 2) + try: + line = int(lineno) + except ValueError: + line = 0 + return Frame(filename=filename, function=function, lineno=line) + + +def parse_collapsed(text: str) -> Iterator[FoldedStack]: + """Yield a `FoldedStack` for each non-empty line of collapsed-stack text.""" + for raw in text.splitlines(): + line = raw.strip() + if not line: + continue + stack_part, _, count_part = line.rpartition(' ') + if not stack_part: + continue + try: + count = int(count_part) + except ValueError: + # Not a folded-stack line (e.g. a stray header) - skip it. + continue + + tokens = stack_part.split(';') + thread_id = 0 + if tokens and tokens[0].startswith('tid:'): + try: + thread_id = int(tokens[0][len('tid:') :]) + except ValueError: + thread_id = 0 + tokens = tokens[1:] + + frames = [_parse_frame(token) for token in tokens if token] + frames.reverse() # outermost-first on the wire -> leaf-first model + yield FoldedStack(thread_id=thread_id, frames=frames, count=count) diff --git a/logfire/_internal/profiling/exporter.py b/logfire/_internal/profiling/exporter.py new file mode 100644 index 000000000..a8758bad7 --- /dev/null +++ b/logfire/_internal/profiling/exporter.py @@ -0,0 +1,67 @@ +"""Export OTLP profiles over HTTP. + +The OpenTelemetry profiles signal has no SDK exporter yet, so this posts the +protobuf directly. In production the `session` should be Logfire's +`OTLPExporterHttpSession` (disk-retry, gzip, the token `Authorization` header), +and `endpoint` the region base URL joined with the profiles path below. + +Everything here fails soft: a profiling problem must never disrupt the app or +the other signals, so `export()` returns a bool and never raises. +""" + +from __future__ import annotations + +import gzip +import warnings +from typing import Protocol + +from ._proto.profiles_service_pb2 import ExportProfilesServiceRequest + +# Profiles is still a development signal - note the path is NOT `/v1/profiles`. +PROFILES_PATH = '/v1development/profiles' + + +class _Response(Protocol): + # Read-only properties so a `requests.Response` (whose `text` is a property) satisfies this. + @property + def status_code(self) -> int: ... + + @property + def text(self) -> str: ... + + +class _PostSession(Protocol): + def post(self, url: str, *, data: bytes, headers: dict[str, str], timeout: float) -> _Response: ... + + +class ProfilesExporter: + """Posts OTLP profiles to an HTTP endpoint, failing soft on any error.""" + + def __init__(self, session: _PostSession, endpoint: str, *, timeout: float = 10.0) -> None: + self._session = session + self._endpoint = endpoint + self._timeout = timeout + + def export(self, request: ExportProfilesServiceRequest) -> bool: + """Serialize, gzip and POST the request. Returns True on a 2xx response.""" + try: + payload = gzip.compress(request.SerializeToString()) + response = self._session.post( + self._endpoint, + data=payload, + headers={ + 'Content-Type': 'application/x-protobuf', + 'Content-Encoding': 'gzip', + }, + timeout=self._timeout, + ) + except Exception as exc: + warnings.warn(f'Logfire profiling: failed to export profile: {exc!r}') + return False + + if not (200 <= response.status_code < 300): + warnings.warn( + f'Logfire profiling: profile export rejected ({response.status_code}): {response.text[:200]!r}' + ) + return False + return True diff --git a/logfire/_internal/profiling/otlp.py b/logfire/_internal/profiling/otlp.py new file mode 100644 index 000000000..5c0e91026 --- /dev/null +++ b/logfire/_internal/profiling/otlp.py @@ -0,0 +1,171 @@ +"""Build an OTLP profiles export request from Tachyon folded stacks. + +This uses the protobuf bindings shipped by `opentelemetry-proto` directly +(`opentelemetry.proto.profiles.v1development`) - there is no hand-written +profile model. The current OTLP profiles model keeps a single request-level +`ProfilesDictionary` holding the shared string/function/location/stack/ +attribute tables; every `Sample` references into it by index. +""" + +from __future__ import annotations + +import os +from collections.abc import Iterable, Mapping + +# common/resource are stable signals - the installed opentelemetry-proto is fine. +from opentelemetry.proto.common.v1.common_pb2 import AnyValue, InstrumentationScope, KeyValue +from opentelemetry.proto.resource.v1.resource_pb2 import Resource + +# profiles is an alpha signal; use the vendored (current) bindings, not the +# outdated snapshot in the opentelemetry-proto PyPI package. See _proto/. +from ._proto import profiles_pb2 as pb +from ._proto.profiles_service_pb2 import ExportProfilesServiceRequest +from .collapsed import FoldedStack + + +class _DictionaryBuilder: + """Interns entries into a shared `ProfilesDictionary`, returning each entry's index.""" + + def __init__(self) -> None: + self.dictionary = pb.ProfilesDictionary() + self.dictionary.string_table.append('') # index 0 is "" by convention + self._strings: dict[str, int] = {'': 0} + self._functions: dict[tuple[int, int], int] = {} + self._locations: dict[tuple[int, int], int] = {} + self._stacks: dict[tuple[int, ...], int] = {} + self._attributes: dict[tuple[str, int], int] = {} + # OTLP profiles require every Location to reference a Mapping. A pure + # Python profile has no native mappings, so use a single synthetic + # entry at index 0 - consumers (pprof, Pyroscope) reject a Location + # whose mapping_index points outside the mapping table. + self.dictionary.mapping_table.append(pb.Mapping(filename_strindex=self.string('python'))) + + def string(self, value: str) -> int: + idx = self._strings.get(value) + if idx is None: + idx = len(self.dictionary.string_table) + self.dictionary.string_table.append(value) + self._strings[value] = idx + return idx + + def function(self, filename: str, name: str) -> int: + key = (self.string(filename), self.string(name)) + idx = self._functions.get(key) + if idx is None: + idx = len(self.dictionary.function_table) + self.dictionary.function_table.append(pb.Function(filename_strindex=key[0], name_strindex=key[1])) + self._functions[key] = idx + return idx + + def location(self, filename: str, name: str, line: int) -> int: + function_index = self.function(filename, name) + key = (function_index, line) + idx = self._locations.get(key) + if idx is None: + idx = len(self.dictionary.location_table) + self.dictionary.location_table.append( + pb.Location( + mapping_index=0, + lines=[pb.Line(function_index=function_index, line=line)], + ) + ) + self._locations[key] = idx + return idx + + def stack(self, location_indices: tuple[int, ...]) -> int: + idx = self._stacks.get(location_indices) + if idx is None: + idx = len(self.dictionary.stack_table) + self.dictionary.stack_table.append(pb.Stack(location_indices=location_indices)) + self._stacks[location_indices] = idx + return idx + + def int_attribute(self, key: str, value: int) -> int: + cache_key = (key, value) + idx = self._attributes.get(cache_key) + if idx is None: + idx = len(self.dictionary.attribute_table) + self.dictionary.attribute_table.append( + pb.KeyValueAndUnit( + key_strindex=self.string(key), + value=AnyValue(int_value=value), + ) + ) + self._attributes[cache_key] = idx + return idx + + +def build_export_request( + stacks: Iterable[FoldedStack], + *, + resource: Resource | None = None, + scope_name: str = 'logfire.profiling', + scope_version: str = '', + sample_type: str = 'samples', + sample_unit: str = 'count', + period_type: str = '', + period_unit: str = '', + period: int = 0, + start_time_unix_nano: int = 0, + duration_nano: int = 0, + profile_id: bytes | None = None, +) -> ExportProfilesServiceRequest: + """Convert folded stacks into a ready-to-POST `ExportProfilesServiceRequest`. + + `period`/`period_type` describe the sampling interval (e.g. type `cpu`, + unit `nanoseconds`, period 500000 for a 2 kHz profiler) - the collapsed + format does not carry it, so the caller supplies it from the `-r` rate. + """ + builder = _DictionaryBuilder() + profile = pb.Profile( + sample_type=pb.ValueType( + type_strindex=builder.string(sample_type), + unit_strindex=builder.string(sample_unit), + ), + period_type=pb.ValueType( + type_strindex=builder.string(period_type), + unit_strindex=builder.string(period_unit), + ), + period=period, + time_unix_nano=start_time_unix_nano, + duration_nano=duration_nano, + profile_id=profile_id if profile_id is not None else os.urandom(16), + ) + + for stack in stacks: + location_indices = tuple( + builder.location(frame.filename, frame.function, frame.lineno) for frame in stack.frames + ) + sample = pb.Sample( + stack_index=builder.stack(location_indices), + values=[stack.count], + ) + if stack.thread_id: + # `thread.id` is an OTel semantic-convention attribute. A later + # step will add `link_index` pointing at a Link(trace_id, span_id). + sample.attribute_indices.append(builder.int_attribute('thread.id', stack.thread_id)) + profile.samples.append(sample) + + return ExportProfilesServiceRequest( + dictionary=builder.dictionary, + resource_profiles=[ + pb.ResourceProfiles( + resource=resource or Resource(), + scope_profiles=[ + pb.ScopeProfiles( + scope=InstrumentationScope(name=scope_name, version=scope_version), + profiles=[profile], + ) + ], + ) + ], + ) + + +def resource_from_attributes(attributes: Mapping[str, object]) -> Resource: + """Build an OTLP `Resource` from a flat attribute mapping (values stringified).""" + return Resource( + attributes=[ + KeyValue(key=str(key), value=AnyValue(string_value=str(value))) for key, value in attributes.items() + ] + ) diff --git a/logfire/_internal/profiling/supervisor.py b/logfire/_internal/profiling/supervisor.py new file mode 100644 index 000000000..9d1af78f8 --- /dev/null +++ b/logfire/_internal/profiling/supervisor.py @@ -0,0 +1,175 @@ +"""Run the Python sampling profiler against this process and export OTLP profiles. + +This drives the Python 3.15 `profiling.sampling` (Tachyon) profiler as a child +process that attaches back to this one, in repeated fixed-duration chunks. Each +chunk's collapsed-stack output is converted to OTLP profiles and exported. + +Everything degrades gracefully: if the profiler is unavailable (Python < 3.15) +or the platform / permissions do not allow attaching, profiling is disabled +with a warning and the rest of Logfire is unaffected. Nothing here raises. +""" + +from __future__ import annotations + +import ctypes +import importlib.util +import os +import subprocess +import sys +import tempfile +import threading +import time +import warnings +from pathlib import Path + +from opentelemetry.proto.resource.v1.resource_pb2 import Resource + +from .collapsed import parse_collapsed +from .exporter import ProfilesExporter +from .otlp import build_export_request + +# prctl(PR_SET_PTRACER, PR_SET_PTRACER_ANY) lets a same-uid descendant ptrace +# us without root - which is what the profiler child needs on Linux under the +# common Yama `ptrace_scope=1`. The constant 0x59616d61 spells "Yama". +_PR_SET_PTRACER = 0x59616D61 +_PR_SET_PTRACER_ANY = ctypes.c_ulong(-1) + +_NS_PER_SECOND = 1_000_000_000 + + +def profiler_available() -> bool: + """Return True if the `profiling.sampling` profiler is importable (Python 3.15+).""" + try: + return importlib.util.find_spec('profiling.sampling') is not None + except (ImportError, ValueError): + return False + + +def _allow_child_ptrace() -> None: + """Best-effort: let a child process ptrace this one (Linux / Yama only).""" + if not sys.platform.startswith('linux'): + return # macOS / Windows need elevation instead; handled by failing soft + try: + libc = ctypes.CDLL(None, use_errno=True) # not "libc.so.6" - works on musl too + libc.prctl(_PR_SET_PTRACER, _PR_SET_PTRACER_ANY, 0, 0, 0) + except (OSError, AttributeError, ValueError): + pass # hardened kernel / no prctl - the profiler simply fails its first chunk + + +class ProfilingSupervisor: + """Continuously profiles this process in a background thread. + + Each cycle runs the profiler for `chunk_duration_seconds`, converts the + result to OTLP profiles and hands it to `exporter`. + """ + + def __init__( + self, + exporter: ProfilesExporter, + *, + resource: Resource | None = None, + sample_rate_hz: int = 1000, + chunk_duration_seconds: float = 60.0, + scope_version: str = '', + ) -> None: + self._exporter = exporter + self._resource = resource + self._sample_rate_hz = sample_rate_hz + self._chunk_duration = chunk_duration_seconds + self._scope_version = scope_version + self._stop = threading.Event() + self._thread: threading.Thread | None = None + self._proc: subprocess.Popen[str] | None = None + + def start(self) -> bool: + """Start background profiling. Returns False (with a warning) if unsupported.""" + if not profiler_available(): + warnings.warn('Logfire profiling needs Python 3.15+ (the `profiling.sampling` module); disabled.') + return False + if self._thread is not None: + return True # already running + _allow_child_ptrace() + self._thread = threading.Thread(target=self._run, name='logfire-profiling', daemon=True) + self._thread.start() + return True + + def shutdown(self, timeout: float = 5.0) -> None: + """Stop profiling, kill the profiler subprocess and join the background thread.""" + self._stop.set() + proc = self._proc + if proc is not None and proc.poll() is None: + proc.terminate() + thread = self._thread + if thread is not None: + thread.join(timeout=timeout) + self._thread = None + + def _run(self) -> None: + pid = os.getpid() + while not self._stop.is_set(): + if not self._run_once(pid): + # A failed chunk is almost always permanent (permissions, platform), + # so stop rather than spin and emit the same warning forever. + break + + def _run_once(self, pid: int) -> bool: + """Capture, convert and export one profiling chunk. Returns False on failure.""" + start_time = time.time_ns() + collapsed = self._capture_chunk(pid, self._chunk_duration, self._sample_rate_hz) + if collapsed is None: + return False + if not collapsed.strip(): + return True # nothing sampled this chunk (e.g. an idle process) - keep going + + request = build_export_request( + parse_collapsed(collapsed), + resource=self._resource, + scope_version=self._scope_version, + period_type='cpu', + period_unit='nanoseconds', + period=_NS_PER_SECOND // self._sample_rate_hz, + start_time_unix_nano=start_time, + duration_nano=int(self._chunk_duration * _NS_PER_SECOND), + ) + self._exporter.export(request) # fails soft internally + return True + + def _capture_chunk(self, pid: int, duration: float, rate: int) -> str | None: + """Run one profiler subprocess; return collapsed-stack text, or None on failure. + + The profiler exits 0 even when it cannot read the target's memory, so + success is judged by whether non-empty output was actually written. + """ + with tempfile.TemporaryDirectory(prefix='logfire-profiling-') as tmp: + out = Path(tmp) / 'chunk.collapsed' + cmd = [ + sys.executable, '-m', 'profiling.sampling', 'attach', + '--collapsed', '--all-threads', + '-d', str(duration), '-r', str(rate), + '-o', str(out), str(pid), + ] # fmt: skip + stderr = '' + try: + self._proc = subprocess.Popen(cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE, text=True) + _, stderr = self._proc.communicate(timeout=duration + 30) + except subprocess.TimeoutExpired: + self._kill_proc() + warnings.warn('Logfire profiling: profiler subprocess timed out.') + return None + except OSError as exc: + warnings.warn(f'Logfire profiling: could not run the profiler: {exc!r}') + return None + finally: + self._proc = None + + if not out.exists() or out.stat().st_size == 0: + if not self._stop.is_set(): # an empty file during shutdown is expected + warnings.warn(f'Logfire profiling: profiler produced no data. {stderr.strip()[:300]}') + return None + return out.read_text() + + def _kill_proc(self) -> None: + proc = self._proc + if proc is not None and proc.poll() is None: + proc.kill() + proc.communicate() diff --git a/pyproject.toml b/pyproject.toml index 747135abb..3ce3e6e0d 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -252,7 +252,11 @@ include = ["/README.md", "CHANGELOG.md", "/Makefile", "/logfire", "/tests"] # https://beta.ruff.rs/docs/configuration/ [tool.ruff] line-length = 120 -extend-exclude = ["logfire-api/logfire_api/*", "tests/auto_trace_samples/param_spec.py"] +extend-exclude = [ + "logfire-api/logfire_api/*", + "tests/auto_trace_samples/param_spec.py", + "logfire/_internal/profiling/_proto/*", # vendored generated protobuf code +] [tool.ruff.lint] extend-select = [ @@ -297,6 +301,7 @@ typeCheckingMode = "strict" reportUnnecessaryTypeIgnoreComment = true reportMissingTypeStubs = false include = ["logfire", "tests"] +exclude = ["logfire/_internal/profiling/_proto"] # vendored generated protobuf code venvPath = "." venv = ".venv" diff --git a/tests/profiling_fixtures/tachyon_demo.collapsed b/tests/profiling_fixtures/tachyon_demo.collapsed new file mode 100644 index 000000000..0bbd7b4a4 --- /dev/null +++ b/tests/profiling_fixtures/tachyon_demo.collapsed @@ -0,0 +1,21 @@ +tid:57922729;:_run_module_as_main:196;:_run_code:87;prof_demo.py::13;prof_demo.py:main:12;prof_demo.py:busy:7 632 +tid:57922729;:_run_module_as_main:196;:_run_code:87;prof_demo.py::13;prof_demo.py:main:12;prof_demo.py:busy:6 82 +tid:57922729;:_run_module_as_main:196;:_run_code:87;prof_demo.py::13;prof_demo.py:main:11;prof_demo.py:fib:3;prof_demo.py:fib:3;prof_demo.py:fib:3;prof_demo.py:fib:3;prof_demo.py:fib:3;prof_demo.py:fib:3;prof_demo.py:fib:3;prof_demo.py:fib:3;prof_demo.py:fib:3;prof_demo.py:fib:3;prof_demo.py:fib:3;prof_demo.py:fib:3;prof_demo.py:fib:3;prof_demo.py:fib:3;prof_demo.py:fib:3;prof_demo.py:fib:3;prof_demo.py:fib:3;prof_demo.py:fib:3;prof_demo.py:fib:3;prof_demo.py:fib:3 47 +tid:57922729;:_run_module_as_main:196;:_run_code:87;prof_demo.py::13;prof_demo.py:main:11;prof_demo.py:fib:3;prof_demo.py:fib:3;prof_demo.py:fib:3;prof_demo.py:fib:3;prof_demo.py:fib:3;prof_demo.py:fib:3;prof_demo.py:fib:3;prof_demo.py:fib:3;prof_demo.py:fib:3;prof_demo.py:fib:3;prof_demo.py:fib:3;prof_demo.py:fib:3;prof_demo.py:fib:3;prof_demo.py:fib:3;prof_demo.py:fib:3;prof_demo.py:fib:3;prof_demo.py:fib:3;prof_demo.py:fib:3;prof_demo.py:fib:3 36 +tid:57922729;:_run_module_as_main:196;:_run_code:87;prof_demo.py::13;prof_demo.py:main:11;prof_demo.py:fib:3;prof_demo.py:fib:3;prof_demo.py:fib:3;prof_demo.py:fib:3;prof_demo.py:fib:3;prof_demo.py:fib:3;prof_demo.py:fib:3;prof_demo.py:fib:3;prof_demo.py:fib:3;prof_demo.py:fib:3;prof_demo.py:fib:3;prof_demo.py:fib:3;prof_demo.py:fib:3;prof_demo.py:fib:3;prof_demo.py:fib:3;prof_demo.py:fib:3;prof_demo.py:fib:3;prof_demo.py:fib:3;prof_demo.py:fib:3;prof_demo.py:fib:3;prof_demo.py:fib:3 34 +tid:57922729;:_run_module_as_main:196;:_run_code:87;prof_demo.py::13;prof_demo.py:main:11;prof_demo.py:fib:3;prof_demo.py:fib:3;prof_demo.py:fib:3;prof_demo.py:fib:3;prof_demo.py:fib:3;prof_demo.py:fib:3;prof_demo.py:fib:3;prof_demo.py:fib:3;prof_demo.py:fib:3;prof_demo.py:fib:3;prof_demo.py:fib:3;prof_demo.py:fib:3;prof_demo.py:fib:3;prof_demo.py:fib:3;prof_demo.py:fib:3;prof_demo.py:fib:3;prof_demo.py:fib:3;prof_demo.py:fib:3 29 +tid:57922729;:_run_module_as_main:196;:_run_code:87;prof_demo.py::13;prof_demo.py:main:11;prof_demo.py:fib:3;prof_demo.py:fib:3;prof_demo.py:fib:3;prof_demo.py:fib:3;prof_demo.py:fib:3;prof_demo.py:fib:3;prof_demo.py:fib:3;prof_demo.py:fib:3;prof_demo.py:fib:3;prof_demo.py:fib:3;prof_demo.py:fib:3;prof_demo.py:fib:3;prof_demo.py:fib:3;prof_demo.py:fib:3;prof_demo.py:fib:3;prof_demo.py:fib:3;prof_demo.py:fib:3;prof_demo.py:fib:3;prof_demo.py:fib:3;prof_demo.py:fib:3;prof_demo.py:fib:3;prof_demo.py:fib:3 28 +tid:57922729;:_run_module_as_main:196;:_run_code:87;prof_demo.py::13;prof_demo.py:main:11;prof_demo.py:fib:3;prof_demo.py:fib:3;prof_demo.py:fib:3;prof_demo.py:fib:3;prof_demo.py:fib:3;prof_demo.py:fib:3;prof_demo.py:fib:3;prof_demo.py:fib:3;prof_demo.py:fib:3;prof_demo.py:fib:3;prof_demo.py:fib:3;prof_demo.py:fib:3;prof_demo.py:fib:3;prof_demo.py:fib:3;prof_demo.py:fib:3;prof_demo.py:fib:3;prof_demo.py:fib:3 18 +tid:57922729;:_run_module_as_main:196;:_run_code:87;prof_demo.py::13;prof_demo.py:main:11;prof_demo.py:fib:3;prof_demo.py:fib:3;prof_demo.py:fib:3;prof_demo.py:fib:3;prof_demo.py:fib:3;prof_demo.py:fib:3;prof_demo.py:fib:3;prof_demo.py:fib:3;prof_demo.py:fib:3;prof_demo.py:fib:3;prof_demo.py:fib:3;prof_demo.py:fib:3;prof_demo.py:fib:3;prof_demo.py:fib:3;prof_demo.py:fib:3;prof_demo.py:fib:3 11 +tid:57922729;:_run_module_as_main:196;:_run_code:87;prof_demo.py::13;prof_demo.py:main:11;prof_demo.py:fib:3;prof_demo.py:fib:3;prof_demo.py:fib:3;prof_demo.py:fib:3;prof_demo.py:fib:3;prof_demo.py:fib:3;prof_demo.py:fib:3;prof_demo.py:fib:3;prof_demo.py:fib:3;prof_demo.py:fib:3;prof_demo.py:fib:3;prof_demo.py:fib:3;prof_demo.py:fib:3;prof_demo.py:fib:3;prof_demo.py:fib:3;prof_demo.py:fib:3;prof_demo.py:fib:3;prof_demo.py:fib:3;prof_demo.py:fib:3;prof_demo.py:fib:3;prof_demo.py:fib:3;prof_demo.py:fib:3;prof_demo.py:fib:3 10 +tid:57922729;:_run_module_as_main:196;:_run_code:87;prof_demo.py::13;prof_demo.py:main:11;prof_demo.py:fib:3;prof_demo.py:fib:3;prof_demo.py:fib:3;prof_demo.py:fib:3;prof_demo.py:fib:3;prof_demo.py:fib:3;prof_demo.py:fib:3;prof_demo.py:fib:3;prof_demo.py:fib:3;prof_demo.py:fib:3;prof_demo.py:fib:3;prof_demo.py:fib:3;prof_demo.py:fib:3 4 +tid:57922729;:_run_module_as_main:196;:_run_code:87;prof_demo.py::13;prof_demo.py:main:11;prof_demo.py:fib:3;prof_demo.py:fib:3;prof_demo.py:fib:3;prof_demo.py:fib:3;prof_demo.py:fib:3;prof_demo.py:fib:3;prof_demo.py:fib:3;prof_demo.py:fib:3;prof_demo.py:fib:3;prof_demo.py:fib:3;prof_demo.py:fib:3;prof_demo.py:fib:3;prof_demo.py:fib:3;prof_demo.py:fib:3 4 +tid:57922729;:_run_module_as_main:196;:_run_code:87;prof_demo.py::13;prof_demo.py:main:11;prof_demo.py:fib:3;prof_demo.py:fib:3;prof_demo.py:fib:3;prof_demo.py:fib:3;prof_demo.py:fib:3;prof_demo.py:fib:3;prof_demo.py:fib:3;prof_demo.py:fib:3;prof_demo.py:fib:3;prof_demo.py:fib:3;prof_demo.py:fib:3;prof_demo.py:fib:3;prof_demo.py:fib:3;prof_demo.py:fib:3;prof_demo.py:fib:3;prof_demo.py:fib:3;prof_demo.py:fib:3;prof_demo.py:fib:3;prof_demo.py:fib:3;prof_demo.py:fib:3;prof_demo.py:fib:3;prof_demo.py:fib:2 4 +tid:57922729;:_run_module_as_main:196;:_run_code:87;prof_demo.py::13;prof_demo.py:main:11;prof_demo.py:fib:3;prof_demo.py:fib:3;prof_demo.py:fib:3;prof_demo.py:fib:3;prof_demo.py:fib:3;prof_demo.py:fib:3;prof_demo.py:fib:3;prof_demo.py:fib:3;prof_demo.py:fib:3;prof_demo.py:fib:3;prof_demo.py:fib:3;prof_demo.py:fib:3;prof_demo.py:fib:3;prof_demo.py:fib:3;prof_demo.py:fib:3;prof_demo.py:fib:3;prof_demo.py:fib:3;prof_demo.py:fib:3;prof_demo.py:fib:3;prof_demo.py:fib:3;prof_demo.py:fib:3;prof_demo.py:fib:3;prof_demo.py:fib:3;prof_demo.py:fib:3 4 +tid:57922729;:_run_module_as_main:196;:_run_code:87;prof_demo.py::13;prof_demo.py:main:11;prof_demo.py:fib:3;prof_demo.py:fib:3;prof_demo.py:fib:3;prof_demo.py:fib:3;prof_demo.py:fib:3;prof_demo.py:fib:3;prof_demo.py:fib:3;prof_demo.py:fib:3;prof_demo.py:fib:3;prof_demo.py:fib:3;prof_demo.py:fib:3;prof_demo.py:fib:3 2 +tid:57922729;:_run_module_as_main:196;:_run_code:87;prof_demo.py::13;prof_demo.py:main:11;prof_demo.py:fib:3;prof_demo.py:fib:3;prof_demo.py:fib:3;prof_demo.py:fib:3;prof_demo.py:fib:3;prof_demo.py:fib:3;prof_demo.py:fib:3;prof_demo.py:fib:3;prof_demo.py:fib:3;prof_demo.py:fib:3;prof_demo.py:fib:3;prof_demo.py:fib:3;prof_demo.py:fib:3;prof_demo.py:fib:3;prof_demo.py:fib:3 2 +tid:57922729;:_run_module_as_main:196;:_run_code:87;prof_demo.py::13;prof_demo.py:main:11;prof_demo.py:fib:3;prof_demo.py:fib:3;prof_demo.py:fib:3;prof_demo.py:fib:3;prof_demo.py:fib:3;prof_demo.py:fib:3;prof_demo.py:fib:3;prof_demo.py:fib:3;prof_demo.py:fib:3;prof_demo.py:fib:3;prof_demo.py:fib:3;prof_demo.py:fib:3;prof_demo.py:fib:3;prof_demo.py:fib:3;prof_demo.py:fib:3;prof_demo.py:fib:3;prof_demo.py:fib:3;prof_demo.py:fib:3;prof_demo.py:fib:2 2 +tid:57922729;:_run_module_as_main:196;:_run_code:87;prof_demo.py::13;prof_demo.py:main:11;prof_demo.py:fib:3;prof_demo.py:fib:3;prof_demo.py:fib:3;prof_demo.py:fib:3;prof_demo.py:fib:3;prof_demo.py:fib:3;prof_demo.py:fib:3;prof_demo.py:fib:3;prof_demo.py:fib:3;prof_demo.py:fib:3;prof_demo.py:fib:3;prof_demo.py:fib:3;prof_demo.py:fib:3;prof_demo.py:fib:3;prof_demo.py:fib:3;prof_demo.py:fib:3;prof_demo.py:fib:3;prof_demo.py:fib:3;prof_demo.py:fib:3;prof_demo.py:fib:3;prof_demo.py:fib:2 2 +tid:57922729;:_run_module_as_main:196;:_run_code:87;prof_demo.py::13;prof_demo.py:main:11;prof_demo.py:fib:3;prof_demo.py:fib:3;prof_demo.py:fib:3;prof_demo.py:fib:3;prof_demo.py:fib:3;prof_demo.py:fib:3;prof_demo.py:fib:3;prof_demo.py:fib:3;prof_demo.py:fib:3;prof_demo.py:fib:3;prof_demo.py:fib:3;prof_demo.py:fib:3;prof_demo.py:fib:3;prof_demo.py:fib:3;prof_demo.py:fib:3;prof_demo.py:fib:3;prof_demo.py:fib:3;prof_demo.py:fib:3;prof_demo.py:fib:3;prof_demo.py:fib:3;prof_demo.py:fib:3;prof_demo.py:fib:3;prof_demo.py:fib:3;prof_demo.py:fib:3;prof_demo.py:fib:3 2 +tid:57922729;:_run_module_as_main:196;:_run_code:87;prof_demo.py::13;prof_demo.py:main:11;prof_demo.py:fib:3;prof_demo.py:fib:3;prof_demo.py:fib:3;prof_demo.py:fib:3;prof_demo.py:fib:3;prof_demo.py:fib:3;prof_demo.py:fib:3;prof_demo.py:fib:3;prof_demo.py:fib:3;prof_demo.py:fib:3;prof_demo.py:fib:3;prof_demo.py:fib:3;prof_demo.py:fib:3;prof_demo.py:fib:3;prof_demo.py:fib:3;prof_demo.py:fib:3;prof_demo.py:fib:2 1 +tid:57922729;:_run_module_as_main:196;:_run_code:87;prof_demo.py::13;prof_demo.py:main:11;prof_demo.py:fib:3;prof_demo.py:fib:3;prof_demo.py:fib:3;prof_demo.py:fib:3;prof_demo.py:fib:3;prof_demo.py:fib:3;prof_demo.py:fib:3;prof_demo.py:fib:3;prof_demo.py:fib:3;prof_demo.py:fib:3;prof_demo.py:fib:3;prof_demo.py:fib:3;prof_demo.py:fib:3;prof_demo.py:fib:3;prof_demo.py:fib:3;prof_demo.py:fib:3;prof_demo.py:fib:3;prof_demo.py:fib:3;prof_demo.py:fib:3;prof_demo.py:fib:3;prof_demo.py:fib:3;prof_demo.py:fib:3;prof_demo.py:fib:2 1 diff --git a/tests/test_profiling.py b/tests/test_profiling.py new file mode 100644 index 000000000..eefcbdfb3 --- /dev/null +++ b/tests/test_profiling.py @@ -0,0 +1,228 @@ +"""Tests for Tachyon collapsed-stack output -> OTLP profiles export request. + +Fixture `tachyon_demo.collapsed` is real output from the Python 3.15 +`python -m profiling.sampling run --collapsed -r 2khz` profiler. +""" + +from __future__ import annotations + +import gzip +import sys +from pathlib import Path +from typing import TypedDict + +import pytest +from inline_snapshot import snapshot + +from logfire._internal.profiling._proto import profiles_pb2 +from logfire._internal.profiling._proto.profiles_service_pb2 import ExportProfilesServiceRequest +from logfire._internal.profiling.collapsed import parse_collapsed +from logfire._internal.profiling.exporter import ProfilesExporter +from logfire._internal.profiling.otlp import build_export_request +from logfire._internal.profiling.supervisor import ProfilingSupervisor, profiler_available + +FIXTURE = Path(__file__).parent / 'profiling_fixtures' / 'tachyon_demo.collapsed' +PROFILE_ID = b'\x11' * 16 + + +def _resolve_sample(request: ExportProfilesServiceRequest, sample: profiles_pb2.Sample) -> list[str]: + """Resolve a sample's stack back to readable `file:func:line`, leaf-first.""" + d = request.dictionary + out: list[str] = [] + for location_index in d.stack_table[sample.stack_index].location_indices: + line = d.location_table[location_index].lines[0] + function = d.function_table[line.function_index] + filename = d.string_table[function.filename_strindex] + name = d.string_table[function.name_strindex] + out.append(f'{filename}:{name}:{line.line}') + return out + + +def test_parse_collapsed(): + stacks = list(parse_collapsed(FIXTURE.read_text())) + + assert len(stacks) == snapshot(21) + assert sum(stack.count for stack in stacks) == snapshot(955) + # Every line in this fixture is the same single thread. + assert {stack.thread_id for stack in stacks} == snapshot({57922729}) + + # Aggregate samples by innermost (leaf) frame -> a hotspot table. + by_leaf: dict[tuple[str, int], int] = {} + for stack in stacks: + leaf = stack.frames[0] + key = (leaf.function, leaf.lineno) + by_leaf[key] = by_leaf.get(key, 0) + stack.count + assert sorted(by_leaf.items(), key=lambda kv: -kv[1]) == snapshot( + [(('busy', 7), 632), (('fib', 3), 231), (('busy', 6), 82), (('fib', 2), 10)] + ) + + +def test_build_export_request(): + request = build_export_request( + parse_collapsed(FIXTURE.read_text()), + scope_version='0.spike', + profile_id=PROFILE_ID, + ) + d = request.dictionary + + # One Resource -> one Scope -> one Profile. + [resource_profiles] = request.resource_profiles + [scope_profiles] = resource_profiles.scope_profiles + [profile] = scope_profiles.profiles + assert scope_profiles.scope.name == snapshot('logfire.profiling') + + # Shared dictionary tables, all deduplicated. + assert ( + len(d.string_table), + len(d.function_table), + len(d.location_table), + len(d.stack_table), + len(d.attribute_table), + ) == snapshot((13, 6, 9, 21, 1)) + + # The single attribute interned across every sample. + assert d.string_table[d.attribute_table[0].key_strindex] == snapshot('thread.id') + assert d.attribute_table[0].value.int_value == snapshot(57922729) + + # sample_type resolves through the string table. + assert ( + d.string_table[profile.sample_type.type_strindex], + d.string_table[profile.sample_type.unit_strindex], + ) == snapshot(('samples', 'count')) + + # The hottest sample, resolved leaf-first. + hottest = max(profile.samples, key=lambda s: s.values[0]) + assert hottest.values[0] == snapshot(632) + assert _resolve_sample(request, hottest) == snapshot( + [ + 'prof_demo.py:busy:7', + 'prof_demo.py:main:12', + 'prof_demo.py::13', + ':_run_code:87', + ':_run_module_as_main:196', + ] + ) + + +def test_export_request_round_trips(): + request = build_export_request(parse_collapsed(FIXTURE.read_text()), profile_id=PROFILE_ID) + reparsed = ExportProfilesServiceRequest() + reparsed.ParseFromString(request.SerializeToString()) + assert reparsed == request + + +class _Call(TypedDict): + url: str + data: bytes + headers: dict[str, str] + timeout: float + + +class _FakeResponse: + def __init__(self, status_code: int, text: str = '') -> None: + self.status_code = status_code + self.text = text + + +class _FakeSession: + def __init__(self, response: _FakeResponse | Exception) -> None: + self._response = response + self.calls: list[_Call] = [] + + def post(self, url: str, *, data: bytes, headers: dict[str, str], timeout: float) -> _FakeResponse: + self.calls.append({'url': url, 'data': data, 'headers': headers, 'timeout': timeout}) + if isinstance(self._response, Exception): + raise self._response + return self._response + + +def test_exporter_posts_gzipped_protobuf(): + request = build_export_request(parse_collapsed(FIXTURE.read_text()), profile_id=PROFILE_ID) + session = _FakeSession(_FakeResponse(200)) + exporter = ProfilesExporter(session, 'https://logfire.example/v1development/profiles') + + assert exporter.export(request) is True + [call] = session.calls + assert call['headers'] == snapshot({'Content-Type': 'application/x-protobuf', 'Content-Encoding': 'gzip'}) + # The posted body is gzipped and decodes back to the same request. + reparsed = ExportProfilesServiceRequest() + reparsed.ParseFromString(gzip.decompress(call['data'])) + assert reparsed == request + + +def test_exporter_fails_soft_on_rejection(): + request = build_export_request(parse_collapsed(FIXTURE.read_text()), profile_id=PROFILE_ID) + exporter = ProfilesExporter( + _FakeSession(_FakeResponse(503, 'overloaded')), 'https://logfire.example/v1development/profiles' + ) + with pytest.warns(UserWarning, match='profile export rejected'): + assert exporter.export(request) is False + + +ENDPOINT = 'https://logfire.example/v1development/profiles' + + +def test_profiler_available_matches_python_version(): + # `profiling.sampling` is stdlib from Python 3.15. + assert profiler_available() is (sys.version_info >= (3, 15)) + + +def test_supervisor_disabled_without_profiler(monkeypatch: pytest.MonkeyPatch): + monkeypatch.setattr('logfire._internal.profiling.supervisor.profiler_available', lambda: False) + supervisor = ProfilingSupervisor(ProfilesExporter(_FakeSession(_FakeResponse(200)), ENDPOINT)) + with pytest.warns(UserWarning, match='Python 3.15'): + assert supervisor.start() is False + supervisor.shutdown() # a no-op, but must be safe even when never started + + +def test_supervisor_profiles_and_exports(monkeypatch: pytest.MonkeyPatch): + monkeypatch.setattr('logfire._internal.profiling.supervisor.profiler_available', lambda: True) + session = _FakeSession(_FakeResponse(200)) + supervisor = ProfilingSupervisor( + ProfilesExporter(session, ENDPOINT), sample_rate_hz=1000, chunk_duration_seconds=1.0 + ) + + fixture_text = FIXTURE.read_text() + calls: list[int] = [] + + def fake_capture(pid: int, duration: float, rate: int) -> str | None: + calls.append(pid) + # One real chunk, then None to end the loop (None == permanent failure). + return fixture_text if len(calls) == 1 else None + + monkeypatch.setattr(supervisor, '_capture_chunk', fake_capture) + assert supervisor.start() is True + supervisor.shutdown(timeout=5.0) # joins the background thread + + # The one chunk was converted and exported. + [call] = session.calls + reparsed = ExportProfilesServiceRequest() + reparsed.ParseFromString(gzip.decompress(call['data'])) + profile = reparsed.resource_profiles[0].scope_profiles[0].profiles[0] + assert len(profile.samples) == 21 + assert profile.period == 1_000_000 # 1e9 ns / 1000 Hz + + +def test_supervisor_skips_empty_chunks(monkeypatch: pytest.MonkeyPatch): + monkeypatch.setattr('logfire._internal.profiling.supervisor.profiler_available', lambda: True) + session = _FakeSession(_FakeResponse(200)) + supervisor = ProfilingSupervisor(ProfilesExporter(session, ENDPOINT)) + + calls: list[int] = [] + + def fake_capture(pid: int, duration: float, rate: int) -> str | None: + calls.append(pid) + return '' if len(calls) == 1 else None # an idle chunk, then stop + + monkeypatch.setattr(supervisor, '_capture_chunk', fake_capture) + supervisor.start() + supervisor.shutdown(timeout=5.0) + + assert session.calls == [] # an empty chunk is not exported + + +def test_exporter_fails_soft_on_exception(): + request = build_export_request(parse_collapsed(FIXTURE.read_text()), profile_id=PROFILE_ID) + exporter = ProfilesExporter(_FakeSession(ConnectionError('boom')), 'https://logfire.example/v1development/profiles') + with pytest.warns(UserWarning, match='failed to export profile'): + assert exporter.export(request) is False