From eb4069ba7927b111477390694b61ff828a8c26af Mon Sep 17 00:00:00 2001 From: eric-forte-elastic Date: Wed, 3 Jun 2026 18:07:16 -0400 Subject: [PATCH 1/8] Add ESQL patch version support and related integrations datastream fix --- detection_rules/integrations.py | 49 ++++++++++++++++++++++++++++-- detection_rules/rule_validators.py | 15 +++++++-- pyproject.toml | 2 +- 3 files changed, 60 insertions(+), 6 deletions(-) diff --git a/detection_rules/integrations.py b/detection_rules/integrations.py index 4476caea191..46b8e2957d4 100644 --- a/detection_rules/integrations.py +++ b/detection_rules/integrations.py @@ -9,7 +9,7 @@ import gzip import json from collections import OrderedDict, defaultdict -from collections.abc import Iterator +from collections.abc import Iterable, Iterator from pathlib import Path from typing import TYPE_CHECKING, Any @@ -244,6 +244,38 @@ def _satisfies_kibana_range(stack: Version, version_requirement: str) -> bool: return any(lo <= stack and (hi is None or stack < hi) for lo, hi in _parse_kibana_range(version_requirement)) +def find_latest_integration_patch_for_minor(packages: Iterable[str], major: int, minor: int) -> int: + """Find the latest stack patch the given integration packages need for a major.minor.""" + # The stack-schema-map keys stacks at MAJOR.MINOR.0, but an integration may gate its latest + # package (and newly-added data streams) behind a later patch (e.g. azure ~8.19.10). Resolving + # against the literal .0 falls back to an older package that predates the stream. Return the + # latest patch a package gates on for the minor, i.e. the stack patch needed to receive the most + # up-to-date integration package on that minor. Only the supplied packages are inspected rather + # than the full manifest, so the scan stays small. + manifests = load_integrations_manifests() + latest_patch = 0 + for package in packages: + # Manifests sorted newest-first: a package raises its floor as it adds features, so the newest + # version gating the minor carries the controlling patch and we can stop at the first match. + package_manifests = sorted( + manifests.get(package, {}).items(), key=lambda kv: Version.parse(kv[0]), reverse=True + ) + for _, manifest in package_manifests: + version_requirement = manifest.get("conditions", {}).get("kibana", {}).get("version") + if not version_requirement: + continue + try: + clauses = _parse_kibana_range(version_requirement) + except ValueError: + # Skip manifests whose kibana condition uses tokens we cannot parse. + continue + floors = [lo.patch for lo, _ in clauses if lo.major == major and lo.minor == minor] + if floors: + latest_patch = max(latest_patch, *floors) + break + return latest_patch + + def find_least_compatible_version( package: str, integration: str, @@ -254,6 +286,14 @@ def find_least_compatible_version( integration_manifests = dict(sorted(packages_manifest[package].items(), key=lambda x: Version.parse(x[0]))) stack_version = Version.parse(current_stack_version, optional_minor_and_patch=True) + # The manifest's kibana condition only tells us whether the *package* installs on the stack, not + # whether this particular integration/data stream exists yet in that package version (e.g. azure + # added aadgraphactivitylogs in 1.37.0, but 1.0.0 already installs on 8.19). The schemas record + # the data streams present per package version, so use them to skip versions that predate the + # integration. Only filter when schema data exists for a version, otherwise fall back to kibana + # compatibility alone (e.g. for synthetic manifests in tests). + package_schemas = load_integrations_schemas().get(package, {}) + # filter integration_manifests to only the latest major entries major_versions = sorted( {Version.parse(manifest_version).major for manifest_version in integration_manifests}, @@ -270,8 +310,11 @@ def find_least_compatible_version( sorted(major_integration_manifests.items(), key=lambda x: Version.parse(x[0])) ).items(): version_requirement = manifest["conditions"]["kibana"]["version"] - if _satisfies_kibana_range(stack_version, version_requirement): - return f"^{version}" + if not _satisfies_kibana_range(stack_version, version_requirement): + continue + if integration and version in package_schemas and integration not in package_schemas[version]: + continue + return f"^{version}" raise ValueError(f"no compatible version for integration {package}:{integration}") diff --git a/detection_rules/rule_validators.py b/detection_rules/rule_validators.py index 9b3f412c885..cc07e183141 100644 --- a/detection_rules/rule_validators.py +++ b/detection_rules/rule_validators.py @@ -41,6 +41,7 @@ prepare_mappings, ) from .integrations import ( + find_latest_integration_patch_for_minor, get_integration_schema_data, load_integrations_manifests, parse_datasets, @@ -924,8 +925,18 @@ def remote_validate_rule( # noqa: PLR0913 # mismatch error, as the EsqlSchemaError and EsqlSyntaxError errors from the stack # will not be impacted by the difference in schema type mapping. mappings_lookup: dict[str, dict[str, Any]] = {stack_version: combined_mappings} - versions = get_stack_versions() - for version in versions: + + # The schema-map keys stacks at MAJOR.MINOR.0, but an integration may gate its data stream + # behind a later patch (e.g. azure ~8.19.10). Validating at the literal .0 resolves an older + # package that predates the stream, so for each minor use the latest patch the rule's own + # integrations gate on. Only the rule's packages are inspected, not the full manifest. + rule_packages = set(get_rule_integrations(metadata)) + rule_packages.update(integration.package for integration in event_dataset_integrations) + + for version in get_stack_versions(): + parsed = Version.parse(version) + inferred_patch = find_latest_integration_patch_for_minor(rule_packages, parsed.major, parsed.minor) + version = str(parsed.replace(patch=max(parsed.patch, inferred_patch))) # noqa: PLW2901 if version in mappings_lookup: continue _, _, combined_mappings = prepare_mappings( diff --git a/pyproject.toml b/pyproject.toml index 9f55da53cd1..aa052f021a6 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,6 +1,6 @@ [project] name = "detection_rules" -version = "1.6.47" +version = "1.6.48" description = "Detection Rules is the home for rules used by Elastic Security. This repository is used for the development, maintenance, testing, validation, and release of rules for Elastic Security’s Detection Engine." readme = "README.md" requires-python = ">=3.12" From f67b60a89ad8c102c7a6af35c16e6ae2ad6f08f6 Mon Sep 17 00:00:00 2001 From: eric-forte-elastic Date: Thu, 4 Jun 2026 10:08:40 -0400 Subject: [PATCH 2/8] remove sort now O n log n --- detection_rules/integrations.py | 23 ++++++++++++----------- 1 file changed, 12 insertions(+), 11 deletions(-) diff --git a/detection_rules/integrations.py b/detection_rules/integrations.py index 46b8e2957d4..32078a2f647 100644 --- a/detection_rules/integrations.py +++ b/detection_rules/integrations.py @@ -250,17 +250,14 @@ def find_latest_integration_patch_for_minor(packages: Iterable[str], major: int, # package (and newly-added data streams) behind a later patch (e.g. azure ~8.19.10). Resolving # against the literal .0 falls back to an older package that predates the stream. Return the # latest patch a package gates on for the minor, i.e. the stack patch needed to receive the most - # up-to-date integration package on that minor. Only the supplied packages are inspected rather - # than the full manifest, so the scan stays small. + # up-to-date integration package on that minor. Scan each package once and track the newest + # matching package manifest. manifests = load_integrations_manifests() latest_patch = 0 for package in packages: - # Manifests sorted newest-first: a package raises its floor as it adds features, so the newest - # version gating the minor carries the controlling patch and we can stop at the first match. - package_manifests = sorted( - manifests.get(package, {}).items(), key=lambda kv: Version.parse(kv[0]), reverse=True - ) - for _, manifest in package_manifests: + latest_package_version: Version | None = None + latest_package_patch = 0 + for package_version, manifest in manifests.get(package, {}).items(): version_requirement = manifest.get("conditions", {}).get("kibana", {}).get("version") if not version_requirement: continue @@ -270,9 +267,13 @@ def find_latest_integration_patch_for_minor(packages: Iterable[str], major: int, # Skip manifests whose kibana condition uses tokens we cannot parse. continue floors = [lo.patch for lo, _ in clauses if lo.major == major and lo.minor == minor] - if floors: - latest_patch = max(latest_patch, *floors) - break + if not floors: + continue + parsed_package_version = Version.parse(package_version) + if latest_package_version is None or parsed_package_version > latest_package_version: + latest_package_version = parsed_package_version + latest_package_patch = max(floors) + latest_patch = max(latest_patch, latest_package_patch) return latest_patch From d4ca0b0ce848229c6f20f7b034e03b08f40af895 Mon Sep 17 00:00:00 2001 From: eric-forte-elastic Date: Thu, 4 Jun 2026 12:39:54 -0400 Subject: [PATCH 3/8] Update unit test to not use remote route when available --- tests/test_schemas.py | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/tests/test_schemas.py b/tests/test_schemas.py index 52eac327bd3..3fad2499aeb 100644 --- a/tests/test_schemas.py +++ b/tests/test_schemas.py @@ -364,6 +364,15 @@ def test_stack_schema_map(self): class TestESQLValidation(unittest.TestCase): """Test ESQL rule validation""" + def setUp(self): + """Force local validation for these tests.""" + # These cases exercise local AST/semantic validation (KEEP/METADATA checks). Routing them + # through remote validation is possible, but the explicit goal of these is to use local vs remote, + # so we patch the environment variable to force local validation regardless of other settings. + patcher = unittest.mock.patch.dict(os.environ, {"DR_REMOTE_ESQL_VALIDATION": ""}) + patcher.start() + self.addCleanup(patcher.stop) + def test_esql_data_validation(self): """Test ESQL rule data validation""" From 4d0a2363480a02f6568f465819b6a19ff42702f1 Mon Sep 17 00:00:00 2001 From: eric-forte-elastic Date: Thu, 4 Jun 2026 12:47:05 -0400 Subject: [PATCH 4/8] Add unit tests for new functionality --- tests/test_integrations.py | 41 ++++++++++++++++++++++++++++++++++++++ 1 file changed, 41 insertions(+) diff --git a/tests/test_integrations.py b/tests/test_integrations.py index aafd5fea869..80eb1135275 100644 --- a/tests/test_integrations.py +++ b/tests/test_integrations.py @@ -6,6 +6,7 @@ """Test integration version resolution against EPR manifest ranges.""" import unittest +import unittest.mock from semver import Version @@ -257,3 +258,43 @@ def test_or_clause(self): """OR'd clauses are honored by the least-compatible search.""" manifests = {"pkg": {"1.0.0": _manifest("^8.12.0 || ^9.0.0")}} self.assertEqual(find_least_compatible_version("pkg", "pkg", "9.1.0", manifests), "^1.0.0") + + def test_skips_versions_missing_integration(self): + """Kibana-compatible versions whose schema lacks the integration are skipped for a later one.""" + # Mirrors a data stream added in a later package (e.g. azure aadgraphactivitylogs in 1.37.0): + # older packages still install on the stack but predate the data stream. + manifests = { + "pkg": { + "1.0.0": _manifest("^8.12.0"), + "1.5.0": _manifest("^8.12.0"), + "1.9.0": _manifest("^8.12.0"), + } + } + schemas = { + "pkg": { + "1.0.0": {"existing_ds": {}}, + "1.5.0": {"existing_ds": {}}, + "1.9.0": {"existing_ds": {}, "new_ds": {}}, + } + } + with unittest.mock.patch("detection_rules.integrations.load_integrations_schemas", return_value=schemas): + # 1.0.0/1.5.0 are kibana-compatible but lack new_ds; 1.9.0 is the oldest that has it. + self.assertEqual(find_least_compatible_version("pkg", "new_ds", "8.12.0", manifests), "^1.9.0") + # An integration present in every version is unaffected and still resolves to the oldest. + self.assertEqual(find_least_compatible_version("pkg", "existing_ds", "8.12.0", manifests), "^1.0.0") + + def test_no_schema_data_falls_back_to_kibana_only(self): + """Versions without schema data are not filtered; kibana compatibility alone decides.""" + manifests = {"pkg": {"1.0.0": _manifest("^8.12.0"), "1.5.0": _manifest("^8.12.0")}} + with unittest.mock.patch("detection_rules.integrations.load_integrations_schemas", return_value={}): + self.assertEqual(find_least_compatible_version("pkg", "new_ds", "8.12.0", manifests), "^1.0.0") + + def test_all_compatible_versions_missing_integration_raises(self): + """Raise when every kibana-compatible version's schema lacks the requested integration.""" + manifests = {"pkg": {"1.0.0": _manifest("^8.12.0"), "1.5.0": _manifest("^8.12.0")}} + schemas = {"pkg": {"1.0.0": {"existing_ds": {}}, "1.5.0": {"existing_ds": {}}}} + with ( + unittest.mock.patch("detection_rules.integrations.load_integrations_schemas", return_value=schemas), + self.assertRaises(ValueError), + ): + find_least_compatible_version("pkg", "new_ds", "8.12.0", manifests) From 911c0b7d84742bd1f3d318bc9e725915f42d9f47 Mon Sep 17 00:00:00 2001 From: Eric Forte <119343520+eric-forte-elastic@users.noreply.github.com> Date: Thu, 4 Jun 2026 12:48:40 -0400 Subject: [PATCH 5/8] Potential fix for pull request finding Co-authored-by: Copilot Autofix powered by AI <175728472+Copilot@users.noreply.github.com> --- detection_rules/integrations.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/detection_rules/integrations.py b/detection_rules/integrations.py index 32078a2f647..3e0196e33e2 100644 --- a/detection_rules/integrations.py +++ b/detection_rules/integrations.py @@ -279,7 +279,7 @@ def find_latest_integration_patch_for_minor(packages: Iterable[str], major: int, def find_least_compatible_version( package: str, - integration: str, + integration: str | None, current_stack_version: str, packages_manifest: dict[str, Any], ) -> str: From 512229ad5f6efee1172ce83f12f333e57782b130 Mon Sep 17 00:00:00 2001 From: eric-forte-elastic Date: Thu, 4 Jun 2026 12:53:26 -0400 Subject: [PATCH 6/8] Small optimization --- detection_rules/integrations.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/detection_rules/integrations.py b/detection_rules/integrations.py index 3e0196e33e2..8ef57023887 100644 --- a/detection_rules/integrations.py +++ b/detection_rules/integrations.py @@ -293,7 +293,9 @@ def find_least_compatible_version( # the data streams present per package version, so use them to skip versions that predate the # integration. Only filter when schema data exists for a version, otherwise fall back to kibana # compatibility alone (e.g. for synthetic manifests in tests). - package_schemas = load_integrations_schemas().get(package, {}) + # Loaded only when an integration is specified, to avoid decompressing the schemas for + # package-only lookups where the schema check is never consulted. + package_schemas = load_integrations_schemas().get(package, {}) if integration else {} # filter integration_manifests to only the latest major entries major_versions = sorted( From e3fb1319af3a238393e6247db6c5572f82ea1954 Mon Sep 17 00:00:00 2001 From: eric-forte-elastic Date: Thu, 4 Jun 2026 13:08:03 -0400 Subject: [PATCH 7/8] Add type hint --- detection_rules/integrations.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/detection_rules/integrations.py b/detection_rules/integrations.py index 8ef57023887..88b74fc41ed 100644 --- a/detection_rules/integrations.py +++ b/detection_rules/integrations.py @@ -295,7 +295,7 @@ def find_least_compatible_version( # compatibility alone (e.g. for synthetic manifests in tests). # Loaded only when an integration is specified, to avoid decompressing the schemas for # package-only lookups where the schema check is never consulted. - package_schemas = load_integrations_schemas().get(package, {}) if integration else {} + package_schemas: dict[str, Any] = load_integrations_schemas().get(package, {}) if integration else {} # filter integration_manifests to only the latest major entries major_versions = sorted( From d8bc77978d6cfd20dd1f1f372ddb919331ce5eaf Mon Sep 17 00:00:00 2001 From: eric-forte-elastic Date: Thu, 4 Jun 2026 13:14:32 -0400 Subject: [PATCH 8/8] Update comment --- detection_rules/integrations.py | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/detection_rules/integrations.py b/detection_rules/integrations.py index 88b74fc41ed..e4a5bad9482 100644 --- a/detection_rules/integrations.py +++ b/detection_rules/integrations.py @@ -250,8 +250,13 @@ def find_latest_integration_patch_for_minor(packages: Iterable[str], major: int, # package (and newly-added data streams) behind a later patch (e.g. azure ~8.19.10). Resolving # against the literal .0 falls back to an older package that predates the stream. Return the # latest patch a package gates on for the minor, i.e. the stack patch needed to receive the most - # up-to-date integration package on that minor. Scan each package once and track the newest - # matching package manifest. + # up-to-date integration package on that minor. + # + # Track the *newest* package version's floor (not the max floor across all versions): Fleet always + # installs the latest compatible package, so that floor is the patch a stack actually needs. A + # newer package occasionally lowers its floor (e.g. apm 7.16.1 gates ^7.16.1 but the newer 7.16.2 + # gates ^7.16.0); honoring the newest version matches what Fleet installs rather than an older, + # higher floor that would never be installed on that stack. manifests = load_integrations_manifests() latest_patch = 0 for package in packages: