From 6d54352e79cdf47e82fe2325d7ea14d03e8941cf Mon Sep 17 00:00:00 2001 From: Vinayak Date: Mon, 6 Jul 2026 12:51:01 +0530 Subject: [PATCH] feat(correlation): add incident enrichments to correlation rules and support raw alert storage - Introduced `incident_enrichments` field in correlation rules to allow metadata attachment to incidents. - Updated the `KEEP_STORE_RAW_ALERTS` configuration to enable storing raw webhooks for incident enrichment. - Enhanced the rules engine to process and apply incident enrichments during incident creation. - Added database migrations for new fields in the `rule` and `alertraw` tables. - Updated relevant API routes and UI components to support the new enrichment functionality. --- docs/deployment/configuration.mdx | 2 +- docs/overview/correlation-rules.mdx | 118 ++++++++++++++ .../CorrelationSidebar/CorrelationForm.tsx | 72 ++++++++- .../CorrelationSidebarBody.tsx | 19 +++ .../(keep)/rules/CorrelationSidebar/index.tsx | 8 + .../(keep)/rules/CorrelationSidebar/types.ts | 1 + keep-ui/utils/hooks/useRules.ts | 1 + keep/api/consts.py | 1 + keep/api/core/db.py | 22 ++- keep/api/models/alert.py | 9 +- .../versions/2026-06-30-10-30_a38f1c29d7e4.py | 28 ++++ keep/api/models/db/rule.py | 1 + keep/api/routes/rules.py | 7 +- keep/api/routes/settings.py | 4 +- keep/api/tasks/process_event_task.py | 25 ++- keep/rulesengine/rulesengine.py | 145 +++++++++++++++++- 16 files changed, 441 insertions(+), 22 deletions(-) create mode 100644 keep/api/models/db/migrations/versions/2026-06-30-10-30_a38f1c29d7e4.py diff --git a/docs/deployment/configuration.mdx b/docs/deployment/configuration.mdx index e70738df53..2197da66ca 100644 --- a/docs/deployment/configuration.mdx +++ b/docs/deployment/configuration.mdx @@ -25,7 +25,7 @@ Keep is highly configurable through environment variables. This allows you to cu | **CONSUMER** | Enables or disables the consumer | No | "true" | "true" or "false" | | **KEEP_VERSION** | Specifies the Keep version | No | "unknown" | Valid version string | | **KEEP_API_URL** | Specifies the Keep API URL | No | Constructed from HOST and PORT | Valid URL | -| **KEEP_STORE_RAW_ALERTS** | Enables storing of raw alerts | No | "false" | "true" or "false" | +| **KEEP_STORE_RAW_ALERTS** | When `true`: stores full provider webhooks in `alertraw` (one row per webhook) and enables **correlation rule incident enrichments** (`{{ raw.* }}` templates). Raw data for enrichments is passed in-memory on `AlertDto.keep_raw_payload` during ingestion. UI incident-enrichments section is shown only when this flag is on. | No | "false" | "true" or "false" | | **TENANT_CONFIGURATION_RELOAD_TIME** | Time in minutes to reload tenant configurations | No | 5 | Positive integer | | **KEEP_LIVE_DEMO_MODE** | Keep will simulate incoming alerts and other activity | No | "false" | "true" or "false" | diff --git a/docs/overview/correlation-rules.mdx b/docs/overview/correlation-rules.mdx index 63583fbe8b..55cd4afcad 100644 --- a/docs/overview/correlation-rules.mdx +++ b/docs/overview/correlation-rules.mdx @@ -79,6 +79,124 @@ When an incident contains multiple alerts: Service Issue on host1,host2 +## Incident Enrichments + +Correlation rules can attach **key/value metadata** to incidents when they are **first created**. Configure this in the Correlation rule editor under **Incident enrichments**. + +Each row has: +- **Key** — field name on the incident (e.g. `threshold`, `monitor`, `team`) +- **Value** — static text or a template using `{{ ... }}` syntax + +Enrichments appear on the incident detail page and are stored in the incident enrichment record. + +### Requirements + +Incident enrichments from correlation rules require: + +```bash +KEEP_STORE_RAW_ALERTS=true +``` + +Set this in your `.env` or deployment environment and restart the Keep API. + +When `KEEP_STORE_RAW_ALERTS` is **disabled**, correlation rules still create incidents, but **no incident enrichments** are applied. + +### When enrichments run + +| Scenario | Enrichments | +|----------|-------------| +| **New incident** created by the rule | Templates are rendered and saved | +| **Existing incident** reused (same rule fingerprint) | Not re-run — first alert's enrichments are kept | + +Enrichments are resolved from the **triggering alert** at incident creation time, using: + +1. **Formatted alert** (`alert.*` / top-level fields on `AlertDto`) +2. **In-memory raw webhook** (`keep_raw_payload` on `AlertDto` during ingestion — not saved on the `alert` row) + +When `KEEP_STORE_RAW_ALERTS=true`, the full provider webhook is also written once per webhook to the `alertraw` table (for debugging/archive). Incident enrichments read raw fields from **in-memory** `keep_raw_payload`, not from a DB lookup. + +### Template syntax + +Use `{{ variable }}` placeholders. Whitespace inside braces is trimmed automatically. + +#### Alert fields (formatted `AlertDto`) + +Reference normalized alert fields stored in `alert.event`: + +``` +{{ alert.labels.monitor }} +{{ alert.name }} +{{ alert.severity }} +{{ labels.monitor }} # same as alert.labels.monitor when on AlertDto +``` + +#### Raw webhook fields (`KEEP_STORE_RAW_ALERTS=true`) + +Reference the provider's original webhook payload (full webhook stored in `alertraw`, passed in-memory during ingestion): + +``` +{{ raw.alerts[0].condition }} +{{ raw.alerts[0].labels.monitor }} +{{ raw.alerts[0].data[0].model.conditions[0].evaluator.params[0] }} +``` + +Dot index works too: `{{ raw.alerts.0.data.0.model... }}` + +- Prefix with `raw.` to read **only** from the raw webhook +- Without `raw.`, Keep tries the formatted alert first, then falls back to raw +- Bracket notation `alerts[0]` and dot notation `alerts.0` both work + +For Grafana (and similar providers), the stored raw payload is the **full webhook**. Fields like `data`, `condition`, and `labels` live under `alerts[index]` — **not** at the top level. + +| Wrong (returns `N/A`) | Correct | +|-----------------------|---------| +| `{{ raw.data[0].model... }}` | `{{ raw.alerts[0].data[0].model... }}` | +| `{{ raw.condition }}` | `{{ raw.alerts[0].condition }}` | +| `{{ raw.labels.monitor }}` | `{{ raw.alerts[0].labels.monitor }}` or `{{ alert.labels.monitor }}` | + +For multi-alert webhooks, use the index of the alert you care about (`alerts[0]`, `alerts[1]`, …). Prefer `{{ alert.* }}` when Grafana already maps the field onto the normalized alert. + +### Grafana example + +**Webhook payload (simplified):** + +```json +{ + "title": "NetworkLatencyIsHigh", + "alerts": [{ + "condition": "C", + "labels": { "monitor": "router1" }, + "data": [{ + "model": { + "conditions": [{ + "evaluator": { "params": [100], "type": "gt" } + }] + } + }] + }] +} +``` + +**Correlation rule — Incident enrichments:** + +| Key | Value | +|-----|-------| +| `monitor` | `{{ alert.labels.monitor }}` | +| `condition` | `{{ raw.alerts[0].condition }}` | +| `threshold` | `{{ raw.alerts[0].data[0].model.conditions[0].evaluator.params[0] }}` | + +**Result on incident:** `monitor=router1`, `condition=C`, `threshold=100` + +### Troubleshooting + +| Symptom | Likely cause | +|---------|----------------| +| Enrichment value is `N/A` | Wrong template path — for Grafana use `raw.alerts[0].…`, not `raw.data[0].…` | +| No enrichments on incident | `KEEP_STORE_RAW_ALERTS` is not `true`, or API was not restarted after changing `.env` | +| `raw.*` returns `N/A` | Flag off, API not restarted, or no `keep_raw_payload` on alert — send a **new** alert after enabling | +| Incident enrichments UI hidden | `KEEP_STORE_RAW_ALERTS` is not `true` on the API (UI reads `store_raw_alerts_enabled`) | +| Wrong value on reused incident | Enrichments only run at **create**; delete/recreate incident or wait for expiry | + ## Examples - **Metric-based alerts**: Construct a rule to pinpoint alerts associated with specific metrics, such as high CPU usage on servers. This can be achieved by grouping alerts that share a common attribute, like a 'CPU usage' tag, ensuring you quickly identify and address performance issues. - **Feature-related alerts**: Establish rules to create incident by specific features or services. For instance, you can start incident based on a 'service' or 'URL' tag. This approach is particularly useful for tracking and managing alerts related to distinct functionalities or components within your application. diff --git a/keep-ui/app/(keep)/rules/CorrelationSidebar/CorrelationForm.tsx b/keep-ui/app/(keep)/rules/CorrelationSidebar/CorrelationForm.tsx index 43ec5627fd..16921c5be0 100644 --- a/keep-ui/app/(keep)/rules/CorrelationSidebar/CorrelationForm.tsx +++ b/keep-ui/app/(keep)/rules/CorrelationSidebar/CorrelationForm.tsx @@ -9,9 +9,18 @@ import { Text, TextInput, } from "@tremor/react"; -import { Controller, get, useFormContext } from "react-hook-form"; +import { + Controller, + get, + useFieldArray, + useFormContext, +} from "react-hook-form"; import { AlertDto } from "@/entities/alerts/model"; -import { QuestionMarkCircleIcon } from "@heroicons/react/24/outline"; +import { + PlusIcon, + QuestionMarkCircleIcon, + TrashIcon, +} from "@heroicons/react/24/outline"; import React from "react"; import { CorrelationFormType } from "./types"; import { useTenantConfiguration } from "@/utils/hooks/useTenantConfiguration"; @@ -33,6 +42,14 @@ export const CorrelationForm = ({ watch, formState: { errors, isSubmitted }, } = useFormContext(); + const { + fields: incidentEnrichmentFields, + append: appendIncidentEnrichment, + remove: removeIncidentEnrichment, + } = useFieldArray({ + control, + name: "incidentEnrichments", + }); const { data: tenantConfiguration } = useTenantConfiguration(); const { data: users = [] } = useUsers(); @@ -352,6 +369,57 @@ export const CorrelationForm = ({ Created incidents require manual approve + {tenantConfiguration?.["store_raw_alerts_enabled"] && ( +
+
+
+ {incidentEnrichmentFields.map((field, index) => ( +
+ + +
+ ))} +
+ )} {tenantConfiguration?.["multi_level_enabled"] && (
{ const api = useApi(); const { data: config } = useConfig(); + const { data: tenantConfiguration } = useTenantConfiguration(); + const isStoreRawAlertsEnabled = + !!tenantConfiguration?.["store_raw_alerts_enabled"]; const methods = useForm({ defaultValues: defaultValue, @@ -79,8 +83,22 @@ export const CorrelationSidebarBody = ({ multiLevelPropertyName, threshold, assignee, + incidentEnrichments, } = correlationFormData; + const incidentEnrichmentsObject = isStoreRawAlertsEnabled + ? incidentEnrichments.reduce( + (acc, enrichment) => { + const key = enrichment.key.trim(); + if (key) { + acc[key] = enrichment.value; + } + return acc; + }, + {} as Record + ) + : {}; + const body = { sqlQuery: formatQuery(query, "parameterized_named"), groupDescription: description, @@ -98,6 +116,7 @@ export const CorrelationSidebarBody = ({ multiLevelPropertyName, threshold, assignee, + incidentEnrichments: incidentEnrichmentsObject, }; try { diff --git a/keep-ui/app/(keep)/rules/CorrelationSidebar/index.tsx b/keep-ui/app/(keep)/rules/CorrelationSidebar/index.tsx index 3d61d925e3..29390b799c 100644 --- a/keep-ui/app/(keep)/rules/CorrelationSidebar/index.tsx +++ b/keep-ui/app/(keep)/rules/CorrelationSidebar/index.tsx @@ -29,6 +29,7 @@ export const DEFAULT_CORRELATION_FORM_VALUES: CorrelationFormType = { multiLevelPropertyName: "", threshold: 1, assignee: undefined, + incidentEnrichments: [], query: { combinator: "or", rules: [ @@ -63,6 +64,12 @@ export const CorrelationSidebar = ({ ); const timeunit = selectedRule.timeunit ?? "seconds"; + const incidentEnrichments = Object.entries( + selectedRule.incident_enrichments || {} + ).map(([key, value]) => ({ + key, + value: String(value), + })); return { name: selectedRule.name, @@ -83,6 +90,7 @@ export const CorrelationSidebar = ({ multiLevelPropertyName: selectedRule.multi_level_property_name || "", threshold: selectedRule.threshold || 1, assignee: selectedRule.assignee, + incidentEnrichments, }; } diff --git a/keep-ui/app/(keep)/rules/CorrelationSidebar/types.ts b/keep-ui/app/(keep)/rules/CorrelationSidebar/types.ts index dc9b512e13..cea5727f5f 100644 --- a/keep-ui/app/(keep)/rules/CorrelationSidebar/types.ts +++ b/keep-ui/app/(keep)/rules/CorrelationSidebar/types.ts @@ -16,4 +16,5 @@ export type CorrelationFormType = { multiLevelPropertyName?: string; threshold: number; assignee?: string; + incidentEnrichments: { key: string; value: string }[]; }; diff --git a/keep-ui/utils/hooks/useRules.ts b/keep-ui/utils/hooks/useRules.ts index affe7e2287..63271bef28 100644 --- a/keep-ui/utils/hooks/useRules.ts +++ b/keep-ui/utils/hooks/useRules.ts @@ -29,6 +29,7 @@ export type Rule = { multi_level_property_name: string | null; threshold: number; assignee: string | undefined; + incident_enrichments: Record; }; export const useRules = (options?: SWRConfiguration) => { diff --git a/keep/api/consts.py b/keep/api/consts.py index f2f8ce9e00..49011904d4 100644 --- a/keep/api/consts.py +++ b/keep/api/consts.py @@ -54,6 +54,7 @@ OPENAI_MODEL_NAME = os.environ.get("OPENAI_MODEL_NAME", "gpt-4o-2024-08-06") KEEP_CORRELATION_ENABLED = os.environ.get("KEEP_CORRELATION_ENABLED", "true") == "true" +KEEP_STORE_RAW_ALERTS = os.environ.get("KEEP_STORE_RAW_ALERTS", "false") == "true" FINGERPRINT_PAYLOAD_LIMIT = 100 diff --git a/keep/api/core/db.py b/keep/api/core/db.py index 878d0d877b..8da1bdf0d0 100644 --- a/keep/api/core/db.py +++ b/keep/api/core/db.py @@ -1296,11 +1296,12 @@ def _enrich_entity( """ enrichment = get_enrichment_with_session(session, tenant_id, fingerprint) if enrichment: + previous_enrichments = enrichment.enrichments or {} # if force - override exisitng enrichments. being used to dispose enrichments if necessary if force: new_enrichment_data = enrichments else: - new_enrichment_data = {**enrichment.enrichments, **enrichments} + new_enrichment_data = {**previous_enrichments, **enrichments} # SQLAlchemy doesn't support updating JSON fields, so we need to do it manually # https://github.com/sqlalchemy/sqlalchemy/discussions/8396#discussion-4308891 stmt = ( @@ -2241,8 +2242,10 @@ def create_rule( multi_level_property_name=None, threshold=1, assignee=None, + incident_enrichments=None, ): grouping_criteria = grouping_criteria or [] + incident_enrichments = incident_enrichments or {} with Session(engine) as session: rule = Rule( tenant_id=tenant_id, @@ -2264,6 +2267,7 @@ def create_rule( multi_level_property_name=multi_level_property_name, threshold=threshold, assignee=assignee, + incident_enrichments=incident_enrichments, ) session.add(rule) session.commit() @@ -2290,6 +2294,7 @@ def update_rule( multi_level_property_name, threshold, assignee=None, + incident_enrichments=None, ): rule_uuid = __convert_to_uuid(rule_id) if not rule_uuid: @@ -2318,6 +2323,7 @@ def update_rule( rule.multi_level_property_name = multi_level_property_name rule.threshold = threshold rule.assignee = assignee + rule.incident_enrichments = incident_enrichments or {} session.commit() session.refresh(rule) return rule @@ -2418,9 +2424,9 @@ def create_incident_for_grouping_rule( incident_name: str = None, past_incident: Optional[Incident] = None, assignee: str | None = None, + enrichments: dict | None = None, session: Optional[Session] = None, ): - with existed_or_new_session(session) as session: # Create and add a new incident if it doesn't exist incident = Incident( @@ -2442,6 +2448,16 @@ def create_incident_for_grouping_rule( incident.user_generated_name = f"{rule.incident_prefix}-{incident.running_number} - {incident.user_generated_name}" session.commit() session.refresh(incident) + if enrichments: + enrich_entity( + tenant_id=tenant_id, + fingerprint=str(incident.id), + enrichments=enrichments, + action_type=ActionType.INCIDENT_ENRICH, + action_callee="correlation-engine", + action_description="Incident enriched during correlation", + session=session, + ) return incident @@ -5989,4 +6005,4 @@ def recover_prev_alert_status(alert: Alert, session: Optional[Session] = None): ) ) session.exec(query) - session.commit() \ No newline at end of file + session.commit() diff --git a/keep/api/models/alert.py b/keep/api/models/alert.py index d6f5ac341a..97872c4aaf 100644 --- a/keep/api/models/alert.py +++ b/keep/api/models/alert.py @@ -8,7 +8,7 @@ from typing import TYPE_CHECKING, Any, Dict, Optional import pytz -from pydantic import AnyHttpUrl, BaseModel, Extra, root_validator, validator +from pydantic import AnyHttpUrl, BaseModel, Extra, Field, root_validator, validator from keep.api.models.severity_base import SeverityBaseInterface @@ -113,6 +113,11 @@ class AlertDto(BaseModel): enriched_fields: list = [] incident: str | None = None + keep_raw_payload: dict | None = Field( + default=None, + exclude=True, + description="In-memory only: full provider webhook for correlation enrichments; not persisted.", + ) def __str__(self) -> str: # Convert the model instance to a dictionary @@ -127,7 +132,7 @@ def __eq__(self, other): # Fields to exclude from comparison since they are bit different in different db's # todo: solve it in a better way - exclude_fields = {"lastReceived", "startedAt", "event_id"} + exclude_fields = {"lastReceived", "startedAt", "event_id", "keep_raw_payload"} # Remove excluded fields from both dictionaries for field in exclude_fields: diff --git a/keep/api/models/db/migrations/versions/2026-06-30-10-30_a38f1c29d7e4.py b/keep/api/models/db/migrations/versions/2026-06-30-10-30_a38f1c29d7e4.py new file mode 100644 index 0000000000..577a01593d --- /dev/null +++ b/keep/api/models/db/migrations/versions/2026-06-30-10-30_a38f1c29d7e4.py @@ -0,0 +1,28 @@ +"""feat: add incident enrichments to correlation rules + +Revision ID: a38f1c29d7e4 +Revises: 67ff7efffed4 +Create Date: 2026-06-30 10:30:00.000000 + +""" + +import sqlalchemy as sa +from alembic import op + +# revision identifiers, used by Alembic. +revision = "a38f1c29d7e4" +down_revision = "67ff7efffed4" +branch_labels = None +depends_on = None + + +def upgrade() -> None: + with op.batch_alter_table("rule", schema=None) as batch_op: + batch_op.add_column( + sa.Column("incident_enrichments", sa.JSON(), nullable=True) + ) + + +def downgrade() -> None: + with op.batch_alter_table("rule", schema=None) as batch_op: + batch_op.drop_column("incident_enrichments") diff --git a/keep/api/models/db/rule.py b/keep/api/models/db/rule.py index 2669571bf5..eadbfe2dc6 100644 --- a/keep/api/models/db/rule.py +++ b/keep/api/models/db/rule.py @@ -58,3 +58,4 @@ class Rule(SQLModel, table=True): multi_level_property_name: str | None = None threshold: int = Field(sa_column_args=(CheckConstraint("threshold>0"),), default=1) assignee: str | None = None + incident_enrichments: dict = Field(sa_column=Column(JSON), default_factory=dict) diff --git a/keep/api/routes/rules.py b/keep/api/routes/rules.py index 1dba243163..94f0a9045c 100644 --- a/keep/api/routes/rules.py +++ b/keep/api/routes/rules.py @@ -1,7 +1,7 @@ import logging from fastapi import APIRouter, Depends, HTTPException, Request -from pydantic import BaseModel +from pydantic import BaseModel, Field from keep.api.core.cel_to_sql.cel_ast_converter import CelToAstConverter from keep.api.core.db import create_rule as create_rule_db @@ -36,6 +36,7 @@ class RuleCreateDto(BaseModel): multiLevelPropertyName: str = None threshold: int = 1 assignee: str = None + incidentEnrichments: dict = Field(default_factory=dict) @router.get( @@ -96,6 +97,7 @@ async def create_rule( multi_level_property_name = rule_create_request.multiLevelPropertyName threshold = rule_create_request.threshold assignee = rule_create_request.assignee + incident_enrichments = rule_create_request.incidentEnrichments or {} if not sql: raise HTTPException(status_code=400, detail="SQL is required") @@ -147,6 +149,7 @@ async def create_rule( multi_level_property_name=multi_level_property_name, threshold=threshold, assignee=assignee, + incident_enrichments=incident_enrichments, ) logger.info("Rule created") return rule @@ -204,6 +207,7 @@ async def update_rule( multi_level_property_name = body.get("multiLevelPropertyName", None) threshold = body.get("threshold", 1) assignee = body.get("assignee", None) + incident_enrichments = body.get("incidentEnrichments", {}) or {} except Exception: raise HTTPException(status_code=400, detail="Invalid request body") @@ -261,6 +265,7 @@ async def update_rule( multi_level_property_name=multi_level_property_name, threshold=threshold, assignee=assignee, + incident_enrichments=incident_enrichments, ) if rule: diff --git a/keep/api/routes/settings.py b/keep/api/routes/settings.py index 54a40b4576..b204db1407 100644 --- a/keep/api/routes/settings.py +++ b/keep/api/routes/settings.py @@ -10,6 +10,7 @@ from pydantic import BaseModel, Field from sqlmodel import Session +from keep.api.consts import KEEP_STORE_RAW_ALERTS from keep.api.core.config import config from keep.api.core.db import get_session from keep.api.core.tenant_configuration import TenantConfiguration @@ -433,5 +434,6 @@ def get_tenant_configuration( ) -> dict: tenant_id = authenticated_entity.tenant_id tenant_configuration = TenantConfiguration() - config_value = tenant_configuration.get_configuration(tenant_id=tenant_id) + config_value = tenant_configuration.get_configuration(tenant_id=tenant_id) or {} + config_value["store_raw_alerts_enabled"] = KEEP_STORE_RAW_ALERTS return JSONResponse(status_code=200, content=config_value) diff --git a/keep/api/tasks/process_event_task.py b/keep/api/tasks/process_event_task.py index d74362e92b..d10499d97d 100644 --- a/keep/api/tasks/process_event_task.py +++ b/keep/api/tasks/process_event_task.py @@ -21,7 +21,12 @@ from keep.api.bl.enrichments_bl import EnrichmentsBl from keep.api.bl.incidents_bl import IncidentBl from keep.api.bl.maintenance_windows_bl import MaintenanceWindowsBl -from keep.api.consts import KEEP_CORRELATION_ENABLED, MAINTENANCE_WINDOW_ALERT_STRATEGY, fingerprints_for_poll_payload +from keep.api.consts import ( + KEEP_CORRELATION_ENABLED, + KEEP_STORE_RAW_ALERTS, + MAINTENANCE_WINDOW_ALERT_STRATEGY, + fingerprints_for_poll_payload, +) from keep.api.core.db import ( bulk_upsert_alert_fields, enrich_alerts_with_incidents, @@ -141,16 +146,17 @@ def __save_to_db( # keep raw events in the DB if the user wants to # this is mainly for debugging and research purposes if KEEP_STORE_RAW_ALERTS: - if isinstance(raw_events, dict): - raw_events = [raw_events] + events_to_store = raw_events + if isinstance(events_to_store, dict): + events_to_store = [events_to_store] - for raw_event in raw_events: - alert = AlertRaw( + for raw_event in events_to_store: + alert_raw = AlertRaw( tenant_id=tenant_id, raw_alert=raw_event, provider_type=provider_type, ) - session.add(alert) + session.add(alert_raw) enrichments_bl = EnrichmentsBl(tenant_id, session) # add audit to the deduplicated events @@ -185,7 +191,7 @@ def __save_to_db( tenant_id, fingerprints, session=session ) - for formatted_event in formatted_events: + for alert_index, formatted_event in enumerate(formatted_events): formatted_event.pushed = True started_at = started_at_for_fingerprints.get( @@ -555,6 +561,11 @@ def __handle_formatted_events( }, ) + if KEEP_STORE_RAW_ALERTS and raw_events: + stored_raw = raw_events[0] if isinstance(raw_events, list) else raw_events + for alert in enriched_formatted_events: + alert.keep_raw_payload = copy.deepcopy(stored_raw) + incidents = [] with tracer.start_as_current_span("process_event_run_rules_engine"): # Now we need to run the rules engine diff --git a/keep/rulesengine/rulesengine.py b/keep/rulesengine/rulesengine.py index 96c01996f3..bf1388019d 100644 --- a/keep/rulesengine/rulesengine.py +++ b/keep/rulesengine/rulesengine.py @@ -28,6 +28,7 @@ from keep.api.models.db.rule import Rule from keep.api.models.incident import IncidentDto from keep.api.utils.cel_utils import preprocess_cel_expression +from keep.api.consts import KEEP_STORE_RAW_ALERTS from keep.api.utils.enrichment_helpers import convert_db_alerts_to_dto_alerts # Shahar: this is performance enhancment https://github.com/cloud-custodian/cel-python/issues/68 @@ -224,12 +225,18 @@ def _run_cel_rules( return list(incidents_dto.values()) - def get_value_from_event(self, event: AlertDto, var: str) -> str: + def get_value_from_event( + self, event: AlertDto, var: str, raw_payload: dict | None = None + ) -> str: """ Extract value from event based on template variable e.g., alert.labels.host -> event['labels']['host'] alert.service -> event['service'] """ + var = var.strip() + if var.startswith("raw.") or "[" in var: + return self._resolve_template_value(event, var, raw_payload) + # Remove 'alert.' prefix path = var.replace("alert.", "").split(".") @@ -238,13 +245,19 @@ def get_value_from_event(self, event: AlertDto, var: str) -> str: for part in path: part = part.strip() current = current.get(part) - return str(current) if current is not None else "N/A" + value = str(current) if current is not None else "N/A" except (KeyError, AttributeError): - return "N/A" + value = "N/A" + + if value != "N/A": + return value + if KEEP_STORE_RAW_ALERTS: + return self._resolve_template_value(event, var, raw_payload) + return "N/A" def get_vaiables(self, incident_name_template): regex = r"\{\{\s*([^}]+)\s*\}\}" - return re.findall(regex, incident_name_template) + return [var.strip() for var in re.findall(regex, incident_name_template)] def _get_or_create_incident( self, rule: Rule, rule_fingerprint, session, event, creation_allowed=True @@ -257,6 +270,7 @@ def _get_or_create_incident( session=session, ) + if existed_incident and not expired and rule.incident_prefix: if rule.incident_prefix not in existed_incident.user_generated_name: existed_incident.user_generated_name = f"{rule.incident_prefix}-{existed_incident.running_number} - {existed_incident.user_generated_name}" @@ -346,6 +360,9 @@ def _get_or_create_incident( else f"{rule_fingerprint} - {incident_name}" ) + incident_enrichments = ( + self._enrich_incident(rule, event) if KEEP_STORE_RAW_ALERTS else {} + ) incident = create_incident_for_grouping_rule( tenant_id=self.tenant_id, rule=rule, @@ -354,8 +371,10 @@ def _get_or_create_incident( incident_name=incident_name, past_incident=existed_incident, assignee=rule.assignee, + enrichments=incident_enrichments, ) - return incident, True + if incident: + return incident, True return None, False def _process_event_for_history_based_rule( @@ -753,6 +772,122 @@ def filter_alerts( return filtered_alerts + @staticmethod + def _parse_template_path(path: str) -> list[str]: + """Split a template path into steps. Supports dots and brackets, e.g. data[0].model -> ['data', '0', 'model'].""" + parts: list[str] = [] + current = "" + index = 0 + while index < len(path): + char = path[index] + if char == ".": + if current: + parts.append(current) + current = "" + index += 1 + continue + if char == "[": + if current: + parts.append(current) + current = "" + closing = path.find("]", index + 1) + if closing == -1: + parts.append(path[index:]) + break + parts.append(path[index + 1 : closing].strip()) + index = closing + 1 + continue + current += char + index += 1 + if current: + parts.append(current) + return [part.strip() for part in parts if part.strip()] + + def _resolve_path(self, data, path_parts: list[str]) -> str: + """Walk a dict/list by path_parts and return the value as a string, or 'N/A' if missing.""" + current = data + for part in path_parts: + part = part.strip() + if current is None: + return "N/A" + if isinstance(current, dict): + current = current.get(part) + elif isinstance(current, list): + try: + current = current[int(part)] + except (ValueError, IndexError, TypeError): + return "N/A" + else: + return "N/A" + if current is None: + return "N/A" + if isinstance(current, (dict, list)): + return json.dumps(current) + return str(current) + + def _load_raw_payload(self, event: AlertDto) -> dict | None: + """Raw webhook is only available in-memory during ingestion (keep_raw_payload).""" + return None + + def _resolve_raw_payload(self, event: AlertDto) -> dict | None: + """Get raw payload from in-memory keep_raw_payload set during ingestion.""" + if isinstance(event.keep_raw_payload, dict): + return event.keep_raw_payload + return self._load_raw_payload(event) + + def _resolve_template_value( + self, event: AlertDto, var: str, raw_payload: dict | None = None + ) -> str: + """ + Resolve a {{ var }} placeholder that uses raw.* paths or bracket notation. + Tries formatted alert first, then raw payload fallback. + """ + var = var.strip() + if var.startswith("raw."): + if raw_payload is None: + raw_payload = self._resolve_raw_payload(event) + if raw_payload is None: + return "N/A" + path = self._parse_template_path(var.replace("raw.", "", 1)) + return self._resolve_path(raw_payload, path) + + path_str = var.replace("alert.", "", 1) if var.startswith("alert.") else var + path = self._parse_template_path(path_str) + alert_value = self._resolve_path(event.dict(), path) + if alert_value != "N/A": + return alert_value + + if raw_payload is None: + raw_payload = self._resolve_raw_payload(event) + if raw_payload is None: + return "N/A" + return self._resolve_path(raw_payload, path) + + def _enrich_incident(self, rule: Rule, event: AlertDto) -> dict: + """Render rule.incident_enrichments templates into key/value pairs for a new incident.""" + templates = rule.incident_enrichments or {} + if not isinstance(templates, dict) or not templates: + return {} + + raw_payload = self._resolve_raw_payload(event) + enrichments: dict = {} + for key, template in templates.items(): + if not key: + continue + if not isinstance(template, str): + enrichments[key] = template + continue + rendered = template + for var in self.get_vaiables(template): + resolved = self.get_value_from_event( + event, var, raw_payload=raw_payload + ) + rendered = re.sub( + r"\{\{\s*" + re.escape(var) + r"\s*\}\}", resolved, rendered + ) + enrichments[key] = rendered + return enrichments + @staticmethod def send_workflow_event( tenant_id: str, session: Session, incident_dto: IncidentDto, action: str