diff --git a/external-import/metras/.dockerignore b/external-import/metras/.dockerignore new file mode 100644 index 0000000000..33cdfff009 --- /dev/null +++ b/external-import/metras/.dockerignore @@ -0,0 +1,11 @@ +__metadata__ +**/__pycache__ +**/__docs__ +**/.venv +**/venv +**/logs +**/config.yml +**/*.egg-info +**/*.gql +.plan +tests diff --git a/external-import/metras/Dockerfile b/external-import/metras/Dockerfile new file mode 100644 index 0000000000..7d366ae2b0 --- /dev/null +++ b/external-import/metras/Dockerfile @@ -0,0 +1,22 @@ +FROM python:3.12-alpine + +ENV CONNECTOR_TYPE=EXTERNAL_IMPORT +ENV PYTHONDONTWRITEBYTECODE=1 +ENV PYTHONUNBUFFERED=1 + +COPY src /opt/opencti-connector-metras-feed + +RUN apk update && apk upgrade --no-cache && \ + apk --no-cache add libmagic libffi libxml2 libxslt && \ + apk --no-cache add --virtual .build-deps git build-base libffi-dev libxml2-dev libxslt-dev && \ + cd /opt/opencti-connector-metras-feed && \ + pip3 install --no-cache-dir -r requirements.txt && \ + apk del .build-deps + +COPY entrypoint.sh / +RUN chmod +x /entrypoint.sh && \ + adduser -D -u 1000 connector && \ + chown -R connector:connector /opt/opencti-connector-metras-feed +USER connector + +ENTRYPOINT ["/entrypoint.sh"] diff --git a/external-import/metras/README.md b/external-import/metras/README.md new file mode 100644 index 0000000000..c63333bf18 --- /dev/null +++ b/external-import/metras/README.md @@ -0,0 +1,70 @@ +# OpenCTI Metras Feed Connector (EXTERNAL_IMPORT) + +Imports telemetry from the [Metras](https://dashboard.metras.sa/) security platform +(`api.metras.sa`) into OpenCTI on a schedule: EDR alerts (with MITRE ATT&CK mapping), +file binaries, and endpoint inventory. + +## What it imports + +| Metras source | OpenCTI output | +|---|---| +| EDR alerts (`/v1/edr/alerts`) | **Incident** (name, severity, score, labels) | +| └ `mitre_ids` | **Attack-Pattern** (keyed by `external_id`, merges with MITRE ATT&CK) + `uses` relationship | +| └ `url` | **Url** observable + `related-to` (external destination — a real IOC) | +| └ `endpoint_name` (+ `agent_ip`) | **Identity** (`identity_class: system`) + `related-to` | +| Binaries (`/v1/edr/binary/list`) | **StixFile** (MD5 / SHA-1 / SHA-256, name, size, score) + `related-to` System | +| Endpoints (`/v1/endpoints`) | **Identity** (`identity_class: system`); interface/tunnel IPs in the description | + +> **Internal assets are System identities, not IOCs** — your fleet endpoints (and their internal +> IPs: `agent_ip`, interface/tunnel IPs) are modelled as `Identity(identity_class="system")` with the +> IPs in the description, **not** as IPv4-Addr observables. Only genuinely external artifacts (alert +> `url`, file hashes) become observables. (Pattern per OpenCTI maintainer review, PR #6164.) +> +> **Observables only** — no STIX Indicators are auto-created from binaries (analysts promote manually). +> Process name/GUID from alerts is folded into the Incident description (STIX 2.1 `Process` has no `name`). + +## Incremental behavior +- **EDR alerts** have no time filter on the API → the connector filters client-side on + `last_occurrence_time` against stored state (`alerts_last_occurrence`). +- **Binaries** use the server-side `fromTime` window (state `binaries_last_seen`). +- **Endpoints** are re-listed each run (deduped by deterministic STIX IDs in OpenCTI). +- A category that errors does not advance its cursor (no data loss; safe retry next cycle). + +## Requirements +- OpenCTI **7.260529.0** (pinned; `pycti==7.260529.0`). +- A Metras API key (`X-API-KEY`). + +## Configuration + +| Env var | Required | Default | Description | +|---|---|---|---| +| `OPENCTI_URL` | yes | — | OpenCTI base URL | +| `OPENCTI_TOKEN` | yes | — | OpenCTI API token | +| `CONNECTOR_ID` | yes | — | UUIDv4 for this connector | +| `CONNECTOR_NAME` | no | `Metras-Feed` | Connector name | +| `CONNECTOR_SCOPE` | no | `Metras` | Import scope | +| `CONNECTOR_LOG_LEVEL` | no | `info` | Log level | +| `CONNECTOR_DURATION_PERIOD` | yes | `PT1H` | ISO-8601 poll interval | +| `METRAS_API_BASE_URL` | no | `https://api.metras.sa/api` | Metras API base URL | +| `METRAS_API_KEY` | yes | — | Metras API key | +| `METRAS_VERIFY_SSL` | no | `true` | Verify TLS certificates | +| `METRAS_IMPORT_ALERTS` | no | `true` | Import EDR alerts | +| `METRAS_IMPORT_BINARIES` | no | `true` | Import binaries | +| `METRAS_IMPORT_ENDPOINTS` | no | `true` | Import endpoints | +| `METRAS_BINARY_MALICIOUS_ONLY` | no | `true` | Only import banned/unsigned binaries | +| `METRAS_PAGE_SIZE` | no | `50` | Records per page | +| `METRAS_TLP_LEVEL` | no | `amber` | TLP marking (`clear`/`white`/`green`/`amber`/`red`) | + +## Installation +```bash +cp config.yml.sample src/config.yml # or use env vars / docker-compose +docker compose up -d --build +``` + +## Troubleshooting +| Symptom | Cause / fix | +|---|---| +| `Metras API ping failed at startup` then exit | Bad `METRAS_API_KEY` or unreachable base URL | +| `Imported 0 incidents` after first run | Normal — alerts already imported; cursor advanced | +| `Cannot query field "s3"` | pycti/platform mismatch — keep `pycti==7.260529.0` | +| No Works in OpenCTI UI | Check `OPENCTI_TOKEN` permissions (Work API) | diff --git a/external-import/metras/VERSION b/external-import/metras/VERSION new file mode 100644 index 0000000000..3eefcb9dd5 --- /dev/null +++ b/external-import/metras/VERSION @@ -0,0 +1 @@ +1.0.0 diff --git a/external-import/metras/__metadata__/connector_config_schema.json b/external-import/metras/__metadata__/connector_config_schema.json new file mode 100644 index 0000000000..0e6b70c1ce --- /dev/null +++ b/external-import/metras/__metadata__/connector_config_schema.json @@ -0,0 +1,149 @@ +{ + "$schema": "https://json-schema.org/draft/2020-12/schema", + "$id": "https://www.filigran.io/connectors/w_config.schema.json", + "type": "object", + "properties": { + "OPENCTI_URL": { + "description": "The base URL of the OpenCTI instance.", + "format": "uri", + "maxLength": 2083, + "minLength": 1, + "type": "string" + }, + "OPENCTI_TOKEN": { + "description": "The API token to connect to OpenCTI.", + "type": "string" + }, + "CONNECTOR_NAME": { + "default": "Metras-Feed", + "examples": [ + "Metras-Feed" + ], + "type": "string" + }, + "CONNECTOR_SCOPE": { + "description": "The scope of the connector, e.g. 'flashpoint'.", + "items": { + "type": "string" + }, + "type": "array" + }, + "CONNECTOR_LOG_LEVEL": { + "default": "error", + "description": "The minimum level of logs to display.", + "enum": [ + "debug", + "info", + "warn", + "warning", + "error" + ], + "type": "string" + }, + "CONNECTOR_TYPE": { + "const": "EXTERNAL_IMPORT", + "default": "EXTERNAL_IMPORT", + "type": "string" + }, + "CONNECTOR_DURATION_PERIOD": { + "default": "PT1H", + "examples": [ + "PT1H" + ], + "format": "duration", + "type": "string" + }, + "METRAS_API_BASE_URL": { + "default": "https://api.metras.sa/api", + "description": "Base URL of the Metras API.", + "examples": [ + "https://api.metras.sa/api" + ], + "format": "uri", + "maxLength": 2083, + "minLength": 1, + "type": "string" + }, + "METRAS_API_KEY": { + "description": "Metras API key (X-API-KEY header).", + "examples": [ + "ChangeMe" + ], + "format": "password", + "type": "string", + "writeOnly": true + }, + "METRAS_VERIFY_SSL": { + "default": true, + "description": "Verify TLS certificates.", + "examples": [ + true + ], + "type": "boolean" + }, + "METRAS_IMPORT_ALERTS": { + "default": true, + "description": "Import EDR alerts.", + "examples": [ + true + ], + "type": "boolean" + }, + "METRAS_IMPORT_BINARIES": { + "default": true, + "description": "Import binaries as StixFile.", + "examples": [ + true + ], + "type": "boolean" + }, + "METRAS_IMPORT_ENDPOINTS": { + "default": true, + "description": "Import endpoints as System identities.", + "examples": [ + true + ], + "type": "boolean" + }, + "METRAS_BINARY_MALICIOUS_ONLY": { + "default": true, + "description": "Only import banned/unsigned binaries (reduces noise).", + "examples": [ + true + ], + "type": "boolean" + }, + "METRAS_PAGE_SIZE": { + "default": 50, + "description": "Records per page.", + "examples": [ + 50 + ], + "maximum": 500, + "minimum": 1, + "type": "integer" + }, + "METRAS_TLP_LEVEL": { + "default": "amber", + "description": "TLP marking applied to imported objects.", + "enum": [ + "clear", + "white", + "green", + "amber", + "red" + ], + "examples": [ + "amber" + ], + "type": "string" + } + }, + "required": [ + "OPENCTI_URL", + "OPENCTI_TOKEN", + "CONNECTOR_SCOPE", + "METRAS_API_KEY" + ], + "additionalProperties": true +} \ No newline at end of file diff --git a/external-import/metras/__metadata__/connector_manifest.json b/external-import/metras/__metadata__/connector_manifest.json new file mode 100644 index 0000000000..70ea962ddd --- /dev/null +++ b/external-import/metras/__metadata__/connector_manifest.json @@ -0,0 +1,19 @@ +{ + "title": "Metras Feed Connector", + "slug": "metras-feed", + "description": "Imports Metras EDR alerts (with MITRE ATT&CK mapping), file binaries, and endpoint inventory into OpenCTI as STIX incidents, observables and system identities.", + "short_description": "External import connector for Metras EDR telemetry", + "logo": null, + "use_cases": ["Threat Intelligence"], + "verified": false, + "last_verified_date": null, + "playbook_supported": false, + "max_confidence_level": 50, + "support_version": ">=7.260529.0", + "subscription_link": null, + "source_code": "https://github.com/OpenCTI-Platform/connectors/tree/master/external-import/metras", + "manager_supported": false, + "container_version": "1.0.0", + "container_image": "opencti/connector-metras-feed", + "container_type": "EXTERNAL_IMPORT" +} diff --git a/external-import/metras/config.yml.sample b/external-import/metras/config.yml.sample new file mode 100644 index 0000000000..bbbd3ce6c4 --- /dev/null +++ b/external-import/metras/config.yml.sample @@ -0,0 +1,22 @@ +opencti: + url: 'http://localhost:8080' + token: 'ChangeMe' + +connector: + type: 'EXTERNAL_IMPORT' + id: 'ChangeMe' # UUIDv4 + name: 'Metras-Feed' + scope: 'Metras' + log_level: 'info' + duration_period: 'PT1H' # ISO-8601: poll interval + +metras: + api_base_url: 'https://api.metras.sa/api' + api_key: 'ChangeMe' + verify_ssl: true + import_alerts: true + import_binaries: true + import_endpoints: true + binary_malicious_only: true # only banned/unsigned binaries + page_size: 50 + tlp_level: 'amber' diff --git a/external-import/metras/docker-compose.yml b/external-import/metras/docker-compose.yml new file mode 100644 index 0000000000..257f6c5e3b --- /dev/null +++ b/external-import/metras/docker-compose.yml @@ -0,0 +1,34 @@ +services: + connector-metras-feed: + build: + context: . + dockerfile: Dockerfile + image: opencti/connector-metras-feed:1.0.0 + environment: + - OPENCTI_URL=http://opencti:8080 + - OPENCTI_TOKEN=${OPENCTI_ADMIN_TOKEN} + - CONNECTOR_ID=${CONNECTOR_METRAS_FEED_ID} + - CONNECTOR_TYPE=EXTERNAL_IMPORT + - CONNECTOR_NAME=Metras-Feed + - CONNECTOR_SCOPE=Metras + - CONNECTOR_LOG_LEVEL=info + - CONNECTOR_DURATION_PERIOD=PT1H + - METRAS_API_BASE_URL=https://api.metras.sa/api + - METRAS_API_KEY=${METRAS_API_KEY} + - METRAS_VERIFY_SSL=true + - METRAS_IMPORT_ALERTS=true + - METRAS_IMPORT_BINARIES=true + - METRAS_IMPORT_ENDPOINTS=true + - METRAS_BINARY_MALICIOUS_ONLY=true + - METRAS_PAGE_SIZE=50 + - METRAS_TLP_LEVEL=amber + restart: always + cap_drop: + - ALL + security_opt: + - no-new-privileges:true + read_only: true + tmpfs: + - /tmp + mem_limit: 512m + pids_limit: 100 diff --git a/external-import/metras/entrypoint.sh b/external-import/metras/entrypoint.sh new file mode 100644 index 0000000000..e69a624342 --- /dev/null +++ b/external-import/metras/entrypoint.sh @@ -0,0 +1,3 @@ +#!/bin/sh +cd /opt/opencti-connector-metras-feed +exec python3 main.py diff --git a/external-import/metras/pyproject.toml b/external-import/metras/pyproject.toml new file mode 100644 index 0000000000..1a5498cf32 --- /dev/null +++ b/external-import/metras/pyproject.toml @@ -0,0 +1,5 @@ +[tool.coverage.run] +source = ["connector", "metras_client"] + +[tool.coverage.report] +show_missing = true diff --git a/external-import/metras/src/connector/__init__.py b/external-import/metras/src/connector/__init__.py new file mode 100644 index 0000000000..4864558736 --- /dev/null +++ b/external-import/metras/src/connector/__init__.py @@ -0,0 +1,4 @@ +from connector.connector import MetrasFeedConnector +from connector.settings import ConnectorSettings + +__all__ = ["MetrasFeedConnector", "ConnectorSettings"] diff --git a/external-import/metras/src/connector/connector.py b/external-import/metras/src/connector/connector.py new file mode 100644 index 0000000000..b15e92cddf --- /dev/null +++ b/external-import/metras/src/connector/connector.py @@ -0,0 +1,207 @@ +"""Metras Feed connector (EXTERNAL_IMPORT). + +Polls EDR alerts, binaries and endpoints from Metras and imports them into +OpenCTI as STIX. Incremental: alerts filter client-side on last_occurrence_time +(no fromTime param), binaries use server-side fromTime windowing. +""" + +import sys +import time +from datetime import datetime, timedelta, timezone +from typing import TYPE_CHECKING + +from connector.converter_to_stix import ConverterToStix +from connector.utils import is_newer_than, normalize_timestamp, stix_timestamp +from metras_client import MetrasAPIError, MetrasClient +from pycti import OpenCTIConnectorHelper + +if TYPE_CHECKING: + from connector.settings import ConnectorSettings + + +class MetrasFeedConnector: + def __init__( + self, config: "ConnectorSettings", helper: OpenCTIConnectorHelper + ) -> None: + self.config = config + self.helper = helper + cfg = config.metras + self.client = MetrasClient( + helper=helper, + base_url=str(cfg.api_base_url), + api_key=cfg.api_key.get_secret_value(), + verify_ssl=cfg.verify_ssl, + ) + self.converter = ConverterToStix(helper, tlp_level=cfg.tlp_level) + self.cfg = cfg + self._interval = self._duration_seconds(config.connector.duration_period) + + @staticmethod + def _duration_seconds(duration: timedelta | str) -> int: + """Resolve CONNECTOR_DURATION_PERIOD (timedelta or ISO8601 string) to seconds.""" + if hasattr(duration, "total_seconds"): + return max(60, int(duration.total_seconds())) + # Minimal ISO8601 duration parse (PT#H#M#S / P#D) as a fallback. + import re + + text = str(duration or "PT1H").upper() + match = re.fullmatch( + r"P(?:(\d+)D)?(?:T(?:(\d+)H)?(?:(\d+)M)?(?:(\d+)S)?)?", text + ) + if not match: + return 3600 + days, hours, minutes, seconds = (int(g or 0) for g in match.groups()) + total = days * 86400 + hours * 3600 + minutes * 60 + seconds + return max(60, total or 3600) + + # ------------------------------------------------------------------ # + def run(self) -> None: + try: + self.client.ping() + self.helper.connector_logger.info( + "[CONNECTOR] Metras API connection verified" + ) + except MetrasAPIError as exc: + self.helper.connector_logger.error( + "[CONNECTOR] Metras API ping failed at startup", {"error": str(exc)} + ) + sys.exit(1) + + self.helper.connector_logger.info( + "[CONNECTOR] Starting Metras Feed import loop", + {"interval_seconds": self._interval}, + ) + while True: + try: + self._import_data() + except (KeyboardInterrupt, SystemExit): + self.helper.connector_logger.info("[CONNECTOR] Stopping") + break + except Exception as exc: # noqa: BLE001 - keep the loop alive + self.helper.connector_logger.error( + "[CONNECTOR] Import cycle crashed", {"error": str(exc)} + ) + time.sleep(self._interval) + + # ------------------------------------------------------------------ # + def _import_data(self) -> None: + state = self.helper.get_state() or {} + alerts_cursor = normalize_timestamp(state.get("alerts_last_occurrence")) + binaries_cursor = state.get("binaries_last_seen") # ISO string for fromTime + + now_iso = stix_timestamp(datetime.now(timezone.utc)) + friendly = f"Metras Feed import @ {now_iso}" + work_id = self.helper.api.work.initiate_work(self.helper.connect_id, friendly) + + all_objects = [self.converter.author_object()] + new_alerts_max = alerts_cursor + new_binaries_max = binaries_cursor + counts = {"alerts": 0, "binaries": 0, "endpoints": 0} + errors = [] + + # --- EDR alerts (client-side incremental) --- + if self.cfg.import_alerts: + try: + for alert in self.client.iter_edr_alerts(page_size=self.cfg.page_size): + ts = alert.get("last_occurrence_time") + if not is_newer_than(ts, alerts_cursor): + continue + objs = self.converter.process_alert(alert) + if objs: + all_objects.extend(objs) + counts["alerts"] += 1 + parsed = normalize_timestamp(ts) + if parsed and (new_alerts_max is None or parsed > new_alerts_max): + new_alerts_max = parsed + self.helper.connector_logger.info( + "[CONNECTOR] Alerts processed", {"new_incidents": counts["alerts"]} + ) + except MetrasAPIError as exc: + errors.append(f"alerts: {exc}") + self.helper.connector_logger.error( + "[CONNECTOR] Alert import failed", {"error": str(exc)} + ) + + # --- Binaries (server-side fromTime window) --- + if self.cfg.import_binaries: + try: + for binary in self.client.iter_binaries( + from_time=binaries_cursor, page_size=self.cfg.page_size + ): + objs = self.converter.process_binary( + binary, malicious_only=self.cfg.binary_malicious_only + ) + if objs: + all_objects.extend(objs) + counts["binaries"] += 1 + last_seen = binary.get("last_seen") + if last_seen and ( + new_binaries_max is None or last_seen > new_binaries_max + ): + new_binaries_max = last_seen + self.helper.connector_logger.info( + "[CONNECTOR] Binaries processed", {"new_files": counts["binaries"]} + ) + except MetrasAPIError as exc: + errors.append(f"binaries: {exc}") + self.helper.connector_logger.error( + "[CONNECTOR] Binary import failed", {"error": str(exc)} + ) + + # --- Endpoints (full inventory each run) --- + if self.cfg.import_endpoints: + try: + payload = self.client.list_endpoints() + for endpoint in payload.get("endpoints") or []: + objs = self.converter.process_endpoint(endpoint) + if objs: + all_objects.extend(objs) + counts["endpoints"] += 1 + self.helper.connector_logger.info( + "[CONNECTOR] Endpoints processed", + {"endpoints": counts["endpoints"]}, + ) + except MetrasAPIError as exc: + errors.append(f"endpoints: {exc}") + self.helper.connector_logger.error( + "[CONNECTOR] Endpoint import failed", {"error": str(exc)} + ) + + total = counts["alerts"] + counts["binaries"] + counts["endpoints"] + + # Total failure: every enabled category errored and nothing produced. + if errors and total == 0: + msg = "Metras import failed: " + "; ".join(errors) + self.helper.api.work.to_processed(work_id, msg, in_error=True) + self.helper.connector_logger.error( + "[CONNECTOR] Import cycle failed", {"msg": msg} + ) + return + + if len(all_objects) > 1: # more than just the author + bundle = self.helper.stix2_create_bundle(all_objects) + self.helper.send_stix2_bundle( + bundle, work_id=work_id, cleanup_inconsistent_bundle=True + ) + msg = ( + f"Imported {counts['alerts']} incidents, {counts['binaries']} files, " + f"{counts['endpoints']} endpoints ({len(all_objects)} STIX objects)" + ) + else: + msg = "No new Metras data to import" + + if errors: + msg += " | partial errors: " + "; ".join(errors) + self.helper.api.work.to_processed(work_id, msg) + self.helper.connector_logger.info( + "[CONNECTOR] Import cycle done", {"summary": msg} + ) + + # Advance cursors only for categories that did not hard-fail. + new_state = dict(state) + if "alerts" not in "".join(errors) and new_alerts_max is not None: + new_state["alerts_last_occurrence"] = stix_timestamp(new_alerts_max) + if "binaries" not in "".join(errors) and new_binaries_max: + new_state["binaries_last_seen"] = new_binaries_max + new_state["last_run"] = now_iso + self.helper.set_state(new_state) diff --git a/external-import/metras/src/connector/converter_to_stix.py b/external-import/metras/src/connector/converter_to_stix.py new file mode 100644 index 0000000000..bbedfbd66f --- /dev/null +++ b/external-import/metras/src/connector/converter_to_stix.py @@ -0,0 +1,386 @@ +"""STIX conversion for the Metras Feed (EXTERNAL_IMPORT). + +Bulk converters: EDR alerts -> Incident (+ Attack-Pattern from mitre_ids, external Url, +and a System identity for the affected endpoint); binaries -> StixFile observables; +endpoints -> System identities. Internal asset IPs (agent_ip, interface IPs) are recorded +on the System identity description, NOT emitted as IPv4-Addr IOCs. Observables only — no +Indicators are auto-created (per build decision). +""" + +from typing import TYPE_CHECKING + +import stix2 +from connector.utils import ( + is_mitre_attack_id, + is_valid_hash, + is_valid_url, + severity_to_score, + stix_timestamp, +) +from pycti import ( + AttackPattern, + Identity, + Incident, + StixCoreRelationship, +) + +if TYPE_CHECKING: + from pycti import OpenCTIConnectorHelper + +_TLP_BY_NAME = { + "clear": stix2.TLP_WHITE, + "white": stix2.TLP_WHITE, + "green": stix2.TLP_GREEN, + "amber": stix2.TLP_AMBER, + "red": stix2.TLP_RED, +} + + +class ConverterToStix: + """Factory for all STIX objects produced by the Feed connector.""" + + def __init__( + self, helper: "OpenCTIConnectorHelper", tlp_level: str = "amber" + ) -> None: + self.helper = helper + self.tlp = _TLP_BY_NAME.get((tlp_level or "amber").lower(), stix2.TLP_AMBER) + self.author = self._create_author() + self._confidence = getattr(helper, "connect_confidence_level", None) or 50 + + # ------------------------------------------------------------------ # + # Author / common + # ------------------------------------------------------------------ # + @staticmethod + def _create_author() -> stix2.Identity: + return stix2.Identity( + id=Identity.generate_id(name="Metras", identity_class="organization"), + name="Metras", + identity_class="organization", + description="Metras endpoint detection & response (EDR) platform.", + external_references=[ + stix2.ExternalReference( + source_name="Metras", url="https://dashboard.metras.sa/" + ) + ], + ) + + def _marking_refs(self) -> list[str]: + return [self.tlp["id"]] + + def create_relationship( + self, source_id: str, rel_type: str, target_id: str, **kwargs + ) -> stix2.Relationship: + return stix2.Relationship( + id=StixCoreRelationship.generate_id(rel_type, source_id, target_id), + relationship_type=rel_type, + source_ref=source_id, + target_ref=target_id, + created_by_ref=self.author["id"], + object_marking_refs=self._marking_refs(), + confidence=self._confidence, + allow_custom=True, + **kwargs, + ) + + # ------------------------------------------------------------------ # + # Observables + # ------------------------------------------------------------------ # + def _custom_props(self, score: int | None = None) -> dict: + props = {"x_opencti_created_by_ref": self.author["id"]} + if score is not None: + props["x_opencti_score"] = score + return props + + def create_url(self, value: str, score: int | None = None) -> stix2.URL | None: + if not is_valid_url(value): + return None + return stix2.URL( + value=value, + object_marking_refs=self._marking_refs(), + custom_properties=self._custom_props(score), + ) + + def create_file( + self, + *, + md5: str | None = None, + sha1: str | None = None, + sha256: str | None = None, + name: str | None = None, + size: int | None = None, + score: int | None = None, + ) -> stix2.File | None: + # Only keep well-formed hashes — a malformed digest from the API would make + # stix2.File raise and crash the whole import cycle. + hashes = { + algo: val + for algo, val in (("MD5", md5), ("SHA-1", sha1), ("SHA-256", sha256)) + if is_valid_hash(algo, val) + } + if not hashes and not name: + return None + kwargs = { + "object_marking_refs": self._marking_refs(), + "custom_properties": self._custom_props(score), + } + if hashes: + kwargs["hashes"] = hashes + if name: + kwargs["name"] = name + if isinstance(size, int) and size >= 0: + kwargs["size"] = size + try: + return stix2.File(**kwargs) + except Exception: # noqa: BLE001 - never let one bad record kill the cycle + return None + + # ------------------------------------------------------------------ # + # Domain objects + # ------------------------------------------------------------------ # + def create_attack_pattern(self, mitre_id: str) -> stix2.AttackPattern | None: + if not is_mitre_attack_id(mitre_id): + return None + return stix2.AttackPattern( + id=AttackPattern.generate_id(name=mitre_id, x_mitre_id=mitre_id), + name=mitre_id, + created_by_ref=self.author["id"], + object_marking_refs=self._marking_refs(), + custom_properties={"x_mitre_id": mitre_id}, + external_references=[ + stix2.ExternalReference( + source_name="mitre-attack", + external_id=mitre_id, + url=f"https://attack.mitre.org/techniques/{mitre_id.replace('.', '/')}/", + ) + ], + allow_custom=True, + ) + + def create_system( + self, name: str, *, description: str | None = None + ) -> stix2.Identity | None: + """A fleet endpoint/host modelled as a System identity (an internal asset, + not an IOC). Internal asset IPs are kept in the description, not emitted as + IPv4-Addr observables.""" + if not name: + return None + return stix2.Identity( + id=Identity.generate_id(name=name, identity_class="system"), + name=name, + identity_class="system", + description=description, + created_by_ref=self.author["id"], + object_marking_refs=self._marking_refs(), + confidence=self._confidence, + allow_custom=True, + ) + + def create_incident( + self, + *, + name: str, + description: str, + severity: int | float | str | None, + first_seen: str | None = None, + last_seen: str | None = None, + source: str | None = None, + labels: list[str] | None = None, + external_id: str | int | None = None, + ) -> stix2.Incident: + score, octi_sev = severity_to_score(severity) + refs = [] + if external_id: + refs.append( + stix2.ExternalReference( + source_name="Metras", external_id=str(external_id) + ) + ) + # Seed the deterministic Incident id on the stable Metras alert id so a + # recurring alert (whose last_occurrence_time advances) updates the same + # Incident instead of creating a duplicate. The fixed seed-time keeps the id + # independent of the moving occurrence time; first_seen/last_seen still carry + # the real timestamps. Fall back to name+created when no stable id exists. + if external_id: + incident_id = Incident.generate_id( + name=f"metras-alert-{external_id}", created="1970-01-01T00:00:00Z" + ) + else: + incident_id = Incident.generate_id( + name=name, created=first_seen or stix_timestamp() + ) + return stix2.Incident( + id=incident_id, + name=name, + description=description, + created_by_ref=self.author["id"], + object_marking_refs=self._marking_refs(), + labels=labels or [], + first_seen=first_seen, + last_seen=last_seen, + confidence=self._confidence, + external_references=refs or None, + custom_properties={ + "x_opencti_score": score, + "source": source or "Metras", + "severity": octi_sev, + "incident_type": "alert", + }, + allow_custom=True, + ) + + # ------------------------------------------------------------------ # + # High-level builders (record -> list[STIX]) + # ------------------------------------------------------------------ # + def process_alert(self, alert: dict) -> list: + """Convert one EDR alert into a connected STIX graph.""" + objects = [] + name = ( + alert.get("alert_name") + or alert.get("alert_source_name") + or "Metras EDR Alert" + ) + endpoint_name = alert.get("endpoint_name") + proc = alert.get("process") or {} + proc_name = proc.get("name") + proc_guid = proc.get("guid") + + first_seen = alert.get("last_occurrence_time") + desc_lines = [ + f"Metras EDR alert: {name}", + f"Type: {alert.get('type', 'n/a')}", + f"Source: {alert.get('alert_source_name', 'n/a')}", + f"Endpoint: {endpoint_name or 'n/a'} ({alert.get('endpoint_id', 'n/a')})", + f"Occurrences: {alert.get('occurrence_count', 'n/a')}", + f"Risk score: {alert.get('risk_score', 'n/a')}", + ] + if proc_name: + desc_lines.append(f"Process: {proc_name} {proc_guid or ''}".strip()) + if alert.get("mitre_ids"): + desc_lines.append(f"MITRE ATT&CK: {', '.join(alert['mitre_ids'])}") + + labels = [] + for lbl in alert.get("tags") or []: + if lbl: + labels.append(str(lbl)) + if alert.get("type"): + labels.append(str(alert["type"]).lower()) + if alert.get("alert_source_name"): + labels.append(str(alert["alert_source_name"])) + + incident = self.create_incident( + name=name, + description="\n".join(desc_lines), + severity=alert.get("severity"), + first_seen=first_seen, + last_seen=first_seen, + source="Endpoint", + labels=labels, + external_id=alert.get("id"), + ) + objects.append(incident) + + # MITRE ATT&CK techniques -> Attack-Pattern (uses) + for mid in alert.get("mitre_ids") or []: + ap = self.create_attack_pattern(mid) + if ap: + objects.append(ap) + objects.append( + self.create_relationship(incident["id"], "uses", ap["id"]) + ) + + # url -> Url (external destination — a legitimate observable). agent_ip is the + # affected host's INTERNAL IP and is recorded on the System identity, not as an IOC. + url = alert.get("url") + if url and is_valid_url(url): + url_obj = self.create_url(url) + if url_obj: + objects.append(url_obj) + objects.append( + self.create_relationship( + incident["id"], "related-to", url_obj["id"] + ) + ) + + # endpoint -> System identity (internal asset, not infrastructure/IOC) + if endpoint_name: + agent_ip = alert.get("agent_ip") + desc = "Metras EDR endpoint" + ( + f"; agent_ip={agent_ip}" if agent_ip else "" + ) + system = self.create_system(endpoint_name, description=desc) + if system: + objects.append(system) + objects.append( + self.create_relationship(incident["id"], "related-to", system["id"]) + ) + + return objects + + def process_binary(self, binary: dict, malicious_only: bool = True) -> list: + """Convert one binary into a StixFile observable (+ System identity link).""" + runnability = (binary.get("runnability_status") or "").lower() + signature = (binary.get("signature_status") or "").lower() + is_malicious = runnability == "banned" or signature == "unsigned" + if malicious_only and not is_malicious: + return [] + + score = ( + 80 if runnability == "banned" else (60 if signature == "unsigned" else 30) + ) + size = binary.get("file_size_bytes") + sfile = self.create_file( + md5=binary.get("md5"), + sha1=binary.get("sha1"), + sha256=binary.get("sha256"), + name=binary.get("name"), + size=size if isinstance(size, int) else None, + score=score, + ) + if not sfile: + return [] + objects = [sfile] + + first_endpoint = binary.get("first_endpoint_name") + if first_endpoint: + system = self.create_system( + first_endpoint, description="Metras EDR endpoint" + ) + if system: + objects.append(system) + objects.append( + self.create_relationship(sfile["id"], "related-to", system["id"]) + ) + return objects + + def process_endpoint(self, endpoint: dict) -> list: + """Convert one endpoint into a System identity (internal asset). + + Internal interface/tunnel IPs are recorded in the description, NOT emitted as + IPv4-Addr observables (they are not IOCs). + """ + name = endpoint.get("name") or endpoint.get("id") + if not name: + return [] + os_type = endpoint.get("os") or "unknown" + + ips = set() + sc = endpoint.get("sc_connection_info") or {} + if sc.get("tunnel_ip"): + ips.add(sc["tunnel_ip"]) + nw = endpoint.get("nw_info") or {} + for iface in (nw.get("interfaces") or []) if isinstance(nw, dict) else []: + for ip_val in (iface.get("ips") or []) if isinstance(iface, dict) else []: + ips.add(ip_val) + + ip_note = f"; ips={', '.join(sorted(ips))}" if ips else "" + system = self.create_system( + name, + description=( + f"Metras endpoint ({os_type}); " + f"serial={endpoint.get('serial', 'n/a')}{ip_note}" + ), + ) + return [system] if system else [] + + def author_object(self) -> stix2.Identity: + return self.author diff --git a/external-import/metras/src/connector/settings.py b/external-import/metras/src/connector/settings.py new file mode 100644 index 0000000000..aa3e5cc29d --- /dev/null +++ b/external-import/metras/src/connector/settings.py @@ -0,0 +1,64 @@ +"""Pydantic settings for the Metras Feed connector (EXTERNAL_IMPORT).""" + +from datetime import timedelta +from typing import Literal + +from connectors_sdk import ( + BaseConfigModel, + BaseConnectorSettings, + BaseExternalImportConnectorConfig, +) +from pydantic import Field, HttpUrl, SecretStr + + +class ExternalImportConnectorConfig(BaseExternalImportConnectorConfig): + name: str = Field(default="Metras-Feed", examples=["Metras-Feed"]) + # Poll cadence. ISO-8601 duration (e.g. PT1H). Mandatory for the SDK. + duration_period: timedelta = Field(default=timedelta(hours=1), examples=["PT1H"]) + + +class MetrasConfig(BaseConfigModel): + api_base_url: HttpUrl = Field( + default="https://api.metras.sa/api", + description="Base URL of the Metras API.", + examples=["https://api.metras.sa/api"], + ) + api_key: SecretStr = Field( + description="Metras API key (X-API-KEY header).", + examples=["ChangeMe"], + ) + verify_ssl: bool = Field( + default=True, description="Verify TLS certificates.", examples=[True] + ) + + import_alerts: bool = Field( + default=True, description="Import EDR alerts.", examples=[True] + ) + import_binaries: bool = Field( + default=True, description="Import binaries as StixFile.", examples=[True] + ) + import_endpoints: bool = Field( + default=True, + description="Import endpoints as System identities.", + examples=[True], + ) + binary_malicious_only: bool = Field( + default=True, + description="Only import banned/unsigned binaries (reduces noise).", + examples=[True], + ) + page_size: int = Field( + default=50, ge=1, le=500, description="Records per page.", examples=[50] + ) + tlp_level: Literal["clear", "white", "green", "amber", "red"] = Field( + default="amber", + description="TLP marking applied to imported objects.", + examples=["amber"], + ) + + +class ConnectorSettings(BaseConnectorSettings): + connector: ExternalImportConnectorConfig = Field( + default_factory=ExternalImportConnectorConfig + ) + metras: MetrasConfig = Field(default_factory=MetrasConfig) diff --git a/external-import/metras/src/connector/utils.py b/external-import/metras/src/connector/utils.py new file mode 100644 index 0000000000..64964462f5 --- /dev/null +++ b/external-import/metras/src/connector/utils.py @@ -0,0 +1,119 @@ +"""Pure-function utilities for the Metras Feed connector. + +No side effects, no HTTP, no STIX, no config access. +""" + +import ipaddress +import re +from datetime import datetime, timezone + +# Metras severity -> (OpenCTI x_opencti_score, OpenCTI severity label) +_SEVERITY_MAP = { + "critical": (90, "critical"), + "high": (75, "high"), + "medium": (50, "medium"), + "low": (25, "low"), + "informational": (10, "low"), + "info": (10, "low"), +} + +# Numeric severity (threats/violations use 1-5) +_NUMERIC_SEVERITY = { + 5: (90, "critical"), + 4: (75, "high"), + 3: (50, "medium"), + 2: (25, "low"), + 1: (10, "low"), +} + + +def refang(value: str) -> str: + """Remove common defanging so values match the external API.""" + if not value: + return value + v = value + v = v.replace("[.]", ".").replace("(.)", ".").replace("{.}", ".") + v = v.replace("[:]", ":").replace("[://]", "://") + v = v.replace("hxxp://", "http://").replace("hxxps://", "https://") + v = v.replace("hXXp://", "http://").replace("hXXps://", "https://") + v = v.replace("[at]", "@").replace("[@]", "@") + v = v.replace("[dot]", ".") + return v.strip() + + +def is_valid_ipv4(value: str) -> bool: + try: + return isinstance(ipaddress.ip_address(value), ipaddress.IPv4Address) + except (ValueError, TypeError): + return False + + +def is_valid_url(value: str) -> bool: + return isinstance(value, str) and value.startswith(("http://", "https://")) + + +# Expected hex length per hash algorithm (STIX 2.1 hash names). +_HASH_LENGTHS = {"MD5": 32, "SHA-1": 40, "SHA-256": 64, "SHA-512": 128} +_HEX = set("0123456789abcdefABCDEF") + + +def is_valid_hash(algo: str, value: str) -> bool: + """True if ``value`` is a well-formed hex digest for ``algo`` (MD5/SHA-1/256/512). + + Guards STIX File creation against malformed hashes from the API, which would + otherwise make stix2 raise and crash an import cycle. + """ + length = _HASH_LENGTHS.get(algo) + return ( + bool(value) + and length is not None + and len(value) == length + and all(c in _HEX for c in value) + ) + + +def normalize_timestamp(ts: str | None) -> datetime | None: + """Parse an ISO-8601 timestamp (Z or offset) into a tz-aware datetime.""" + if not ts: + return None + try: + normalized = ts.replace("Z", "+00:00") + dt = datetime.fromisoformat(normalized) + if dt.tzinfo is None: + dt = dt.replace(tzinfo=timezone.utc) + return dt + except (ValueError, TypeError): + return None + + +def is_newer_than(timestamp_str: str | None, cutoff: datetime | None) -> bool: + """True if ``timestamp_str`` is strictly newer than ``cutoff`` (or no cutoff).""" + if cutoff is None: + return True + parsed = normalize_timestamp(timestamp_str) + if parsed is None: + return True + return parsed > cutoff + + +def stix_timestamp(dt: datetime | None = None) -> str: + """Return a Z-suffixed UTC timestamp acceptable to the stix2 library.""" + if dt is None: + dt = datetime.now(timezone.utc) + if dt.tzinfo is None: + dt = dt.replace(tzinfo=timezone.utc) + return dt.astimezone(timezone.utc).strftime("%Y-%m-%dT%H:%M:%SZ") + + +def severity_to_score(severity: int | float | str | None) -> tuple[int, str]: + """Map a Metras severity (string or 1-5 int) to (score, opencti_severity).""" + if isinstance(severity, (int, float)): + return _NUMERIC_SEVERITY.get(int(severity), (50, "medium")) + if isinstance(severity, str): + return _SEVERITY_MAP.get(severity.strip().lower(), (50, "medium")) + return (50, "medium") + + +def is_mitre_attack_id(value: str) -> bool: + """True for MITRE ATT&CK technique IDs like T1059 or T1059.001.""" + return bool(re.fullmatch(r"T\d{4}(?:\.\d{3})?", value or "")) diff --git a/external-import/metras/src/main.py b/external-import/metras/src/main.py new file mode 100644 index 0000000000..fba4650529 --- /dev/null +++ b/external-import/metras/src/main.py @@ -0,0 +1,14 @@ +import traceback + +from connector import ConnectorSettings, MetrasFeedConnector +from pycti import OpenCTIConnectorHelper + +if __name__ == "__main__": + try: + settings = ConnectorSettings() + helper = OpenCTIConnectorHelper(config=settings.to_helper_config()) + connector = MetrasFeedConnector(config=settings, helper=helper) + connector.run() + except Exception: + traceback.print_exc() + exit(1) diff --git a/external-import/metras/src/metras_client/__init__.py b/external-import/metras/src/metras_client/__init__.py new file mode 100644 index 0000000000..9421cb9c4a --- /dev/null +++ b/external-import/metras/src/metras_client/__init__.py @@ -0,0 +1,3 @@ +from metras_client.api_client import MetrasAPIError, MetrasClient + +__all__ = ["MetrasClient", "MetrasAPIError"] diff --git a/external-import/metras/src/metras_client/api_client.py b/external-import/metras/src/metras_client/api_client.py new file mode 100644 index 0000000000..aa91322da0 --- /dev/null +++ b/external-import/metras/src/metras_client/api_client.py @@ -0,0 +1,268 @@ +"""Shared Metras API client. + +This module is identical across the metras-feed, metras-enrichment and +metras-stream connectors (copied verbatim — no cross-directory imports). +It wraps the Metras REST API (https://api.metras.sa/api), authenticated with the +``X-API-KEY`` header, and exposes thin methods used by all three connectors. +""" + +from collections.abc import Iterator +from typing import TYPE_CHECKING +from urllib.parse import quote + +import requests +from requests.adapters import HTTPAdapter +from urllib3.util.retry import Retry + +if TYPE_CHECKING: + from pycti import OpenCTIConnectorHelper + + +class MetrasAPIError(Exception): + """Raised when the Metras API returns an error or is unreachable.""" + + def __init__(self, message: str, status_code: int | None = None) -> None: + self.message = message + self.status_code = status_code + super().__init__(self.message) + + +class MetrasClient: + """Thin HTTP client for the Metras API. + + Methods return parsed JSON (dict) and raise :class:`MetrasAPIError` on + failure. Parsing of business data belongs in the converters, not here. + """ + + # Conservative default timeouts (connect, read) + _TIMEOUT = (10, 60) + + def __init__( + self, + helper: "OpenCTIConnectorHelper", + base_url: str, + api_key: str, + verify_ssl: bool = True, + ) -> None: + self.helper = helper + self.base_url = str(base_url).rstrip("/") + self.verify_ssl = verify_ssl + + self.session = requests.Session() + # Explicit for security scanners; honour the configured value. + self.session.verify = verify_ssl + self.session.headers.update( + { + "X-API-KEY": api_key, + "Accept": "application/json", + "Content-Type": "application/json", + } + ) + + # Retry on transient/server errors, honouring Retry-After on 429. + retry = Retry( + total=3, + connect=3, + read=3, + backoff_factor=1, + status_forcelist=(429, 500, 502, 503, 504), + allowed_methods=("GET", "POST", "PATCH", "DELETE"), + respect_retry_after_header=True, + raise_on_status=False, + ) + adapter = HTTPAdapter(max_retries=retry) + self.session.mount("https://", adapter) + self.session.mount("http://", adapter) + + # ------------------------------------------------------------------ # + # Low-level request helpers + # ------------------------------------------------------------------ # + def _request( + self, method: str, path: str, params: dict | None = None, json_data=None + ) -> dict: + url = f"{self.base_url}{path}" + try: + resp = self.session.request( + method, + url, + params=params, + json=json_data, + timeout=self._TIMEOUT, + ) + except requests.exceptions.RequestException as exc: + raise MetrasAPIError(f"Request to {path} failed: {exc}") from exc + + self.helper.connector_logger.debug( + "[API] Request", + {"method": method, "path": path, "status": resp.status_code}, + ) + + if resp.status_code in (401, 403): + raise MetrasAPIError( + f"Authentication failed (HTTP {resp.status_code}) for {path}. " + "Check METRAS_API_KEY.", + resp.status_code, + ) + if resp.status_code >= 400: + raise MetrasAPIError( + f"Metras API error (HTTP {resp.status_code}) for {path}: " + f"{resp.text[:500]}", + resp.status_code, + ) + + # Some write endpoints return 202/empty bodies. + if not resp.content: + return {} + try: + return resp.json() + except ValueError as exc: + raise MetrasAPIError( + f"Non-JSON response from {path} (HTTP {resp.status_code}): " + f"{resp.text[:200]}", + resp.status_code, + ) from exc + + def _get(self, path: str, params: dict | None = None) -> dict: + return self._request("GET", path, params=params) + + def _post(self, path: str, json_data=None) -> dict: + return self._request("POST", path, json_data=json_data) + + def _patch(self, path: str, json_data=None) -> dict: + return self._request("PATCH", path, json_data=json_data) + + def _delete(self, path: str) -> dict: + return self._request("DELETE", path) + + # ------------------------------------------------------------------ # + # Connectivity / startup + # ------------------------------------------------------------------ # + def ping(self) -> bool: + """Lightweight connectivity + auth check. + + Metras has no dedicated health endpoint, so a tiny endpoints list call + is used. Raises :class:`MetrasAPIError` on failure. + """ + self._get("/v1/endpoints", params={"status": "activated"}) + return True + + # ------------------------------------------------------------------ # + # Feed (EXTERNAL_IMPORT) endpoints + # ------------------------------------------------------------------ # + def list_edr_alerts(self, page_size: int = 50, skip: int = 0, **filters) -> dict: + """One page of EDR 2.0 alerts. Response: {totalCount, more, data[]}. + + Note: this endpoint has no fromTime/toTime — callers filter client-side + on ``last_occurrence_time``. + """ + params = {"limit": page_size, "skip": skip} + params.update({k: v for k, v in filters.items() if v is not None}) + return self._get("/v1/edr/alerts", params=params) + + def iter_edr_alerts( + self, page_size: int = 50, max_pages: int = 200, **filters + ) -> Iterator[dict]: + """Yield EDR alert records across pages until ``more`` is false.""" + skip = 0 + for _ in range(max_pages): + payload = self.list_edr_alerts(page_size=page_size, skip=skip, **filters) + data = payload.get("data") or [] + for record in data: + yield record + if not payload.get("more") or not data: + break + skip += page_size + + def list_binaries( + self, + from_time: str | None = None, + to_time: str | None = None, + page_size: int = 50, + skip: int = 0, + query: str | None = None, + ) -> dict: + """One page of binary inventory. Response: {totalCount, data[]}.""" + params = {"limit": page_size, "skip": skip, "full": "true"} + if from_time: + params["fromTime"] = from_time + if to_time: + params["toTime"] = to_time + if query: + params["query"] = query + return self._get("/v1/edr/binary/list", params=params) + + def iter_binaries( + self, + from_time: str | None = None, + page_size: int = 50, + max_pages: int = 200, + query: str | None = None, + ) -> Iterator[dict]: + """Yield binary records across pages (stops when a short page returns).""" + skip = 0 + for _ in range(max_pages): + payload = self.list_binaries( + from_time=from_time, page_size=page_size, skip=skip, query=query + ) + data = payload.get("data") or [] + for record in data: + yield record + if len(data) < page_size: + break + skip += page_size + + def binary_details(self, md5: str | None = None, name: str | None = None) -> dict: + """Single binary by md5 or name.""" + params = {} + if md5: + params["md5"] = md5 + if name: + params["name"] = name + return self._get("/v1/edr/binary/details", params=params) + + def list_endpoints(self, **filters) -> dict: + """Endpoint/asset inventory. Response: {endpoints[]}.""" + params = {k: v for k, v in filters.items() if v is not None} + return self._get("/v1/endpoints", params=params) + + # ------------------------------------------------------------------ # + # Enrichment (INTERNAL_ENRICHMENT) endpoints + # ------------------------------------------------------------------ # + def threat_details(self, **filters) -> dict: + """Network incident details, filterable by sourceIP/destinationIP/url.""" + params = {k: v for k, v in filters.items() if v is not None} + return self._get("/v4/threats/detail", params=params) + + def binary_by_hash( + self, sha256: str | None = None, sha1: str | None = None + ) -> dict: + """Look up a binary by sha256/sha1 via the list endpoint's query param.""" + if sha256: + return self.list_binaries(query=f"sha256:{sha256}") + if sha1: + return self.list_binaries(query=f"sha1:{sha1}") + return {"data": []} + + def alerts_by_agent_ip(self, agent_ip: str, page_size: int = 50) -> dict: + return self.list_edr_alerts(page_size=page_size, agent_ip=agent_ip) + + # ------------------------------------------------------------------ # + # Stream (STREAM) endpoints — custom blocklist (file-path only) + # ------------------------------------------------------------------ # + def create_blocklist(self, items: list[dict]) -> dict: + """Create one or more custom blocklists (file_paths). Body is an array.""" + return self._post("/v1/custom-blocklist", json_data=items) + + def list_blocklists(self, name: str | None = None, page_size: int = 50) -> dict: + params = {"limit": page_size} + if name: + params["name"] = name + return self._get("/v1/custom-blocklist", params=params) + + def update_blocklist(self, blocklist_id: str, patch: dict) -> dict: + safe_id = quote(str(blocklist_id), safe="") + return self._patch(f"/v1/custom-blocklist/{safe_id}", json_data=patch) + + def delete_blocklist(self, blocklist_id: str) -> dict: + safe_id = quote(str(blocklist_id), safe="") + return self._delete(f"/v1/custom-blocklist/{safe_id}") diff --git a/external-import/metras/src/requirements.txt b/external-import/metras/src/requirements.txt new file mode 100644 index 0000000000..fb7e773723 --- /dev/null +++ b/external-import/metras/src/requirements.txt @@ -0,0 +1,6 @@ +pycti==7.260529.0 +pydantic~=2.11.3 +stix2>=3.0.1,<4.0.0 +requests>=2.32.0,<3.0.0 +pyyaml>=6.0.1,<7.0.0 +connectors-sdk @ git+https://github.com/OpenCTI-Platform/connectors.git@7.260529.0#subdirectory=connectors-sdk diff --git a/external-import/metras/tests/conftest.py b/external-import/metras/tests/conftest.py new file mode 100644 index 0000000000..5ee8fc0e22 --- /dev/null +++ b/external-import/metras/tests/conftest.py @@ -0,0 +1,4 @@ +import os +import sys + +sys.path.append(os.path.join(os.path.dirname(__file__), "..", "src")) diff --git a/external-import/metras/tests/test-requirements.txt b/external-import/metras/tests/test-requirements.txt new file mode 100644 index 0000000000..6ef4498bff --- /dev/null +++ b/external-import/metras/tests/test-requirements.txt @@ -0,0 +1,2 @@ +-r ../src/requirements.txt +pytest>=8.0.0 diff --git a/external-import/metras/tests/test_connector/test_client.py b/external-import/metras/tests/test_connector/test_client.py new file mode 100644 index 0000000000..cc5498b815 --- /dev/null +++ b/external-import/metras/tests/test_connector/test_client.py @@ -0,0 +1,79 @@ +"""Unit tests for the shared MetrasClient (HTTP layer mocked).""" + +from unittest.mock import MagicMock + +import pytest +from metras_client import MetrasAPIError, MetrasClient + + +class FakeResp: + def __init__(self, status_code=200, payload=None, text=""): + self.status_code = status_code + self._payload = payload + self.text = text + self.content = b"x" if (payload is not None or text) else b"" + + def json(self): + if self._payload is None: + raise ValueError("no json") + return self._payload + + +def _client(responses): + client = MetrasClient(helper=MagicMock(), base_url="http://x/api", api_key="k") + session = MagicMock() + session.request.side_effect = list(responses) + client.session = session + return client + + +def test_ping_ok(): + assert _client([FakeResp(200, {"endpoints": []})]).ping() is True + + +def test_auth_error_raises_with_status(): + client = _client([FakeResp(401, text="nope")]) + with pytest.raises(MetrasAPIError) as exc: + client.ping() + assert exc.value.status_code == 401 + + +def test_server_error_raises(): + with pytest.raises(MetrasAPIError): + _client([FakeResp(500, text="boom")])._get("/v1/endpoints") + + +def test_non_json_response_raises(): + # A 200 with a non-JSON body (WAF/HTML/proxy error page) must fail loudly, + # not silently return an unparsable payload that looks like an empty result. + with pytest.raises(MetrasAPIError): + _client([FakeResp(200, text="blocked")])._get("/v1/endpoints") + + +def test_iter_edr_alerts_paginates_until_more_false(): + client = _client( + [ + FakeResp(200, {"data": [{"id": "1"}, {"id": "2"}], "more": True}), + FakeResp(200, {"data": [{"id": "3"}], "more": False}), + ] + ) + assert [a["id"] for a in client.iter_edr_alerts(page_size=2)] == ["1", "2", "3"] + + +def test_iter_binaries_stops_on_short_page(): + client = _client([FakeResp(200, {"data": [{"md5": "a"}]})]) + assert len(list(client.iter_binaries(page_size=50))) == 1 + + +def test_create_blocklist_handles_empty_202(): + assert _client([FakeResp(202)]).create_blocklist([{"name": "x"}]) == {} + + +def test_list_blocklists_filters_by_name(): + client = _client([FakeResp(200, {"data": [{"id": "b1", "name": "n"}]})]) + assert client.list_blocklists(name="n")["data"][0]["id"] == "b1" + + +def test_binary_by_hash_uses_query(): + client = _client([FakeResp(200, {"data": [{"sha256": "abc"}]})]) + assert client.binary_by_hash(sha256="abc")["data"][0]["sha256"] == "abc" diff --git a/external-import/metras/tests/test_connector/test_converter.py b/external-import/metras/tests/test_connector/test_converter.py new file mode 100644 index 0000000000..128c2577e0 --- /dev/null +++ b/external-import/metras/tests/test_connector/test_converter.py @@ -0,0 +1,107 @@ +"""Converter tests (no connectors-sdk / no live OpenCTI needed).""" + +from unittest.mock import MagicMock + +from connector.converter_to_stix import ConverterToStix + + +def _converter(): + helper = MagicMock() + helper.connect_confidence_level = 50 + return ConverterToStix(helper, tlp_level="amber") + + +def test_process_alert_creates_incident_and_attack_pattern(): + conv = _converter() + alert = { + "id": "1748880_NToxMDc0Ng", + "alert_name": "test-rule-1", + "alert_source_name": "behavior", + "type": "WATCHLIST", + "severity": "Critical", + "agent_ip": "10.200.0.214", + "url": "https://evil.example.com", + "endpoint_name": "TEST-ENDPOINT", + "endpoint_id": "1e53d0d4-84e4-4a7b-bff7-d0b216c4ac40", + "mitre_ids": ["T1059", "T1059.001"], + "last_occurrence_time": "2025-11-11T11:36:20.162Z", + "occurrence_count": 1, + "process": {"name": "WUDFHost.exe", "guid": "{bda3de7f}"}, + "tags": ["abc"], + } + objs = conv.process_alert(alert) + types = [o["type"] for o in objs] + assert "incident" in types + assert types.count("attack-pattern") == 2 + assert "url" in types + # Affected endpoint is a System identity; internal agent_ip is NOT an IPv4-Addr IOC. + assert "identity" in types + assert "ipv4-addr" not in types + assert "relationship" in types + + +def test_process_binary_malicious_only(): + conv = _converter() + banned = { + "md5": "2e9fc997dea8b0fc30761e7d2e2c54be", + "sha256": "27e38928588e5153becf77dabe6a5e5df8377ab814ef9127f68155ed176e1181", + "name": "evil.dll", + "file_size_bytes": 1024, + "runnability_status": "banned", + "first_endpoint_name": "TEST-ENDPOINT", + } + signed = { + "md5": "abc", + "name": "good.dll", + "signature_status": "Signed", + "runnability_status": "allowed", + } + banned_objs = conv.process_binary(banned, malicious_only=True) + assert any(o["type"] == "file" for o in banned_objs) + assert any( + o["type"] == "identity" for o in banned_objs + ) # System for first endpoint + assert conv.process_binary(signed, malicious_only=True) == [] + assert any( + o["type"] == "file" for o in conv.process_binary(signed, malicious_only=False) + ) + + +def test_process_endpoint_creates_system_identity(): + conv = _converter() + endpoint = { + "id": "4373f3cc-80ec-4f57-bea6-6d431b2d4b8d", + "name": "win19", + "os": "windows", + "serial": "764QDF2", + "sc_connection_info": {"tunnel_ip": "172.16.68.7"}, + "nw_info": {"interfaces": [{"ips": ["10.0.0.5"]}]}, + } + objs = conv.process_endpoint(endpoint) + types = [o["type"] for o in objs] + # Endpoint is a System identity; internal IPs go in the description, not as IOCs. + assert types == ["identity"] + assert "172.16.68.7" in objs[0]["description"] + assert "10.0.0.5" in objs[0]["description"] + + +def test_incident_id_stable_across_occurrence_time(): + # A recurring alert (same id, newer last_occurrence_time) must map to the SAME + # Incident id so OpenCTI updates it instead of creating a duplicate. + conv = _converter() + base = { + "id": "alert-42", + "alert_name": "recurring-rule", + "severity": "high", + "last_occurrence_time": "2025-11-11T11:36:20.162Z", + } + + def _incident_id(alert): + return next( + o["id"] for o in conv.process_alert(alert) if o["type"] == "incident" + ) + + newer = {**base, "last_occurrence_time": "2025-12-01T09:00:00.000Z"} + assert _incident_id(base) == _incident_id(newer) + # A different alert id yields a different Incident. + assert _incident_id({**base, "id": "alert-99"}) != _incident_id(base) diff --git a/external-import/metras/tests/test_connector/test_orchestration.py b/external-import/metras/tests/test_connector/test_orchestration.py new file mode 100644 index 0000000000..47f3d5c82a --- /dev/null +++ b/external-import/metras/tests/test_connector/test_orchestration.py @@ -0,0 +1,101 @@ +"""Orchestration tests for the Feed connector (helper + client mocked).""" + +from datetime import timedelta +from types import SimpleNamespace +from unittest.mock import MagicMock + +from connector.connector import MetrasFeedConnector +from pydantic import SecretStr + +SHA256 = "27e38928588e5153becf77dabe6a5e5df8377ab814ef9127f68155ed176e1181" + + +def _feed(): + cfg = SimpleNamespace( + metras=SimpleNamespace( + api_base_url="http://x/api", + api_key=SecretStr("k"), + verify_ssl=True, + tlp_level="amber", + page_size=50, + import_alerts=True, + import_binaries=True, + import_endpoints=True, + binary_malicious_only=True, + ), + connector=SimpleNamespace(duration_period=timedelta(hours=1)), + ) + helper = MagicMock() + helper.connect_confidence_level = 50 + helper.get_state.return_value = {} + helper.api.work.initiate_work.return_value = "work-1" + helper.stix2_create_bundle.return_value = "{}" + conn = MetrasFeedConnector(cfg, helper) + conn.client = MagicMock() + return conn, helper + + +def test_import_sends_bundle_with_cleanup_flag_and_advances_state(): + conn, helper = _feed() + conn.client.iter_edr_alerts.return_value = iter( + [ + { + "id": "a1", + "alert_name": "r1", + "severity": "Critical", + "mitre_ids": ["T1059"], + "last_occurrence_time": "2025-11-11T11:36:20Z", + "endpoint_name": "EP", + "agent_ip": "10.0.0.1", + } + ] + ) + conn.client.iter_binaries.return_value = iter( + [ + { + "md5": "2e9fc997dea8b0fc30761e7d2e2c54be", + "sha256": SHA256, + "name": "evil.dll", + "runnability_status": "banned", + "first_endpoint_name": "EP", + "last_seen": "2025-11-12T09:00:00Z", + } + ] + ) + conn.client.list_endpoints.return_value = { + "endpoints": [{"name": "EP", "os": "windows"}] + } + + conn._import_data() + + assert helper.send_stix2_bundle.called + _, kwargs = helper.send_stix2_bundle.call_args + assert kwargs.get("cleanup_inconsistent_bundle") is True + state = helper.set_state.call_args[0][0] + assert "alerts_last_occurrence" in state and "binaries_last_seen" in state + + +def test_import_skips_old_alerts_via_cursor(): + conn, helper = _feed() + helper.get_state.return_value = {"alerts_last_occurrence": "2025-12-01T00:00:00Z"} + conn.client.iter_edr_alerts.return_value = iter( + [ + { + "id": "old", + "alert_name": "r", + "last_occurrence_time": "2025-11-11T11:36:20Z", + } + ] + ) + conn.client.iter_binaries.return_value = iter([]) + conn.client.list_endpoints.return_value = {"endpoints": []} + + conn._import_data() + # Only the author object -> nothing new -> no bundle sent. + assert not helper.send_stix2_bundle.called + + +def test_duration_seconds_parses_iso_and_timedelta(): + assert MetrasFeedConnector._duration_seconds(timedelta(minutes=10)) == 600 + assert MetrasFeedConnector._duration_seconds("PT2H") == 7200 + assert MetrasFeedConnector._duration_seconds("P1D") == 86400 diff --git a/external-import/metras/tests/test_connector/test_settings.py b/external-import/metras/tests/test_connector/test_settings.py new file mode 100644 index 0000000000..c9787deeaa --- /dev/null +++ b/external-import/metras/tests/test_connector/test_settings.py @@ -0,0 +1,21 @@ +"""Settings (config-model) tests for the Metras Feed connector.""" + +from connector.settings import MetrasConfig +from pydantic import SecretStr + + +def test_api_key_is_secret(): + cfg = MetrasConfig(api_key="super-secret-key") + assert isinstance(cfg.api_key, SecretStr) + assert cfg.api_key.get_secret_value() == "super-secret-key" + # The secret must not appear in the repr / str. + assert "super-secret-key" not in repr(cfg) + + +def test_feed_defaults(): + cfg = MetrasConfig(api_key="k") + assert cfg.verify_ssl is True + assert cfg.import_alerts and cfg.import_binaries and cfg.import_endpoints + assert cfg.binary_malicious_only is True + assert cfg.page_size == 50 + assert cfg.tlp_level == "amber" diff --git a/external-import/metras/tests/test_main.py b/external-import/metras/tests/test_main.py new file mode 100644 index 0000000000..fead289458 --- /dev/null +++ b/external-import/metras/tests/test_main.py @@ -0,0 +1,50 @@ +"""Unit tests for shared, dependency-free logic in the Metras Feed connector.""" + +from connector.utils import ( + is_mitre_attack_id, + is_newer_than, + is_valid_ipv4, + is_valid_url, + normalize_timestamp, + refang, + severity_to_score, + stix_timestamp, +) + + +def test_refang(): + assert refang("1[.]2[.]3[.]4") == "1.2.3.4" + assert refang("hxxps://evil[.]com") == "https://evil.com" + + +def test_ipv4_validation(): + assert is_valid_ipv4("10.200.0.214") + assert not is_valid_ipv4("not-an-ip") + assert not is_valid_ipv4("2001:db8::1") + + +def test_url_validation(): + assert is_valid_url("https://example.com") + assert not is_valid_url("example.com") + + +def test_mitre_ids(): + assert is_mitre_attack_id("T1059") + assert is_mitre_attack_id("T1059.001") + assert not is_mitre_attack_id("X1059") + assert not is_mitre_attack_id("1059") + + +def test_severity_mapping(): + assert severity_to_score("Critical")[0] == 90 + assert severity_to_score(5)[0] == 90 + assert severity_to_score("unknown")[1] == "medium" + + +def test_timestamp_helpers(): + dt = normalize_timestamp("2025-11-11T11:36:20.162Z") + assert dt is not None and dt.tzinfo is not None + older = normalize_timestamp("2025-01-01T00:00:00Z") + assert is_newer_than("2025-11-11T11:36:20Z", older) + assert not is_newer_than("2024-01-01T00:00:00Z", dt) + assert stix_timestamp(dt).endswith("Z")