diff --git a/stream/cloudflare-rules-list/.env.sample b/stream/cloudflare-rules-list/.env.sample new file mode 100644 index 0000000000..e97bf755ba --- /dev/null +++ b/stream/cloudflare-rules-list/.env.sample @@ -0,0 +1,24 @@ +# OpenCTI +OPENCTI_URL=http://localhost:8080 +OPENCTI_TOKEN=ChangeMe + +# Connector +CONNECTOR_ID=ChangeMe +CONNECTOR_TYPE=STREAM +CONNECTOR_NAME=Cloudflare Rules List +CONNECTOR_SCOPE=cloudflare +CONNECTOR_LOG_LEVEL=info +CONNECTOR_LIVE_STREAM_ID=live +CONNECTOR_LIVE_STREAM_LISTEN_DELETE=true +CONNECTOR_LIVE_STREAM_NO_DEPENDENCIES=true +# Minimum interval between snapshot uploads ('30m', '1h', '1h30m', or seconds) +CONNECTOR_SYNC_INTERVAL=1h + +# Cloudflare +CLOUDFLARE_ACCOUNT_ID=ChangeMe +# API token with 'Account > Account Filter Lists > Edit' permission +CLOUDFLARE_API_TOKEN=ChangeMe +# ID of the existing Cloudflare Rules List (IP kind) to sync into +CLOUDFLARE_LIST_ID=ChangeMe +# Base URL of the Cloudflare API. Override only for testing or a compatible gateway. +# CLOUDFLARE_API_BASE_URL=https://api.cloudflare.com/client/v4 diff --git a/stream/cloudflare-rules-list/Dockerfile b/stream/cloudflare-rules-list/Dockerfile new file mode 100644 index 0000000000..5cea3c1e2d --- /dev/null +++ b/stream/cloudflare-rules-list/Dockerfile @@ -0,0 +1,19 @@ +FROM python:3.12-alpine + +ENV CONNECTOR_TYPE=STREAM \ + PYTHONDONTWRITEBYTECODE=1 \ + PYTHONUNBUFFERED=1 + +# connectors-sdk is installed from a git URL, so git/build tooling is needed at build time. +# libmagic is a runtime dependency of pycti (python-magic) and must remain installed. +COPY src /opt/opencti-connector-cloudflare-rules-list +RUN apk --no-cache add git build-base libffi-dev openssl-dev libmagic && \ + cd /opt/opencti-connector-cloudflare-rules-list && \ + pip3 install --no-cache-dir --upgrade pip && \ + pip3 install --no-cache-dir -r requirements.txt && \ + apk del git build-base + +COPY entrypoint.sh / +RUN chmod +x /entrypoint.sh + +ENTRYPOINT ["/entrypoint.sh"] diff --git a/stream/cloudflare-rules-list/README.md b/stream/cloudflare-rules-list/README.md new file mode 100644 index 0000000000..1228558f70 --- /dev/null +++ b/stream/cloudflare-rules-list/README.md @@ -0,0 +1,161 @@ +# OpenCTI Cloudflare Rules List Connector + +A **STREAM** connector that pushes **IPv4** threat-intelligence indicators and +observables from OpenCTI into a +[Cloudflare Rules List](https://developers.cloudflare.com/waf/tools/lists/), +where they can be referenced from WAF custom rules, firewall rules, and other +Cloudflare security configurations. + +## Table of Contents + +- [Introduction](#introduction) +- [Requirements](#requirements) +- [Configuration variables](#configuration-variables) +- [Deployment](#deployment) +- [Behavior](#behavior) +- [Capabilities and limitations](#capabilities-and-limitations) +- [Development](#development) + +## Introduction + +Cloudflare Rules Lists are reusable collections of IP addresses scoped to a +Cloudflare account. This connector subscribes to an OpenCTI live stream and +keeps a target Rules List (of **IP** kind) in sync with the IPv4 indicators and +`IPv4-Addr` observables known to OpenCTI. + +Cloudflare's list-items API follows a **replace/snapshot** model: each upload +replaces the full contents of the list. The connector therefore maintains an +in-memory snapshot of all known IPv4 values and pushes the complete snapshot at +most once per `CONNECTOR_SYNC_INTERVAL`. + +## Requirements + +- OpenCTI Platform 7.x. The connector pins `pycti==7.260715.0`; because pycti + follows the platform's CalVer scheme, its major version must match your + OpenCTI platform. Align the pin with your platform version if it differs. +- A Cloudflare account and an existing Rules List of **IP** kind +- A Cloudflare API token with the `Account > Account Filter Lists > Edit` + permission +- Python 3.11 or 3.12 (for local runs; the SDK requires `< 3.13`) + +## Configuration variables + +Configuration is read (in precedence order) from environment variables, then a +`config.yml` or `.env` file next to the connector, then field defaults. + +A full, generated reference is available in +[`__metadata__/CONNECTOR_CONFIG_DOC.md`](__metadata__/CONNECTOR_CONFIG_DOC.md). + +### OpenCTI configuration + +| Parameter | config.yml | Docker env var | Mandatory | Description | +| ------------- | --------------- | ---------------- | --------- | ---------------------------------------- | +| OpenCTI URL | `opencti.url` | `OPENCTI_URL` | Yes | The base URL of the OpenCTI instance. | +| OpenCTI Token | `opencti.token` | `OPENCTI_TOKEN` | Yes | The API token to connect to OpenCTI. | + +### Base connector configuration + +| Parameter | config.yml | Docker env var | Default | Mandatory | Description | +| ------------------ | ---------------------------- | ---------------------------- | --------------------- | --------- | --------------------------------------------------------------------------- | +| Connector ID | `connector.id` | `CONNECTOR_ID` | / | Yes | A unique `UUIDv4` identifier for this connector instance. | +| Connector Name | `connector.name` | `CONNECTOR_NAME` | `Cloudflare Rules List` | No | Name of the connector as shown in OpenCTI. | +| Connector Scope | `connector.scope` | `CONNECTOR_SCOPE` | `cloudflare` | No | The scope of the stream connector (comma-separated). | +| Log Level | `connector.log_level` | `CONNECTOR_LOG_LEVEL` | `error` | No | `debug`, `info`, `warn`, `warning`, or `error`. | +| Live Stream ID | `connector.live_stream_id` | `CONNECTOR_LIVE_STREAM_ID` | / | Yes | The ID of the OpenCTI live stream to connect to (e.g. `live`). | +| Listen Delete | `connector.live_stream_listen_delete` | `CONNECTOR_LIVE_STREAM_LISTEN_DELETE` | `true` | No | Whether to receive delete events from the live stream. Must be `true` for the connector to drop removed IPs from the list. | +| No Dependencies | `connector.live_stream_no_dependencies` | `CONNECTOR_LIVE_STREAM_NO_DEPENDENCIES` | `true` | No | Whether to ignore object dependencies when processing live-stream events. | +| Sync Interval | `connector.sync_interval` | `CONNECTOR_SYNC_INTERVAL` | `1h` | No | Minimum interval between snapshot uploads (`30m`, `1h`, `1h30m`, seconds). | + +### Cloudflare configuration + +| Parameter | config.yml | Docker env var | Default | Mandatory | Description | +| -------------------- | ---------------------- | ----------------------- | ------- | --------- | --------------------------------------------------------------------- | +| Account ID | `cloudflare.account_id`| `CLOUDFLARE_ACCOUNT_ID` | / | Yes | Cloudflare account ID that owns the Rules List. | +| API Token | `cloudflare.api_token` | `CLOUDFLARE_API_TOKEN` | / | Yes | API token with `Account > Account Filter Lists > Edit` permission. | +| List ID | `cloudflare.list_id` | `CLOUDFLARE_LIST_ID` | / | Yes | ID of the existing Rules List (IP kind) to sync into. | +| API Base URL | `cloudflare.api_base_url` | `CLOUDFLARE_API_BASE_URL` | `https://api.cloudflare.com/client/v4` | No | Base URL of the Cloudflare API. Override only for testing or a compatible gateway. | + +> The Rules List must already exist and be of **IP** kind. Create it under +> Cloudflare → Manage Account → Configurations → Lists, then copy its ID. + +## Deployment + +### Docker Deployment + +Build the image and start the container: + +```shell +docker compose up -d +# -or, if you've made local changes- +docker compose up -d --build +``` + +### Manual Deployment + +Create a `config.yml` from the sample, fill in your values, then: + +```shell +cd src +pip install -r requirements.txt +python main.py +``` + +`git` must be available when installing requirements because `connectors-sdk` is +fetched from its repository. + +## Behavior + +1. **Verify** the configured Cloudflare Rules List exists (and log its kind). +2. **Full sync** on startup: load all IPv4 indicators and `IPv4-Addr` + observables from OpenCTI into an in-memory snapshot, then immediately push + that snapshot to Cloudflare. +3. **Listen** to the OpenCTI live stream — cache IPv4 values on create/update, + drop them on delete. +4. **Replace** the entire Cloudflare list with the current snapshot, then poll + the resulting bulk operation to completion. This push is triggered by + live-stream events and throttled to **at most once per + `CONNECTOR_SYNC_INTERVAL`** — an idle stream produces no uploads even after + the interval elapses. + +IPv4 values are extracted from three shapes: STIX indicators with an +`[ipv4-addr:value = '...']` pattern, STIX SCOs with `type: "ipv4-addr"`, and +OpenCTI observables with `entity_type: "IPv4-Addr"`. + +Each entry written to the Cloudflare list is tagged with a comment of the form +`OpenCTI: `, recording the source OpenCTI object ID. + +## Capabilities and limitations + +- **IPv4 only.** IPv6 addresses, domains, URLs, file hashes, and every other + indicator/observable type are ignored. Indicators match only when their STIX + pattern is `[ipv4-addr:value = '...']` (a single address, or a CIDR inside the + quotes); compound patterns contribute only their first IPv4 value. +- **No built-in score, label, confidence, or marking filtering.** Every IPv4 the + connector sees is pushed to the list. To restrict *which* entities reach the + connector, scope the OpenCTI **live stream definition** (e.g. by score or + labels) — the connector itself applies no filtering. +- **The startup full sync is not stream-filtered.** It loads *all* IPv4 + indicators and `IPv4-Addr` observables from the platform via the OpenCTI API, + regardless of the live stream's filters. Only the incremental live updates + honor the stream definition, so a filtered stream and the full sync can + disagree on what belongs in the list. +- **State is in-memory.** The snapshot is rebuilt by a full sync on every + restart; nothing is persisted locally. +- **Removals happen on delete events only.** A revoked or expired indicator is + dropped from the list when its delete event arrives, which requires + `CONNECTOR_LIVE_STREAM_LISTEN_DELETE=true`. +- **The Cloudflare list is owned by the connector.** Because each sync *replaces* + the entire list, any items added to it outside the connector are overwritten on + the next push. Use a dedicated list. + +## Development + +Run the test suite (requires Python 3.11/3.12): + +```shell +pip install -r tests/test-requirements.txt pytest-cov +pytest tests/ --cov=cloudflare_rules_list --cov-report=term-missing +``` + +`pytest-cov` is installed explicitly above (it is intentionally not listed in +`tests/test-requirements.txt`). diff --git a/stream/cloudflare-rules-list/__metadata__/CONNECTOR_CONFIG_DOC.md b/stream/cloudflare-rules-list/__metadata__/CONNECTOR_CONFIG_DOC.md new file mode 100644 index 0000000000..7856415362 --- /dev/null +++ b/stream/cloudflare-rules-list/__metadata__/CONNECTOR_CONFIG_DOC.md @@ -0,0 +1,22 @@ +# Connector Configurations + +Below is an exhaustive enumeration of all configurable parameters available, each accompanied by detailed explanations of their purposes, default behaviors, and usage guidelines to help you understand and utilize them effectively. + +### Type: `object` + +| Property | Type | Required | Possible values | Default | Description | +| -------- | ---- | -------- | --------------- | ------- | ----------- | +| OPENCTI_URL | `string` | ✅ | Format: [`uri`](https://json-schema.org/understanding-json-schema/reference/string#built-in-formats) | | The base URL of the OpenCTI instance. | +| OPENCTI_TOKEN | `string` | ✅ | string | | The API token to connect to OpenCTI. | +| CONNECTOR_LIVE_STREAM_ID | `string` | ✅ | string | | The ID of the live stream to connect to. | +| CLOUDFLARE_ACCOUNT_ID | `string` | ✅ | string | | Cloudflare account ID that owns the Rules List. | +| CLOUDFLARE_API_TOKEN | `string` | ✅ | Format: [`password`](https://json-schema.org/understanding-json-schema/reference/string#built-in-formats) | | Cloudflare API token with the 'Account > Account Filter Lists > Edit' permission. | +| CLOUDFLARE_LIST_ID | `string` | ✅ | string | | ID of the existing Cloudflare Rules List (IP kind) to sync into. | +| CONNECTOR_NAME | `string` | | string | `"Cloudflare Rules List"` | The name of the connector. | +| CONNECTOR_SCOPE | `array` | | string | `["cloudflare"]` | The scope of the stream connector. | +| CONNECTOR_LOG_LEVEL | `string` | | `debug` `info` `warn` `warning` `error` | `"error"` | The minimum level of logs to display. | +| CONNECTOR_TYPE | `const` | | `STREAM` | `"STREAM"` | | +| CONNECTOR_LIVE_STREAM_LISTEN_DELETE | `boolean` | | boolean | `true` | Whether to listen for delete events on the live stream. | +| CONNECTOR_LIVE_STREAM_NO_DEPENDENCIES | `boolean` | | boolean | `true` | Whether to ignore dependencies when processing events from the live stream. | +| CONNECTOR_SYNC_INTERVAL | `string` | | string | `"1h"` | Minimum interval between snapshot uploads to Cloudflare. Accepts a duration like '30m', '1h', '1h30m', or a bare number of seconds. | +| CLOUDFLARE_API_BASE_URL | `string` | | string | `"https://api.cloudflare.com/client/v4"` | Base URL of the Cloudflare API. Override only for testing against a mock server or a Cloudflare-compatible gateway. | diff --git a/stream/cloudflare-rules-list/__metadata__/connector_config_schema.json b/stream/cloudflare-rules-list/__metadata__/connector_config_schema.json new file mode 100644 index 0000000000..3c812b504c --- /dev/null +++ b/stream/cloudflare-rules-list/__metadata__/connector_config_schema.json @@ -0,0 +1,97 @@ +{ + "$schema": "https://json-schema.org/draft/2020-12/schema", + "$id": "https://www.filigran.io/connectors/cloudflare-rules-list_config.schema.json", + "type": "object", + "properties": { + "OPENCTI_URL": { + "description": "The base URL of the OpenCTI instance.", + "format": "uri", + "maxLength": 2083, + "minLength": 1, + "type": "string" + }, + "OPENCTI_TOKEN": { + "description": "The API token to connect to OpenCTI.", + "type": "string" + }, + "CONNECTOR_NAME": { + "default": "Cloudflare Rules List", + "description": "The name of the connector.", + "type": "string" + }, + "CONNECTOR_SCOPE": { + "default": [ + "cloudflare" + ], + "description": "The scope of the stream connector.", + "items": { + "type": "string" + }, + "type": "array" + }, + "CONNECTOR_LOG_LEVEL": { + "default": "error", + "description": "The minimum level of logs to display.", + "enum": [ + "debug", + "info", + "warn", + "warning", + "error" + ], + "type": "string" + }, + "CONNECTOR_TYPE": { + "const": "STREAM", + "default": "STREAM", + "type": "string" + }, + "CONNECTOR_LIVE_STREAM_ID": { + "description": "The ID of the live stream to connect to.", + "type": "string" + }, + "CONNECTOR_LIVE_STREAM_LISTEN_DELETE": { + "default": true, + "description": "Whether to listen for delete events on the live stream.", + "type": "boolean" + }, + "CONNECTOR_LIVE_STREAM_NO_DEPENDENCIES": { + "default": true, + "description": "Whether to ignore dependencies when processing events from the live stream.", + "type": "boolean" + }, + "CONNECTOR_SYNC_INTERVAL": { + "default": "1h", + "description": "Minimum interval between snapshot uploads to Cloudflare. Accepts a duration like '30m', '1h', '1h30m', or a bare number of seconds.", + "type": "string" + }, + "CLOUDFLARE_ACCOUNT_ID": { + "description": "Cloudflare account ID that owns the Rules List.", + "type": "string" + }, + "CLOUDFLARE_API_TOKEN": { + "description": "Cloudflare API token with the 'Account > Account Filter Lists > Edit' permission.", + "format": "password", + "type": "string", + "writeOnly": true + }, + "CLOUDFLARE_LIST_ID": { + "description": "ID of the existing Cloudflare Rules List (IP kind) to sync into.", + "type": "string" + }, + "CLOUDFLARE_API_BASE_URL": { + "default": "https://api.cloudflare.com/client/v4", + "description": "Base URL of the Cloudflare API. Override only for testing against a mock server or a Cloudflare-compatible gateway.", + "type": "string" + } + }, + "required": [ + "OPENCTI_URL", + "OPENCTI_TOKEN", + "CONNECTOR_LIVE_STREAM_ID", + "CLOUDFLARE_ACCOUNT_ID", + "CLOUDFLARE_API_TOKEN", + "CLOUDFLARE_LIST_ID" + ], + "additionalProperties": true +} diff --git a/stream/cloudflare-rules-list/__metadata__/connector_manifest.json b/stream/cloudflare-rules-list/__metadata__/connector_manifest.json new file mode 100644 index 0000000000..f425a6692d --- /dev/null +++ b/stream/cloudflare-rules-list/__metadata__/connector_manifest.json @@ -0,0 +1,21 @@ +{ + "title": "Cloudflare Rules List", + "slug": "cloudflare-rules-list", + "description": "Cloudflare Rules Lists are reusable collections of IP addresses that can be referenced from WAF custom rules, firewall rules, and other security configurations across a Cloudflare account.\n\nThe integration of Cloudflare with OpenCTI enables the automatic dissemination of IPv4 threat-intelligence indicators and observables into a Cloudflare Rules List. The connector consumes IPv4 indicators from an OpenCTI live stream, maintains an in-memory snapshot, and periodically replaces the contents of a target Cloudflare Rules List (IP kind) using Cloudflare's bulk list-items API documented at https://developers.cloudflare.com/api/resources/rules/subresources/lists/.", + "short_description": "Push IPv4 threat-intelligence indicators from OpenCTI into a Cloudflare Rules List for use in WAF and firewall rules.", + "logo": "stream/cloudflare-rules-list/__metadata__/logo.png", + "use_cases": [ + "Threat Intelligence" + ], + "verified": false, + "last_verified_date": null, + "playbook_supported": false, + "max_confidence_level": 50, + "support_version": ">= 6.7.5", + "subscription_link": "https://developers.cloudflare.com/waf/tools/lists/", + "source_code": "https://github.com/OpenCTI-Platform/connectors/tree/master/stream/cloudflare-rules-list", + "manager_supported": true, + "container_version": "rolling", + "container_image": "opencti/connector-cloudflare-rules-list", + "container_type": "STREAM" +} diff --git a/stream/cloudflare-rules-list/__metadata__/logo.png b/stream/cloudflare-rules-list/__metadata__/logo.png new file mode 100644 index 0000000000..01ce9d3028 Binary files /dev/null and b/stream/cloudflare-rules-list/__metadata__/logo.png differ diff --git a/stream/cloudflare-rules-list/docker-compose.yml b/stream/cloudflare-rules-list/docker-compose.yml new file mode 100644 index 0000000000..5a0b104a9c --- /dev/null +++ b/stream/cloudflare-rules-list/docker-compose.yml @@ -0,0 +1,25 @@ +services: + connector-cloudflare-rules-list: + image: opencti/connector-cloudflare-rules-list:rolling + container_name: opencti-connector-cloudflare-rules-list + restart: unless-stopped + environment: + # OpenCTI + - OPENCTI_URL=${OPENCTI_URL} + - OPENCTI_TOKEN=${OPENCTI_TOKEN} + + # Connector + - CONNECTOR_ID=${CONNECTOR_ID} + - CONNECTOR_TYPE=STREAM + - CONNECTOR_NAME=Cloudflare Rules List + - CONNECTOR_SCOPE=cloudflare + - CONNECTOR_LOG_LEVEL=${CONNECTOR_LOG_LEVEL:-info} + - CONNECTOR_LIVE_STREAM_ID=${CONNECTOR_LIVE_STREAM_ID:-live} + - CONNECTOR_LIVE_STREAM_LISTEN_DELETE=true + - CONNECTOR_LIVE_STREAM_NO_DEPENDENCIES=true + - CONNECTOR_SYNC_INTERVAL=${CONNECTOR_SYNC_INTERVAL:-1h} + + # Cloudflare + - CLOUDFLARE_ACCOUNT_ID=${CLOUDFLARE_ACCOUNT_ID} + - CLOUDFLARE_API_TOKEN=${CLOUDFLARE_API_TOKEN} + - CLOUDFLARE_LIST_ID=${CLOUDFLARE_LIST_ID} diff --git a/stream/cloudflare-rules-list/entrypoint.sh b/stream/cloudflare-rules-list/entrypoint.sh new file mode 100644 index 0000000000..b450492dd1 --- /dev/null +++ b/stream/cloudflare-rules-list/entrypoint.sh @@ -0,0 +1,3 @@ +#!/bin/sh +cd /opt/opencti-connector-cloudflare-rules-list +python3 main.py diff --git a/stream/cloudflare-rules-list/src/cloudflare_rules_list/__init__.py b/stream/cloudflare-rules-list/src/cloudflare_rules_list/__init__.py new file mode 100644 index 0000000000..53116f14c9 --- /dev/null +++ b/stream/cloudflare-rules-list/src/cloudflare_rules_list/__init__.py @@ -0,0 +1,11 @@ +"""OpenCTI -> Cloudflare Rules List stream connector (v2). + +Pushes IPv4 threat-intelligence indicators from OpenCTI into a Cloudflare +Rules List, built on the modern OpenCTI ``connectors-sdk`` + Pydantic-settings +architecture (OpenCTI 7.x / pycti 7.x). +""" + +from .connector import Connector +from .settings import ConnectorSettings + +__all__ = ["Connector", "ConnectorSettings"] diff --git a/stream/cloudflare-rules-list/src/cloudflare_rules_list/client.py b/stream/cloudflare-rules-list/src/cloudflare_rules_list/client.py new file mode 100644 index 0000000000..cdf050dc46 --- /dev/null +++ b/stream/cloudflare-rules-list/src/cloudflare_rules_list/client.py @@ -0,0 +1,170 @@ +"""Cloudflare Rules Lists API client. + +Documentation: +https://developers.cloudflare.com/api/resources/rules/subresources/lists/ +""" + +import json +import time +from typing import Any, Optional + +import requests + + +class CloudflareAPIError(Exception): + """Exception raised for Cloudflare API errors.""" + + +class CloudflareRulesListClient: + """Client for the Cloudflare Rules Lists API.""" + + BASE_URL = "https://api.cloudflare.com/client/v4" + + def __init__( + self, + account_id: str, + api_token: str, + timeout: int = 60, + base_url: Optional[str] = None, + ): + """Initialize the client. + + Args: + account_id: Cloudflare account ID. + api_token: API token (Bearer auth). + timeout: Default request timeout in seconds. + base_url: Override the Cloudflare API base URL (for testing or a + compatible gateway). Defaults to the public Cloudflare API. + """ + self.account_id = account_id + self.api_token = api_token + self.timeout = timeout + self.base_url = (base_url or self.BASE_URL).rstrip("/") + self._session = requests.Session() + self._session.verify = True # explicit for security scanners + self._session.headers.update( + { + "Authorization": f"Bearer {self.api_token}", + "Content-Type": "application/json", + } + ) + + def _make_request( + self, + method: str, + endpoint: str, + data: Optional[Any] = None, + timeout: Optional[int] = None, + ) -> dict: + """Make an API request to Cloudflare and return the parsed JSON body.""" + url = f"{self.base_url}/accounts/{self.account_id}{endpoint}" + request_timeout = timeout or self.timeout + + try: + response = self._session.request( + method=method, + url=url, + json=data, + timeout=request_timeout, + ) + response.raise_for_status() + except requests.exceptions.RequestException as exc: + error_msg = str(exc) + err_response = getattr(exc, "response", None) + if err_response is not None: + try: + error_data = err_response.json() + if "errors" in error_data: + error_msg = str(error_data["errors"]) + except (json.JSONDecodeError, ValueError): + error_msg = err_response.text + raise CloudflareAPIError(f"API request failed: {error_msg}") from exc + + try: + return response.json() + except (json.JSONDecodeError, ValueError) as exc: + raise CloudflareAPIError( + f"Invalid JSON in Cloudflare response: {response.text[:200]}" + ) from exc + + def list_lists(self) -> list[dict]: + """List all rules lists in the account.""" + response = self._make_request("GET", "/rules/lists") + return response.get("result", []) + + def get_list(self, list_id: str) -> dict: + """Get a specific list's metadata.""" + response = self._make_request("GET", f"/rules/lists/{list_id}") + return response.get("result", {}) + + def get_list_items(self, list_id: str, cursor: Optional[str] = None) -> dict: + """Get a page of items from a list.""" + endpoint = f"/rules/lists/{list_id}/items" + if cursor: + endpoint += f"?cursor={cursor}" + return self._make_request("GET", endpoint) + + def get_all_list_items(self, list_id: str) -> list[dict]: + """Get all items from a list, following pagination cursors.""" + all_items: list[dict] = [] + cursor = None + + while True: + response = self.get_list_items(list_id, cursor) + all_items.extend(response.get("result", [])) + + result_info = response.get("result_info", {}) + cursor = result_info.get("cursors", {}).get("after") + if not cursor: + break + + return all_items + + def replace_list_items(self, list_id: str, items: list[dict]) -> dict: + """Replace ALL items in a list with the provided items (snapshot). + + Args: + list_id: The list ID. + items: Items in the kind-specific format, e.g. for an IP list: + ``[{"ip": "192.0.2.1"}, {"ip": "10.0.0.0/8"}]``. + + Returns: + Operation result, including an ``operation_id`` for the async bulk job. + """ + response = self._make_request( + "PUT", f"/rules/lists/{list_id}/items", data=items, timeout=300 + ) + return response.get("result", {}) + + def get_bulk_operation(self, operation_id: str) -> dict: + """Get the status of a bulk operation.""" + response = self._make_request( + "GET", f"/rules/lists/bulk_operations/{operation_id}" + ) + return response.get("result", {}) + + def wait_for_operation( + self, operation_id: str, timeout: int = 300, poll_interval: int = 2 + ) -> dict: + """Block until a bulk operation completes. + + Raises: + CloudflareAPIError: If the operation fails or times out. + """ + start_time = time.monotonic() + + while True: + status = self.get_bulk_operation(operation_id) + state = status.get("status") + + if state == "completed": + return status + if state == "failed": + raise CloudflareAPIError( + f"Bulk operation failed: {status.get('error')}" + ) + + if time.monotonic() - start_time > timeout: + raise CloudflareAPIError(f"Bulk operation timed out after {timeout}s") + + time.sleep(poll_interval) diff --git a/stream/cloudflare-rules-list/src/cloudflare_rules_list/connector.py b/stream/cloudflare-rules-list/src/cloudflare_rules_list/connector.py new file mode 100644 index 0000000000..a8c9146a40 --- /dev/null +++ b/stream/cloudflare-rules-list/src/cloudflare_rules_list/connector.py @@ -0,0 +1,272 @@ +"""OpenCTI -> Cloudflare Rules List stream connector. + +Listens to the OpenCTI live stream for IPv4 indicators/observables, maintains an +in-memory snapshot of all known IPv4 values, and periodically pushes the full +snapshot to a Cloudflare Rules List (snapshot/replace model). +""" + +import json +import re +import sys +import time +from typing import Optional + +from pycti import OpenCTIConnectorHelper + +from .client import CloudflareAPIError, CloudflareRulesListClient +from .settings import ConnectorSettings + +# STIX pattern for an IPv4 indicator: [ipv4-addr:value = '...'] +_IPV4_PATTERN_RE = re.compile(r"\[ipv4-addr:value\s*=\s*'([^']+)'\]", re.IGNORECASE) + + +def _parse_interval(interval_str: str) -> int: + """Parse an interval like '1h', '30m', '1h30m', or a bare number of seconds. + + Returns the total number of seconds, defaulting to 3600 if parsing fails. + """ + interval_str = (interval_str or "").lower().strip() + total_seconds = 0 + + hours = re.search(r"(\d+)h", interval_str) + if hours: + total_seconds += int(hours.group(1)) * 3600 + + minutes = re.search(r"(\d+)m", interval_str) + if minutes: + total_seconds += int(minutes.group(1)) * 60 + + seconds = re.search(r"(\d+)s", interval_str) + if seconds: + total_seconds += int(seconds.group(1)) + + if total_seconds == 0 and interval_str.isdigit(): + total_seconds = int(interval_str) + + return total_seconds if total_seconds > 0 else 3600 + + +class Connector: + """OpenCTI connector for Cloudflare Rules Lists (IPv4).""" + + def __init__( + self, + helper: OpenCTIConnectorHelper, + config: ConnectorSettings, + client: CloudflareRulesListClient, + ): + self.helper = helper + self.config = config + self.client = client + self.logger = helper.connector_logger + + self.list_id = config.cloudflare.list_id + self.sync_interval = _parse_interval(config.connector.sync_interval) + + # Snapshot of IPv4 values keyed by OpenCTI id. + self._indicator_cache: dict[str, str] = {} + self._last_sync_time = 0.0 + + # ------------------------------------------------------------------ # + # IPv4 extraction + # ------------------------------------------------------------------ # + def _extract_ipv4(self, data: dict) -> Optional[str]: + """Return the IPv4 value from an OpenCTI/STIX object, or None. + + Two representations reach this method with different type keys: + * Live-stream / STIX shape uses the lowercase STIX ``type`` field + (``"indicator"``, ``"ipv4-addr"``). + * OpenCTI API objects (full sync via ``helper.api.*.list``) use the + capitalized ``entity_type`` field (``"Indicator"``, ``"IPv4-Addr"``) + and leave ``type`` unset. + + Both indicator shapes carry a STIX ``pattern``; both observable shapes + carry the address in ``value`` / ``observable_value``. + """ + stix_type = data.get("type", "") + entity_type = data.get("entity_type", "") + + # Indicator (stream: type=="indicator"; API: entity_type=="Indicator"). + if stix_type == "indicator" or entity_type == "Indicator": + pattern = data.get("pattern", "") + match = _IPV4_PATTERN_RE.search(pattern) if pattern else None + return match.group(1) if match else None + + # IPv4 observable (stream: type=="ipv4-addr"; API: entity_type=="IPv4-Addr"). + observable_type = entity_type or stix_type + if observable_type in ("ipv4-addr", "IPv4-Addr"): + return data.get("value") or data.get("observable_value") + + return None + + @staticmethod + def _object_id(data: dict) -> Optional[str]: + """Return the OpenCTI id for a stream/STIX object.""" + return data.get("id") or data.get("x_opencti_id") + + # ------------------------------------------------------------------ # + # Stream handling + # ------------------------------------------------------------------ # + def process_message(self, msg) -> None: + """Callback for each OpenCTI live-stream event.""" + try: + try: + data = json.loads(msg.data)["data"] + except (json.JSONDecodeError, KeyError, TypeError): + self.logger.warning("Could not parse stream message data") + return + + # The initial catch-up event type may be "message"; treat as create. + event_type = msg.event if getattr(msg, "event", None) else "create" + if event_type == "message": + event_type = "create" + + if event_type in ("create", "update"): + self._handle_upsert(data) + elif event_type == "delete": + self._handle_delete(data) + except (KeyboardInterrupt, SystemExit): + self.logger.info("Connector stopped") + sys.exit(0) + except Exception as exc: # noqa: BLE001 - never let the stream die + self.logger.error( + "Error processing stream message", meta={"error": str(exc)} + ) + + def _handle_upsert(self, data: dict) -> None: + value = self._extract_ipv4(data) + if not value: + return + + indicator_id = self._object_id(data) + if not indicator_id: + return + + self._indicator_cache[indicator_id] = value + self.logger.debug( + "Cached IPv4 indicator", meta={"id": indicator_id, "value": value} + ) + self._check_sync() + + def _handle_delete(self, data: dict) -> None: + indicator_id = self._object_id(data) + if indicator_id and indicator_id in self._indicator_cache: + del self._indicator_cache[indicator_id] + self.logger.debug("Removed indicator from cache", meta={"id": indicator_id}) + self._check_sync() + + # ------------------------------------------------------------------ # + # Sync to Cloudflare + # ------------------------------------------------------------------ # + def _check_sync(self) -> None: + """Sync to Cloudflare if the configured interval has elapsed.""" + if time.monotonic() - self._last_sync_time >= self.sync_interval: + self._sync_to_cloudflare() + + def _sync_to_cloudflare(self) -> None: + """Push the full IPv4 snapshot to the Cloudflare Rules List.""" + if not self._indicator_cache: + # Nothing to push -- do not open the throttle window, otherwise the + # first real indicator to arrive could be delayed by up to + # sync_interval before it is synced. + self.logger.info("No indicators to sync") + return + + self._last_sync_time = time.monotonic() + + self.logger.info( + "Syncing indicators to Cloudflare", + meta={"count": len(self._indicator_cache), "list_id": self.list_id}, + ) + + items = [ + {"ip": value, "comment": f"OpenCTI: {indicator_id}"} + for indicator_id, value in self._indicator_cache.items() + ] + + try: + result = self.client.replace_list_items(self.list_id, items) + operation_id = result.get("operation_id") + if operation_id: + self.logger.info( + "Bulk operation started", meta={"operation_id": operation_id} + ) + final_status = self.client.wait_for_operation(operation_id) + self.logger.info( + "Snapshot uploaded", + meta={"count": len(items), "status": final_status.get("status")}, + ) + else: + self.logger.info("Snapshot uploaded", meta={"count": len(items)}) + except CloudflareAPIError as exc: + self.logger.error("Failed to sync to Cloudflare", meta={"error": str(exc)}) + + # ------------------------------------------------------------------ # + # Full sync (startup) + # ------------------------------------------------------------------ # + def _full_sync(self) -> None: + """Load all IPv4 indicators and observables from OpenCTI, then sync.""" + self.logger.info("Starting full sync from OpenCTI") + self._indicator_cache = {} + + indicators = self.helper.api.indicator.list(getAll=True) + self.logger.info( + "Fetched indicators from OpenCTI", meta={"count": len(indicators)} + ) + for indicator in indicators: + value = self._extract_ipv4(indicator) + indicator_id = indicator.get("id") + if value and indicator_id: + self._indicator_cache[indicator_id] = value + + try: + observables = self.helper.api.stix_cyber_observable.list( + types=["IPv4-Addr"], getAll=True + ) + for observable in observables: + value = self._extract_ipv4(observable) + obs_id = observable.get("id") + if value and obs_id: + self._indicator_cache[obs_id] = value + except Exception as exc: # noqa: BLE001 + self.logger.warning( + "Could not fetch IPv4 observables", meta={"error": str(exc)} + ) + + self.logger.info( + "Loaded IPv4 indicators for sync", + meta={"count": len(self._indicator_cache)}, + ) + self._sync_to_cloudflare() + + # ------------------------------------------------------------------ # + # Lifecycle + # ------------------------------------------------------------------ # + def run(self) -> None: + """Verify the target list, full-sync, then listen to the live stream.""" + self.logger.info("Starting Cloudflare Rules List connector") + + # Verify the configured list exists before doing any work. + try: + list_info = self.client.get_list(self.list_id) + self.logger.info( + "Using Cloudflare list", + meta={ + "name": list_info.get("name"), + "id": self.list_id, + "kind": list_info.get("kind"), + }, + ) + except CloudflareAPIError as exc: + self.logger.error( + "Could not find Cloudflare list", + meta={"list_id": self.list_id, "error": str(exc)}, + ) + raise + + try: + self._full_sync() + except Exception as exc: # noqa: BLE001 + self.logger.error("Initial full sync failed", meta={"error": str(exc)}) + + self.helper.listen_stream(message_callback=self.process_message) diff --git a/stream/cloudflare-rules-list/src/cloudflare_rules_list/settings.py b/stream/cloudflare-rules-list/src/cloudflare_rules_list/settings.py new file mode 100644 index 0000000000..5cba388d2a --- /dev/null +++ b/stream/cloudflare-rules-list/src/cloudflare_rules_list/settings.py @@ -0,0 +1,87 @@ +"""Pydantic-settings configuration models for the connector. + +Configuration is loaded (in precedence order) from environment variables, then +``config.yml`` or ``.env`` next to the connector, then field defaults -- the +same precedence the legacy ``get_config_variable`` helper used. + +Environment variable mapping (nested by the first underscore): + + OPENCTI_URL -> opencti.url + OPENCTI_TOKEN -> opencti.token + CONNECTOR_ID -> connector.id + CONNECTOR_NAME -> connector.name + CONNECTOR_SCOPE -> connector.scope + CONNECTOR_LIVE_STREAM_ID -> connector.live_stream_id + CONNECTOR_SYNC_INTERVAL -> connector.sync_interval + CLOUDFLARE_ACCOUNT_ID -> cloudflare.account_id + CLOUDFLARE_API_TOKEN -> cloudflare.api_token + CLOUDFLARE_LIST_ID -> cloudflare.list_id +""" + +from connectors_sdk import ( + BaseConfigModel, + BaseConnectorSettings, + BaseStreamConnectorConfig, + ListFromString, +) +from pydantic import Field, SecretStr + + +class _StreamConnectorConfig(BaseStreamConnectorConfig): + """Stream connector namespace (``connector.*``). + + Inherits ``id``, ``type`` (forced to ``STREAM``), ``log_level``, + ``live_stream_id``, ``live_stream_listen_delete`` and + ``live_stream_no_dependencies`` from the SDK base class. + """ + + name: str = Field( + default="Cloudflare Rules List", + description="The name of the connector as shown in OpenCTI.", + ) + scope: ListFromString = Field( + default=["cloudflare"], + description="Connector scope (comma-separated).", + ) + sync_interval: str = Field( + default="1h", + description=( + "Minimum interval between snapshot uploads to Cloudflare. " + "Accepts a duration like '30m', '1h', '1h30m', or a bare number " + "of seconds." + ), + ) + + +class CloudflareConfig(BaseConfigModel): + """Cloudflare namespace (``cloudflare.*``).""" + + account_id: str = Field( + description="Cloudflare account ID that owns the Rules List.", + ) + api_token: SecretStr = Field( + description=( + "Cloudflare API token with the 'Account > Account Filter Lists > " + "Edit' permission." + ), + ) + list_id: str = Field( + description="ID of the existing Cloudflare Rules List (IP kind) to sync into.", + ) + api_base_url: str = Field( + default="https://api.cloudflare.com/client/v4", + description=( + "Base URL of the Cloudflare API. Override only for testing against a " + "mock server or a Cloudflare-compatible gateway." + ), + ) + + +class ConnectorSettings(BaseConnectorSettings): + """Top-level settings. + + ``opencti`` (url, token) is provided by :class:`BaseConnectorSettings`. + """ + + connector: _StreamConnectorConfig = Field(default_factory=_StreamConnectorConfig) + cloudflare: CloudflareConfig = Field(default_factory=CloudflareConfig) diff --git a/stream/cloudflare-rules-list/src/config.yml.sample b/stream/cloudflare-rules-list/src/config.yml.sample new file mode 100644 index 0000000000..17b0b48c1c --- /dev/null +++ b/stream/cloudflare-rules-list/src/config.yml.sample @@ -0,0 +1,26 @@ +opencti: + url: 'http://localhost:8080' + token: 'ChangeMe' + +connector: + id: 'ChangeMe' # Unique connector UUID + type: 'STREAM' + name: 'Cloudflare Rules List' + scope: 'cloudflare' + log_level: 'info' # debug | info | warn | error + live_stream_id: 'live' # OpenCTI live stream ID + live_stream_listen_delete: true + live_stream_no_dependencies: true + # Minimum interval between snapshot uploads to Cloudflare. + # Accepts '30m', '1h', '1h30m', or a bare number of seconds. + sync_interval: '1h' + +cloudflare: + # Cloudflare account ID that owns the Rules List. + account_id: 'ChangeMe' + # API token with 'Account > Account Filter Lists > Edit' permission. + api_token: 'ChangeMe' + # ID of the existing Cloudflare Rules List (IP kind) to sync into. + list_id: 'ChangeMe' + # Base URL of the Cloudflare API. Override only for testing or a compatible gateway. + # api_base_url: 'https://api.cloudflare.com/client/v4' diff --git a/stream/cloudflare-rules-list/src/main.py b/stream/cloudflare-rules-list/src/main.py new file mode 100644 index 0000000000..9a672b529f --- /dev/null +++ b/stream/cloudflare-rules-list/src/main.py @@ -0,0 +1,23 @@ +#!/usr/bin/env python3 +"""Entry point for the OpenCTI -> Cloudflare Rules List stream connector (v2).""" + +import traceback + +from cloudflare_rules_list import Connector, ConnectorSettings +from cloudflare_rules_list.client import CloudflareRulesListClient +from pycti import OpenCTIConnectorHelper + +if __name__ == "__main__": + try: + settings = ConnectorSettings() + helper = OpenCTIConnectorHelper(config=settings.to_helper_config()) + client = CloudflareRulesListClient( + account_id=settings.cloudflare.account_id, + api_token=settings.cloudflare.api_token.get_secret_value(), + base_url=settings.cloudflare.api_base_url, + ) + connector = Connector(helper=helper, config=settings, client=client) + connector.run() + except Exception: + traceback.print_exc() + exit(1) diff --git a/stream/cloudflare-rules-list/src/requirements.txt b/stream/cloudflare-rules-list/src/requirements.txt new file mode 100644 index 0000000000..93f2df0452 --- /dev/null +++ b/stream/cloudflare-rules-list/src/requirements.txt @@ -0,0 +1,12 @@ +# OpenCTI Python client (CalVer; must be compatible with your OpenCTI 7.x platform) +pycti==7.260715.0 + +# Modern connector SDK (Pydantic-settings base classes, shared types) +connectors-sdk @ git+https://github.com/OpenCTI-Platform/connectors.git@master#subdirectory=connectors-sdk + +# Configuration +pydantic~=2.11.5 +pydantic-settings~=2.9.1 + +# HTTP client +requests>=2.31.0,<3.0.0 diff --git a/stream/cloudflare-rules-list/tests/__init__.py b/stream/cloudflare-rules-list/tests/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/stream/cloudflare-rules-list/tests/conftest.py b/stream/cloudflare-rules-list/tests/conftest.py new file mode 100644 index 0000000000..5ee8fc0e22 --- /dev/null +++ b/stream/cloudflare-rules-list/tests/conftest.py @@ -0,0 +1,4 @@ +import os +import sys + +sys.path.append(os.path.join(os.path.dirname(__file__), "..", "src")) diff --git a/stream/cloudflare-rules-list/tests/test-requirements.txt b/stream/cloudflare-rules-list/tests/test-requirements.txt new file mode 100644 index 0000000000..96e880d6df --- /dev/null +++ b/stream/cloudflare-rules-list/tests/test-requirements.txt @@ -0,0 +1,3 @@ +-r ../src/requirements.txt +pytest +pytest-mock diff --git a/stream/cloudflare-rules-list/tests/test_client.py b/stream/cloudflare-rules-list/tests/test_client.py new file mode 100644 index 0000000000..7d65c759e6 --- /dev/null +++ b/stream/cloudflare-rules-list/tests/test_client.py @@ -0,0 +1,196 @@ +from unittest.mock import MagicMock + +import pytest +import requests +from cloudflare_rules_list.client import CloudflareAPIError, CloudflareRulesListClient + + +@pytest.fixture +def client(): + c = CloudflareRulesListClient(account_id="acc-1", api_token="token-1", timeout=10) + c._session = MagicMock() + return c + + +def _response(json_body=None, raise_exc=None): + resp = MagicMock() + resp.json.return_value = json_body or {} + if raise_exc is not None: + resp.raise_for_status.side_effect = raise_exc + return resp + + +def test_auth_header_is_set(): + c = CloudflareRulesListClient(account_id="acc-1", api_token="secret") + assert c._session.headers["Authorization"] == "Bearer secret" + + +def test_default_base_url(): + c = CloudflareRulesListClient(account_id="acc-1", api_token="secret") + assert c.base_url == CloudflareRulesListClient.BASE_URL + + +def test_base_url_override_strips_trailing_slash(): + c = CloudflareRulesListClient( + account_id="acc-1", + api_token="secret", + base_url="https://gateway.example/client/v4/", + ) + assert c.base_url == "https://gateway.example/client/v4" + + +def test_session_verify_is_true(): + c = CloudflareRulesListClient(account_id="acc-1", api_token="secret") + assert c._session.verify is True + + +def test_make_request_uses_overridden_base_url(): + c = CloudflareRulesListClient( + account_id="acc-1", api_token="secret", base_url="http://mock:9999/v4" + ) + c._session = MagicMock() + c._session.request.return_value = _response({"result": {}}) + c._make_request("GET", "/rules/lists/abc") + assert ( + c._session.request.call_args.kwargs["url"] + == "http://mock:9999/v4/accounts/acc-1/rules/lists/abc" + ) + + +def test_make_request_success(client): + client._session.request.return_value = _response({"result": {"ok": True}}) + result = client._make_request("GET", "/rules/lists/abc") + assert result == {"result": {"ok": True}} + args, kwargs = client._session.request.call_args + assert kwargs["url"].endswith("/accounts/acc-1/rules/lists/abc") + + +def test_make_request_raises_on_non_json_success(client): + # 2xx response whose body is not JSON (e.g. an HTML challenge page) must be + # wrapped as CloudflareAPIError, not surface a bare ValueError. + resp = _response() + resp.json.side_effect = ValueError("no json") + resp.text = "challenge" + client._session.request.return_value = resp + with pytest.raises(CloudflareAPIError) as exc: + client._make_request("GET", "/x") + assert "Invalid JSON" in str(exc.value) + + +def test_make_request_uses_custom_timeout(client): + client._session.request.return_value = _response({"result": {}}) + client._make_request("PUT", "/x", data=[], timeout=300) + assert client._session.request.call_args.kwargs["timeout"] == 300 + + +def test_make_request_raises_with_structured_errors(client): + err = requests.exceptions.HTTPError("400") + err.response = MagicMock() + err.response.json.return_value = {"errors": [{"code": 10001, "message": "bad"}]} + client._session.request.return_value = _response(raise_exc=err) + + with pytest.raises(CloudflareAPIError) as exc: + client._make_request("GET", "/x") + assert "10001" in str(exc.value) + + +def test_make_request_raises_with_text_body(client): + err = requests.exceptions.HTTPError("500") + err.response = MagicMock() + err.response.json.side_effect = ValueError("not json") + err.response.text = "Internal Server Error" + client._session.request.return_value = _response(raise_exc=err) + + with pytest.raises(CloudflareAPIError) as exc: + client._make_request("GET", "/x") + assert "Internal Server Error" in str(exc.value) + + +def test_make_request_raises_without_response(client): + err = requests.exceptions.ConnectionError("no network") + client._session.request.return_value = _response(raise_exc=err) + with pytest.raises(CloudflareAPIError) as exc: + client._make_request("GET", "/x") + assert "no network" in str(exc.value) + + +def test_list_lists(client): + client._session.request.return_value = _response({"result": [{"id": "l1"}]}) + assert client.list_lists() == [{"id": "l1"}] + + +def test_get_list(client): + client._session.request.return_value = _response({"result": {"id": "l1"}}) + assert client.get_list("l1") == {"id": "l1"} + + +def test_get_list_items_with_cursor(client): + client._session.request.return_value = _response({"result": []}) + client.get_list_items("l1", cursor="CURSOR") + assert "cursor=CURSOR" in client._session.request.call_args.kwargs["url"] + + +def test_get_all_list_items_follows_pagination(client): + page1 = _response( + {"result": [{"ip": "1.1.1.1"}], "result_info": {"cursors": {"after": "C2"}}} + ) + page2 = _response({"result": [{"ip": "2.2.2.2"}], "result_info": {"cursors": {}}}) + client._session.request.side_effect = [page1, page2] + + items = client.get_all_list_items("l1") + assert items == [{"ip": "1.1.1.1"}, {"ip": "2.2.2.2"}] + + +def test_replace_list_items(client): + client._session.request.return_value = _response({"result": {"operation_id": "op"}}) + result = client.replace_list_items("l1", [{"ip": "1.1.1.1"}]) + assert result == {"operation_id": "op"} + assert client._session.request.call_args.kwargs["method"] == "PUT" + + +def test_get_bulk_operation(client): + client._session.request.return_value = _response( + {"result": {"status": "completed"}} + ) + assert client.get_bulk_operation("op") == {"status": "completed"} + + +def test_wait_for_operation_completed(client, monkeypatch): + monkeypatch.setattr("cloudflare_rules_list.client.time.monotonic", lambda: 0.0) + client._session.request.return_value = _response( + {"result": {"status": "completed"}} + ) + assert client.wait_for_operation("op") == {"status": "completed"} + + +def test_wait_for_operation_failed(client, monkeypatch): + monkeypatch.setattr("cloudflare_rules_list.client.time.monotonic", lambda: 0.0) + client._session.request.return_value = _response( + {"result": {"status": "failed", "error": "boom"}} + ) + with pytest.raises(CloudflareAPIError) as exc: + client.wait_for_operation("op") + assert "boom" in str(exc.value) + + +def test_wait_for_operation_times_out(client, monkeypatch): + times = iter([0.0, 0.0, 100.0]) + monkeypatch.setattr( + "cloudflare_rules_list.client.time.monotonic", lambda: next(times) + ) + monkeypatch.setattr("cloudflare_rules_list.client.time.sleep", lambda _: None) + client._session.request.return_value = _response({"result": {"status": "pending"}}) + + with pytest.raises(CloudflareAPIError) as exc: + client.wait_for_operation("op", timeout=10) + assert "timed out" in str(exc.value) + + +def test_wait_for_operation_polls_until_complete(client, monkeypatch): + monkeypatch.setattr("cloudflare_rules_list.client.time.monotonic", lambda: 0.0) + monkeypatch.setattr("cloudflare_rules_list.client.time.sleep", lambda _: None) + client._session.request.side_effect = [ + _response({"result": {"status": "pending"}}), + _response({"result": {"status": "completed"}}), + ] + assert client.wait_for_operation("op") == {"status": "completed"} diff --git a/stream/cloudflare-rules-list/tests/test_connector.py b/stream/cloudflare-rules-list/tests/test_connector.py new file mode 100644 index 0000000000..c1fb76f358 --- /dev/null +++ b/stream/cloudflare-rules-list/tests/test_connector.py @@ -0,0 +1,320 @@ +import json +from types import SimpleNamespace +from unittest.mock import MagicMock + +import pytest +from cloudflare_rules_list.client import CloudflareAPIError +from cloudflare_rules_list.connector import Connector, _parse_interval + + +# --------------------------------------------------------------------------- # +# _parse_interval +# --------------------------------------------------------------------------- # +@pytest.mark.parametrize( + "value, expected", + [ + ("1h", 3600), + ("30m", 1800), + ("1h30m", 5400), + ("45s", 45), + ("90", 90), + ("", 3600), + ("garbage", 3600), + ("0", 3600), + (None, 3600), + ], +) +def test_parse_interval(value, expected): + assert _parse_interval(value) == expected + + +# --------------------------------------------------------------------------- # +# Fixtures +# --------------------------------------------------------------------------- # +@pytest.fixture +def helper(): + h = MagicMock() + h.connector_logger = MagicMock() + return h + + +@pytest.fixture +def client(): + return MagicMock() + + +@pytest.fixture +def config(): + return SimpleNamespace( + cloudflare=SimpleNamespace(list_id="list-123"), + connector=SimpleNamespace(sync_interval="1h"), + ) + + +@pytest.fixture +def connector(helper, config, client): + conn = Connector(helper=helper, config=config, client=client) + # Force sync to trigger on demand in tests. + conn.sync_interval = 0 + return conn + + +# --------------------------------------------------------------------------- # +# _extract_ipv4 +# --------------------------------------------------------------------------- # +def test_extract_ipv4_from_indicator_pattern(connector): + data = {"type": "indicator", "pattern": "[ipv4-addr:value = '192.0.2.1']"} + assert connector._extract_ipv4(data) == "192.0.2.1" + + +def test_extract_ipv4_from_api_indicator(connector): + # OpenCTI API (full-sync) shape: entity_type=="Indicator", no lowercase type. + data = {"entity_type": "Indicator", "pattern": "[ipv4-addr:value = '192.0.2.30']"} + assert connector._extract_ipv4(data) == "192.0.2.30" + + +def test_extract_ipv4_from_api_indicator_no_match(connector): + data = {"entity_type": "Indicator", "pattern": "[domain-name:value = 'evil.com']"} + assert connector._extract_ipv4(data) is None + + +def test_extract_ipv4_from_indicator_no_match(connector): + data = {"type": "indicator", "pattern": "[domain-name:value = 'evil.com']"} + assert connector._extract_ipv4(data) is None + + +def test_extract_ipv4_from_indicator_empty_pattern(connector): + assert connector._extract_ipv4({"type": "indicator"}) is None + + +def test_extract_ipv4_from_stix_sco(connector): + data = {"type": "ipv4-addr", "value": "203.0.113.5"} + assert connector._extract_ipv4(data) == "203.0.113.5" + + +def test_extract_ipv4_from_opencti_observable(connector): + data = {"entity_type": "IPv4-Addr", "observable_value": "198.51.100.7"} + assert connector._extract_ipv4(data) == "198.51.100.7" + + +def test_extract_ipv4_unrelated_type(connector): + assert connector._extract_ipv4({"type": "domain-name", "value": "x"}) is None + + +# --------------------------------------------------------------------------- # +# _object_id +# --------------------------------------------------------------------------- # +def test_object_id_prefers_id(connector): + assert connector._object_id({"id": "a", "x_opencti_id": "b"}) == "a" + + +def test_object_id_falls_back_to_x_opencti_id(connector): + assert connector._object_id({"x_opencti_id": "b"}) == "b" + + +def test_object_id_none(connector): + assert connector._object_id({}) is None + + +# --------------------------------------------------------------------------- # +# process_message routing +# --------------------------------------------------------------------------- # +def _msg(event, payload): + return SimpleNamespace(event=event, data=json.dumps({"data": payload})) + + +def test_process_message_create_caches(connector): + connector.process_message( + _msg("create", {"id": "ind-1", "type": "ipv4-addr", "value": "1.1.1.1"}) + ) + assert connector._indicator_cache["ind-1"] == "1.1.1.1" + + +def test_process_message_message_event_treated_as_create(connector): + connector.process_message( + _msg("message", {"id": "ind-2", "type": "ipv4-addr", "value": "2.2.2.2"}) + ) + assert connector._indicator_cache["ind-2"] == "2.2.2.2" + + +def test_process_message_delete_removes(connector): + connector._indicator_cache["ind-3"] = "3.3.3.3" + connector.process_message(_msg("delete", {"id": "ind-3", "type": "ipv4-addr"})) + assert "ind-3" not in connector._indicator_cache + + +def test_process_message_bad_json_warns(connector): + connector.process_message(SimpleNamespace(event="create", data="not-json")) + connector.logger.warning.assert_called_once() + + +def test_process_message_missing_data_key_warns(connector): + connector.process_message(SimpleNamespace(event="create", data=json.dumps({}))) + connector.logger.warning.assert_called_once() + + +def test_process_message_exits_on_keyboard_interrupt(connector, monkeypatch): + monkeypatch.setattr( + connector, "_handle_upsert", MagicMock(side_effect=KeyboardInterrupt) + ) + with pytest.raises(SystemExit): + connector.process_message( + _msg("create", {"id": "x", "type": "ipv4-addr", "value": "9.9.9.9"}) + ) + + +def test_process_message_swallows_unexpected_errors(connector, monkeypatch): + monkeypatch.setattr( + connector, "_handle_upsert", MagicMock(side_effect=RuntimeError("boom")) + ) + connector.process_message( + _msg("create", {"id": "x", "type": "ipv4-addr", "value": "9.9.9.9"}) + ) + connector.logger.error.assert_called_once() + + +# --------------------------------------------------------------------------- # +# upsert / delete +# --------------------------------------------------------------------------- # +def test_handle_upsert_ignores_without_value(connector): + connector._handle_upsert({"id": "i", "type": "domain-name"}) + assert connector._indicator_cache == {} + + +def test_handle_upsert_ignores_without_id(connector): + connector._handle_upsert({"type": "ipv4-addr", "value": "1.2.3.4"}) + assert connector._indicator_cache == {} + + +def test_handle_delete_unknown_id_is_noop(connector): + connector._handle_delete({"id": "unknown"}) + assert connector._indicator_cache == {} + + +# --------------------------------------------------------------------------- # +# _check_sync / _sync_to_cloudflare +# --------------------------------------------------------------------------- # +def test_check_sync_does_not_sync_before_interval(connector, monkeypatch): + connector.sync_interval = 9999 + connector._last_sync_time = 0.0 + monkeypatch.setattr("cloudflare_rules_list.connector.time.monotonic", lambda: 1.0) + sync = MagicMock() + monkeypatch.setattr(connector, "_sync_to_cloudflare", sync) + connector._check_sync() + sync.assert_not_called() + + +def test_sync_to_cloudflare_empty_cache(connector): + connector._sync_to_cloudflare() + connector.client.replace_list_items.assert_not_called() + connector.logger.info.assert_any_call("No indicators to sync") + + +def test_sync_to_cloudflare_empty_cache_does_not_open_throttle(connector, monkeypatch): + # An empty push must NOT stamp _last_sync_time, or the first real indicator + # could be delayed by up to sync_interval before it is synced. + monkeypatch.setattr("cloudflare_rules_list.connector.time.monotonic", lambda: 123.0) + connector._last_sync_time = 0.0 + connector._indicator_cache = {} + connector._sync_to_cloudflare() + assert connector._last_sync_time == 0.0 + + +def test_sync_to_cloudflare_with_operation(connector): + connector._indicator_cache = {"ind-1": "1.1.1.1"} + connector.client.replace_list_items.return_value = {"operation_id": "op-1"} + connector.client.wait_for_operation.return_value = {"status": "completed"} + + connector._sync_to_cloudflare() + + connector.client.replace_list_items.assert_called_once_with( + "list-123", [{"ip": "1.1.1.1", "comment": "OpenCTI: ind-1"}] + ) + connector.client.wait_for_operation.assert_called_once_with("op-1") + + +def test_sync_to_cloudflare_without_operation_id(connector): + connector._indicator_cache = {"ind-1": "1.1.1.1"} + connector.client.replace_list_items.return_value = {} + connector._sync_to_cloudflare() + connector.client.wait_for_operation.assert_not_called() + + +def test_sync_to_cloudflare_handles_api_error(connector): + connector._indicator_cache = {"ind-1": "1.1.1.1"} + connector.client.replace_list_items.side_effect = CloudflareAPIError("nope") + connector._sync_to_cloudflare() + connector.logger.error.assert_called_once() + + +# --------------------------------------------------------------------------- # +# _full_sync +# --------------------------------------------------------------------------- # +def test_full_sync_loads_indicators_and_observables(connector): + # Use the real OpenCTI API object shape: entity_type (capitalized), no + # lowercase STIX `type`. This mirrors helper.api.*.list() output so a + # regression in _extract_ipv4's indicator handling is caught here. + connector.helper.api.indicator.list.return_value = [ + { + "id": "ind-1", + "entity_type": "Indicator", + "pattern": "[ipv4-addr:value = '1.1.1.1']", + }, + { + "id": "ind-2", + "entity_type": "Indicator", + "pattern": "[domain-name:value = 'x']", + }, + ] + connector.helper.api.stix_cyber_observable.list.return_value = [ + {"id": "obs-1", "entity_type": "IPv4-Addr", "observable_value": "8.8.8.8"}, + ] + connector.client.replace_list_items.return_value = {} + + connector._full_sync() + + assert connector._indicator_cache == {"ind-1": "1.1.1.1", "obs-1": "8.8.8.8"} + connector.client.replace_list_items.assert_called_once() + + +def test_full_sync_handles_observable_error(connector): + connector.helper.api.indicator.list.return_value = [] + connector.helper.api.stix_cyber_observable.list.side_effect = RuntimeError("boom") + connector.client.replace_list_items.return_value = {} + + connector._full_sync() + + connector.logger.warning.assert_called_once() + + +# --------------------------------------------------------------------------- # +# run +# --------------------------------------------------------------------------- # +def test_run_verifies_list_then_listens(connector, monkeypatch): + connector.client.get_list.return_value = {"name": "blocklist", "kind": "ip"} + full_sync = MagicMock() + monkeypatch.setattr(connector, "_full_sync", full_sync) + + connector.run() + + connector.client.get_list.assert_called_once_with("list-123") + full_sync.assert_called_once() + connector.helper.listen_stream.assert_called_once_with( + message_callback=connector.process_message + ) + + +def test_run_raises_when_list_missing(connector): + connector.client.get_list.side_effect = CloudflareAPIError("404") + with pytest.raises(CloudflareAPIError): + connector.run() + connector.helper.listen_stream.assert_not_called() + + +def test_run_continues_when_full_sync_fails(connector, monkeypatch): + connector.client.get_list.return_value = {"name": "blocklist", "kind": "ip"} + monkeypatch.setattr( + connector, "_full_sync", MagicMock(side_effect=RuntimeError("boom")) + ) + connector.run() + connector.helper.listen_stream.assert_called_once() diff --git a/stream/cloudflare-rules-list/tests/test_main.py b/stream/cloudflare-rules-list/tests/test_main.py new file mode 100644 index 0000000000..8567536981 --- /dev/null +++ b/stream/cloudflare-rules-list/tests/test_main.py @@ -0,0 +1,81 @@ +from typing import Any +from unittest.mock import MagicMock + +import pytest +from cloudflare_rules_list import Connector, ConnectorSettings +from cloudflare_rules_list.client import CloudflareRulesListClient +from pycti import OpenCTIConnectorHelper + + +@pytest.fixture +def mock_opencti_connector_helper(monkeypatch): + """Mock all heavy dependencies of OpenCTIConnectorHelper (calls to OpenCTI).""" + module_import_path = "pycti.connector.opencti_connector_helper" + monkeypatch.setattr(f"{module_import_path}.killProgramHook", MagicMock()) + monkeypatch.setattr(f"{module_import_path}.sched.scheduler", MagicMock()) + monkeypatch.setattr(f"{module_import_path}.ConnectorInfo", MagicMock()) + monkeypatch.setattr(f"{module_import_path}.OpenCTIApiClient", MagicMock()) + monkeypatch.setattr(f"{module_import_path}.OpenCTIConnector", MagicMock()) + monkeypatch.setattr(f"{module_import_path}.OpenCTIMetricHandler", MagicMock()) + monkeypatch.setattr(f"{module_import_path}.PingAlive", MagicMock()) + + +class StubConnectorSettings(ConnectorSettings): + """`ConnectorSettings` subclass returning a fixed, valid config dict.""" + + @classmethod + def _load_config_dict(cls, _, handler) -> dict[str, Any]: + return handler( + { + "opencti": {"url": "http://localhost:8080", "token": "test-token"}, + "connector": { + "id": "connector-id", + "name": "Test Connector", + "scope": "cloudflare", + "log_level": "error", + "live_stream_id": "live", + "live_stream_listen_delete": True, + "live_stream_no_dependencies": True, + "sync_interval": "1h", + }, + "cloudflare": { + "account_id": "acc-1", + "api_token": "secret", + "list_id": "list-1", + }, + } + ) + + +def test_connector_settings_is_instantiated(): + settings = StubConnectorSettings() + assert isinstance(settings, ConnectorSettings) + assert isinstance(settings.to_helper_config(), dict) + + +def test_opencti_connector_helper_is_instantiated(mock_opencti_connector_helper): + settings = StubConnectorSettings() + helper = OpenCTIConnectorHelper(config=settings.to_helper_config()) + + assert helper.opencti_url == "http://localhost:8080/" + assert helper.opencti_token == "test-token" + assert helper.connect_id == "connector-id" + assert helper.connect_name == "Test Connector" + assert helper.connect_scope == "cloudflare" + assert helper.log_level == "ERROR" + assert helper.connect_live_stream_id == "live" + + +def test_connector_is_instantiated(mock_opencti_connector_helper): + settings = StubConnectorSettings() + helper = OpenCTIConnectorHelper(config=settings.to_helper_config()) + client = CloudflareRulesListClient( + account_id=settings.cloudflare.account_id, + api_token=settings.cloudflare.api_token.get_secret_value(), + ) + connector = Connector(helper=helper, config=settings, client=client) + + assert connector.config == settings + assert connector.helper == helper + assert connector.list_id == "list-1" + assert connector.sync_interval == 3600 diff --git a/stream/cloudflare-rules-list/tests/test_settings.py b/stream/cloudflare-rules-list/tests/test_settings.py new file mode 100644 index 0000000000..bd63f3f099 --- /dev/null +++ b/stream/cloudflare-rules-list/tests/test_settings.py @@ -0,0 +1,174 @@ +from typing import Any + +import pytest +from cloudflare_rules_list import ConnectorSettings +from connectors_sdk import BaseConfigModel, ConfigValidationError + + +@pytest.mark.parametrize( + "settings_dict", + [ + pytest.param( + { + "opencti": {"url": "http://localhost:8080", "token": "test-token"}, + "connector": { + "id": "connector-id", + "name": "Test Connector", + "scope": "cloudflare", + "log_level": "error", + "live_stream_id": "live", + "live_stream_listen_delete": True, + "live_stream_no_dependencies": True, + "sync_interval": "30m", + }, + "cloudflare": { + "account_id": "ChangeMe", + "api_token": "ChangeMe", + "list_id": "ChangeMe", + }, + }, + id="full_valid_settings_dict", + ), + pytest.param( + { + "opencti": {"url": "http://localhost:8080", "token": "test-token"}, + "connector": { + "id": "connector-id", + "scope": "cloudflare", + "log_level": "error", + "live_stream_id": "live", + }, + "cloudflare": { + "account_id": "ChangeMe", + "api_token": "ChangeMe", + "list_id": "ChangeMe", + }, + }, + id="minimal_valid_settings_dict", + ), + ], +) +def test_settings_should_accept_valid_input(settings_dict): + """`ConnectorSettings` accepts valid input and exposes typed namespaces.""" + + class FakeConnectorSettings(ConnectorSettings): + @classmethod + def _load_config_dict(cls, _, handler) -> dict[str, Any]: + return handler(settings_dict) + + settings = FakeConnectorSettings() + assert isinstance(settings.opencti, BaseConfigModel) is True + assert isinstance(settings.connector, BaseConfigModel) is True + assert isinstance(settings.cloudflare, BaseConfigModel) is True + + +def test_settings_defaults_are_applied(): + """Optional connector fields fall back to their declared defaults.""" + + class FakeConnectorSettings(ConnectorSettings): + @classmethod + def _load_config_dict(cls, _, handler) -> dict[str, Any]: + return handler( + { + "opencti": {"url": "http://localhost:8080", "token": "test-token"}, + "connector": { + "id": "connector-id", + "live_stream_id": "live", + }, + "cloudflare": { + "account_id": "acc", + "api_token": "secret", + "list_id": "list", + }, + } + ) + + settings = FakeConnectorSettings() + assert settings.connector.name == "Cloudflare Rules List" + assert settings.connector.scope == ["cloudflare"] + assert settings.connector.sync_interval == "1h" + assert settings.cloudflare.api_base_url == "https://api.cloudflare.com/client/v4" + + +def test_settings_api_token_is_secret(): + """The Cloudflare API token is wrapped as a SecretStr and not leaked in repr.""" + + class FakeConnectorSettings(ConnectorSettings): + @classmethod + def _load_config_dict(cls, _, handler) -> dict[str, Any]: + return handler( + { + "opencti": {"url": "http://localhost:8080", "token": "test-token"}, + "connector": {"id": "connector-id", "live_stream_id": "live"}, + "cloudflare": { + "account_id": "acc", + "api_token": "super-secret", + "list_id": "list", + }, + } + ) + + settings = FakeConnectorSettings() + assert settings.cloudflare.api_token.get_secret_value() == "super-secret" + assert "super-secret" not in repr(settings.cloudflare.api_token) + + +def test_to_helper_config_returns_dict(): + """`to_helper_config` (inherited) returns a dict for OpenCTIConnectorHelper.""" + + class FakeConnectorSettings(ConnectorSettings): + @classmethod + def _load_config_dict(cls, _, handler) -> dict[str, Any]: + return handler( + { + "opencti": {"url": "http://localhost:8080", "token": "test-token"}, + "connector": {"id": "connector-id", "live_stream_id": "live"}, + "cloudflare": { + "account_id": "acc", + "api_token": "secret", + "list_id": "list", + }, + } + ) + + settings = FakeConnectorSettings() + assert isinstance(settings.to_helper_config(), dict) + + +@pytest.mark.parametrize( + "settings_dict", + [ + pytest.param({}, id="empty_settings_dict"), + pytest.param( + { + "opencti": {"url": "http://localhost:PORT", "token": "test-token"}, + "connector": {"id": "connector-id", "live_stream_id": "live"}, + "cloudflare": { + "account_id": "acc", + "api_token": "secret", + "list_id": "list", + }, + }, + id="invalid_opencti_url", + ), + pytest.param( + { + "opencti": {"url": "http://localhost:8080", "token": "test-token"}, + "connector": {"id": "connector-id", "live_stream_id": "live"}, + "cloudflare": {"account_id": "acc"}, + }, + id="missing_cloudflare_fields", + ), + ], +) +def test_settings_should_raise_when_invalid_input(settings_dict): + """`ConnectorSettings` raises a validation error on invalid/incomplete input.""" + + class FakeConnectorSettings(ConnectorSettings): + @classmethod + def _load_config_dict(cls, _, handler) -> dict[str, Any]: + return handler(settings_dict) + + with pytest.raises(ConfigValidationError) as err: + FakeConnectorSettings() + assert "Error validating configuration" in str(err)