From 23f0e3d0dafdf87d4c1113cb4057f5794960b574 Mon Sep 17 00:00:00 2001 From: Khidr6g <270672697+Khidr6G@users.noreply.github.com> Date: Fri, 17 Jul 2026 17:55:22 +0300 Subject: [PATCH] feat(email-cases-importer): add connector to import emails as incident-response cases (#7036) --- .../email-cases-importer/.dockerignore | 42 + .../email-cases-importer/.env.sample | 27 + .../email-cases-importer/Dockerfile | 38 + .../email-cases-importer/README.md | 564 ++++++++++ external-import/email-cases-importer/VERSION | 1 + .../__metadata__/connector_manifest.json | 19 + .../__metadata__/logo.png | Bin 0 -> 22952 bytes .../email-cases-importer/config.yml.sample | 212 ++++ .../email-cases-importer/docker-compose.yml | 82 ++ .../email-cases-importer/entrypoint.sh | 2 + .../email-cases-importer/pyproject.toml | 28 + .../scripts/test_mailbox_connection.py | 492 +++++++++ .../src/attachment_handler/__init__.py | 8 + .../src/attachment_handler/archive_handler.py | 213 ++++ .../src/attachment_handler/base.py | 58 ++ .../attachment_handler/document_handler.py | 126 +++ .../attachment_handler/passthrough_handler.py | 41 + .../src/attachment_handler/registry.py | 198 ++++ .../src/connector/__init__.py | 8 + .../src/connector/connector.py | 978 ++++++++++++++++++ .../src/connector/converter_to_stix.py | 107 ++ .../src/connector/settings.py | 287 +++++ .../src/connector/utils.py | 124 +++ .../src/email_client/__init__.py | 9 + .../src/email_client/_http.py | 67 ++ .../src/email_client/base.py | 66 ++ .../src/email_client/ews_client.py | 187 ++++ .../src/email_client/factory.py | 52 + .../src/email_client/gmail_client.py | 241 +++++ .../src/email_client/graph_client.py | 199 ++++ .../src/email_client/imap_client.py | 216 ++++ .../email-cases-importer/src/main.py | 17 + .../email-cases-importer/src/requirements.txt | 9 + .../test-requirements.txt | 8 + .../email-cases-importer/tests/__init__.py | 0 .../email-cases-importer/tests/conftest.py | 101 ++ .../tests/test_attachment_handlers.py | 158 +++ .../tests/test_connector.py | 636 ++++++++++++ .../tests/test_converter_to_stix.py | 158 +++ .../tests/test_email_client_factory.py | 123 +++ .../tests/test_ews_client.py | 174 ++++ .../tests/test_ews_filter.py | 87 ++ .../tests/test_gmail_client.py | 296 ++++++ .../tests/test_graph_client.py | 226 ++++ .../tests/test_handlers_extra.py | 178 ++++ .../tests/test_http_helpers.py | 89 ++ .../tests/test_imap_client.py | 204 ++++ .../email-cases-importer/tests/test_main.py | 47 + .../tests/test_settings.py | 234 +++++ .../email-cases-importer/tests/test_utils.py | 243 +++++ 50 files changed, 7680 insertions(+) create mode 100644 external-import/email-cases-importer/.dockerignore create mode 100644 external-import/email-cases-importer/.env.sample create mode 100644 external-import/email-cases-importer/Dockerfile create mode 100644 external-import/email-cases-importer/README.md create mode 100644 external-import/email-cases-importer/VERSION create mode 100644 external-import/email-cases-importer/__metadata__/connector_manifest.json create mode 100644 external-import/email-cases-importer/__metadata__/logo.png create mode 100644 external-import/email-cases-importer/config.yml.sample create mode 100644 external-import/email-cases-importer/docker-compose.yml create mode 100644 external-import/email-cases-importer/entrypoint.sh create mode 100644 external-import/email-cases-importer/pyproject.toml create mode 100644 external-import/email-cases-importer/scripts/test_mailbox_connection.py create mode 100644 external-import/email-cases-importer/src/attachment_handler/__init__.py create mode 100644 external-import/email-cases-importer/src/attachment_handler/archive_handler.py create mode 100644 external-import/email-cases-importer/src/attachment_handler/base.py create mode 100644 external-import/email-cases-importer/src/attachment_handler/document_handler.py create mode 100644 external-import/email-cases-importer/src/attachment_handler/passthrough_handler.py create mode 100644 external-import/email-cases-importer/src/attachment_handler/registry.py create mode 100644 external-import/email-cases-importer/src/connector/__init__.py create mode 100644 external-import/email-cases-importer/src/connector/connector.py create mode 100644 external-import/email-cases-importer/src/connector/converter_to_stix.py create mode 100644 external-import/email-cases-importer/src/connector/settings.py create mode 100644 external-import/email-cases-importer/src/connector/utils.py create mode 100644 external-import/email-cases-importer/src/email_client/__init__.py create mode 100644 external-import/email-cases-importer/src/email_client/_http.py create mode 100644 external-import/email-cases-importer/src/email_client/base.py create mode 100644 external-import/email-cases-importer/src/email_client/ews_client.py create mode 100644 external-import/email-cases-importer/src/email_client/factory.py create mode 100644 external-import/email-cases-importer/src/email_client/gmail_client.py create mode 100644 external-import/email-cases-importer/src/email_client/graph_client.py create mode 100644 external-import/email-cases-importer/src/email_client/imap_client.py create mode 100644 external-import/email-cases-importer/src/main.py create mode 100644 external-import/email-cases-importer/src/requirements.txt create mode 100644 external-import/email-cases-importer/test-requirements.txt create mode 100644 external-import/email-cases-importer/tests/__init__.py create mode 100644 external-import/email-cases-importer/tests/conftest.py create mode 100644 external-import/email-cases-importer/tests/test_attachment_handlers.py create mode 100644 external-import/email-cases-importer/tests/test_connector.py create mode 100644 external-import/email-cases-importer/tests/test_converter_to_stix.py create mode 100644 external-import/email-cases-importer/tests/test_email_client_factory.py create mode 100644 external-import/email-cases-importer/tests/test_ews_client.py create mode 100644 external-import/email-cases-importer/tests/test_ews_filter.py create mode 100644 external-import/email-cases-importer/tests/test_gmail_client.py create mode 100644 external-import/email-cases-importer/tests/test_graph_client.py create mode 100644 external-import/email-cases-importer/tests/test_handlers_extra.py create mode 100644 external-import/email-cases-importer/tests/test_http_helpers.py create mode 100644 external-import/email-cases-importer/tests/test_imap_client.py create mode 100644 external-import/email-cases-importer/tests/test_main.py create mode 100644 external-import/email-cases-importer/tests/test_settings.py create mode 100644 external-import/email-cases-importer/tests/test_utils.py diff --git a/external-import/email-cases-importer/.dockerignore b/external-import/email-cases-importer/.dockerignore new file mode 100644 index 00000000000..e165f2987d5 --- /dev/null +++ b/external-import/email-cases-importer/.dockerignore @@ -0,0 +1,42 @@ +# VCS / editor +.git +.gitignore +.gitattributes +.vscode +.idea +*.swp +*.swo + +# Build artifacts / caches +__pycache__ +*.pyc +*.pyo +*.pyd +*.egg-info +.pytest_cache +.mypy_cache +.ruff_cache +.coverage +htmlcov +dist +build + +# Local virtualenvs and secrets +.venv +venv +env +.env + +# Connector ops files (not needed in the image) +docker-compose.yml +config.yml +config.yml.sample +README.md +.dockerignore +Dockerfile + +# Tests and tooling (image should not ship test code) +tests +test-requirements.txt +pyproject.toml +.pylintrc diff --git a/external-import/email-cases-importer/.env.sample b/external-import/email-cases-importer/.env.sample new file mode 100644 index 00000000000..c56b29c53ae --- /dev/null +++ b/external-import/email-cases-importer/.env.sample @@ -0,0 +1,27 @@ +# Copy to .env and fill in. Docker Compose reads this file to populate the +# ${...} variables referenced in docker-compose.yml. Never commit your real .env. + +# --- OpenCTI platform --- +OPENCTI_ADMIN_TOKEN=ChangeMe # OpenCTI API token (maps to OPENCTI_TOKEN) +CONNECTOR_EMAIL_CASES_ID=ChangeMe # Unique UUIDv4 for this connector instance + +# --- IMAP (default protocol) --- +EMAIL_CASES_IMAP_HOST=imap.example.com +EMAIL_CASES_IMAP_USERNAME=soc@example.com +EMAIL_CASES_IMAP_PASSWORD=ChangeMe + +# --- Email filtering --- +EMAIL_CASES_SENDER_ADDRESS=alerts@example.com +# JSON array of subject filters. Use [] to accept any subject. +EMAIL_CASES_SUBJECT_FILTERS=[] + +# --- Microsoft Graph (only if EMAIL_CASES_PROTOCOL=microsoft_graph) --- +# EMAIL_CASES_GRAPH_TENANT_ID= +# EMAIL_CASES_GRAPH_CLIENT_ID= +# EMAIL_CASES_GRAPH_CLIENT_SECRET= +# EMAIL_CASES_GRAPH_USER_ID= + +# --- EWS (only if EMAIL_CASES_PROTOCOL=ews) --- +# EMAIL_CASES_EWS_SERVER= +# EMAIL_CASES_EWS_USERNAME= +# EMAIL_CASES_EWS_PASSWORD= diff --git a/external-import/email-cases-importer/Dockerfile b/external-import/email-cases-importer/Dockerfile new file mode 100644 index 00000000000..57f2b68da60 --- /dev/null +++ b/external-import/email-cases-importer/Dockerfile @@ -0,0 +1,38 @@ +FROM python:3.12-alpine + +ENV CONNECTOR_TYPE=EXTERNAL_IMPORT +ENV PYTHONUNBUFFERED=1 + +RUN apk update && apk upgrade && \ + apk add --no-cache \ + libmagic \ + p7zip \ + libarchive-tools \ + libffi \ + libxml2 \ + libxslt && \ + apk add --no-cache --virtual .build-deps \ + git \ + gcc \ + musl-dev \ + libffi-dev \ + openssl-dev \ + libxml2-dev \ + libxslt-dev && \ + rm -rf /var/cache/apk/* + +COPY src/requirements.txt /opt/opencti-connector/ +WORKDIR /opt/opencti-connector + +RUN pip install --no-cache-dir -r requirements.txt && \ + apk del .build-deps + +COPY entrypoint.sh / +RUN chmod +x /entrypoint.sh + +COPY src/ /opt/opencti-connector/ + +RUN adduser -D connector +USER connector + +ENTRYPOINT ["/entrypoint.sh"] diff --git a/external-import/email-cases-importer/README.md b/external-import/email-cases-importer/README.md new file mode 100644 index 00000000000..d16b581a187 --- /dev/null +++ b/external-import/email-cases-importer/README.md @@ -0,0 +1,564 @@ +# Email Cases Importer Connector for OpenCTI + +OpenCTI connector that polls email inboxes and creates **Incident Response Cases** from matching emails. Supports **IMAP**, **Microsoft Graph**, **Gmail**, and **EWS** protocols. + +## Features + +- **Multi-protocol support** — IMAP, Microsoft Graph (Office 365), Gmail API, Exchange Web Services +- **Subject filtering** — Exact match, substring, and regex filters to select which emails to process +- **Thread tracking** — Groups email thread replies into the same Case-Incident (provider thread ID, message headers, or subject matching) +- **Password extraction** — Extracts passwords from email bodies using configurable markers, then uses them to decrypt attachments +- **Attachment handling** — Decrypts encrypted 7z, zip, rar, xlsx, pdf; handles nested encryption (e.g. unprotected zip containing an encrypted xlsx) +- **Labels and subject rules** — Static labels on every case, plus conditional labels/response types/severity/priority/case templates based on subject matching +- **Sender rules** — Auto-set author, marking definition, assignees, and participants based on sender email +- **Auto-creation** — Missing labels, vocabulary values (severity, priority, response types), and author identities are auto-created in OpenCTI at startup +- **Extensible handlers** — Plugin-style registry for custom file type parsing +- **Case deduplication** — Finds existing cases by name after state resets to avoid duplicates +- **Unicode support** — Handles non-English email subjects, bodies, and attachment filenames (Arabic, Chinese, Japanese, etc.) +- **Localized prefixes** — Strips RE/FW/FWD and localized variants (AW, WG, TR, RV, etc.) for thread matching + +## How It Works + +1. **Poll** — Connector polls the email inbox at the configured interval +2. **Filter** — Emails are filtered by sender address and subject filters +3. **Group** — Replies are grouped by thread ID into the same Case-Incident +4. **Extract** — Passwords are extracted from email body using prefix/suffix markers +5. **Decrypt** — Attachments are decrypted using extracted passwords +6. **Create** — Case-Incident is created in OpenCTI with labels, response types, severity, priority, author, marking, assignees, and case template applied. Email content is written to the case's **Content** tab as formatted HTML. Both original (encrypted) and extracted (decrypted) attachments are uploaded as files on the case. + +## Installation + +### Docker (Recommended) + +```bash +docker compose up -d +``` + +### Manual + +```bash +pip install -r requirements.txt +cd src +python main.py +``` + +## Configuration + +All settings can be provided via environment variables or a `config.yml` file. See [`config.yml.sample`](config.yml.sample) for a fully commented example. + +--- + +### OpenCTI Connection + +| Parameter | config.yml key | Docker env var | Default | Mandatory | Description | +|-----------|----------------|----------------|---------|-----------|-------------| +| OpenCTI URL | `opencti.url` | `OPENCTI_URL` | *required* | Yes | OpenCTI platform URL | +| OpenCTI token | `opencti.token` | `OPENCTI_TOKEN` | *required* | Yes | OpenCTI API token | +| Connector ID | `connector.id` | `CONNECTOR_ID` | *required* | Yes | Unique connector ID (UUIDv4) | +| Connector name | `connector.name` | `CONNECTOR_NAME` | `Email Cases` | No | Display name | +| Connector scope | `connector.scope` | `CONNECTOR_SCOPE` | `Email Cases` | No | Connector scope | +| Log level | `connector.log_level` | `CONNECTOR_LOG_LEVEL` | `info` | No | Log level (`debug`, `info`, `warning`, `error`) | +| Duration period | `connector.duration_period` | `CONNECTOR_DURATION_PERIOD` | `PT5M` | No | Polling cadence (ISO-8601 duration). **This is the control for polling frequency.** | + +--- + +### Protocol Selection + +| Parameter | config.yml key | Docker env var | Default | Mandatory | Description | +|-----------|----------------|----------------|---------|-----------|-------------| +| Protocol | `email_cases.protocol` | `EMAIL_CASES_PROTOCOL` | `imap` | No | `imap`, `microsoft_graph`, `gmail`, or `ews` | + +--- + +### IMAP Settings + +| Parameter | config.yml key | Docker env var | Default | Mandatory | Description | +|-----------|----------------|----------------|---------|-----------|-------------| +| IMAP host | `email_cases.imap_host` | `EMAIL_CASES_IMAP_HOST` | *required* | Yes | IMAP server hostname | +| IMAP port | `email_cases.imap_port` | `EMAIL_CASES_IMAP_PORT` | `993` | No | IMAP server port | +| IMAP username | `email_cases.imap_username` | `EMAIL_CASES_IMAP_USERNAME` | *required* | Yes | IMAP username | +| IMAP password | `email_cases.imap_password` | `EMAIL_CASES_IMAP_PASSWORD` | *required* | Yes | IMAP password | +| IMAP folder | `email_cases.imap_folder` | `EMAIL_CASES_IMAP_FOLDER` | `INBOX` | No | IMAP folder to monitor | +| IMAP use SSL | `email_cases.imap_use_ssl` | `EMAIL_CASES_IMAP_USE_SSL` | `true` | No | Use SSL/TLS | + +### Microsoft Graph Settings (Office 365) + +| Parameter | config.yml key | Docker env var | Default | Mandatory | Description | +|-----------|----------------|----------------|---------|-----------|-------------| +| Graph tenant ID | `email_cases.graph_tenant_id` | `EMAIL_CASES_GRAPH_TENANT_ID` | *required* | Yes | Azure AD tenant ID | +| Graph client ID | `email_cases.graph_client_id` | `EMAIL_CASES_GRAPH_CLIENT_ID` | *required* | Yes | Azure AD application (client) ID | +| Graph client secret | `email_cases.graph_client_secret` | `EMAIL_CASES_GRAPH_CLIENT_SECRET` | *required* | Yes | Azure AD client secret | +| Graph user ID | `email_cases.graph_user_id` | `EMAIL_CASES_GRAPH_USER_ID` | *required* | Yes | Mailbox user ID or UPN | + +Requires the `Mail.Read` application permission in Azure AD. + +### Gmail Settings + +| Parameter | config.yml key | Docker env var | Default | Mandatory | Description | +|-----------|----------------|----------------|---------|-----------|-------------| +| Gmail credentials file | `email_cases.gmail_credentials_file` | `EMAIL_CASES_GMAIL_CREDENTIALS_FILE` | *required* | Yes | Path to service account credentials JSON | +| Gmail user ID | `email_cases.gmail_user_id` | `EMAIL_CASES_GMAIL_USER_ID` | `me` | No | Gmail user ID | + +Requires a Google service account with domain-wide delegation and the `https://www.googleapis.com/auth/gmail.readonly` scope. + +### EWS Settings (On-Premise Exchange) + +| Parameter | config.yml key | Docker env var | Default | Mandatory | Description | +|-----------|----------------|----------------|---------|-----------|-------------| +| EWS server | `email_cases.ews_server` | `EMAIL_CASES_EWS_SERVER` | *(empty)* | No | Exchange server URL. Leave empty to use Autodiscover. | +| EWS username | `email_cases.ews_username` | `EMAIL_CASES_EWS_USERNAME` | *required* | Yes | Exchange username (`DOMAIN\user`) | +| EWS password | `email_cases.ews_password` | `EMAIL_CASES_EWS_PASSWORD` | *required* | Yes | Exchange password | +| EWS auth type | `email_cases.ews_auth_type` | `EMAIL_CASES_EWS_AUTH_TYPE` | `NTLM` | No | Auth type: `NTLM` or `OAuth2` | + +--- + +### Email Filtering + +| Parameter | config.yml key | Docker env var | Default | Mandatory | Description | +|-----------|----------------|----------------|---------|-----------|-------------| +| Sender address | `email_cases.sender_address` | `EMAIL_CASES_SENDER_ADDRESS` | *required* | Yes | Only process emails from this sender | +| Subject filters | `email_cases.subject_filters` | `EMAIL_CASES_SUBJECT_FILTERS` | *required* | Yes | JSON array of subject filters (see below) | + +An email is processed if its subject matches **any** filter. Supported filter types: + +| Type | Description | Case-sensitive | +|------|-------------|---------------| +| `exact` | Subject must equal the value exactly | Yes | +| `contains` | Subject must contain the value | Yes | +| `regex` | Subject must match the regular expression | Per regex flags | + +**Example:** +```json +[ + {"type": "exact", "value": "Weekly Threat Report"}, + {"type": "contains", "value": "Security Alert"}, + {"type": "regex", "value": "INC-\\d+"} +] +``` + +--- + +### Thread Tracking + +| Parameter | config.yml key | Docker env var | Default | Mandatory | Description | +|-----------|----------------|----------------|---------|-----------|-------------| +| Thread tracking strategy | `email_cases.thread_tracking_strategy` | `EMAIL_CASES_THREAD_TRACKING_STRATEGY` | `provider_thread_id` | No | How replies are grouped into the same case | + +| Strategy | Description | +|----------|-------------| +| `provider_thread_id` | Uses the provider's native thread/conversation ID. Most reliable. Falls back to subject matching if unavailable. | +| `message_headers` | Uses `In-Reply-To` and `References` email headers. Works across all providers. | +| `subject_matching` | Strips `RE:` / `FW:` / `FWD:` prefixes (and localized variants like `AW:`, `WG:`, `TR:`, `RV:`) and matches on the base subject. Simplest but may incorrectly merge unrelated emails with similar subjects. | + +--- + +### Password Extraction + +| Parameter | config.yml key | Docker env var | Default | Mandatory | Description | +|-----------|----------------|----------------|---------|-----------|-------------| +| Password prefix | `email_cases.password_prefix` | `EMAIL_CASES_PASSWORD_PREFIX` | `---BEGIN PASSWORD---` | No | Opening marker for the password in the email body | +| Password suffix | `email_cases.password_suffix` | `EMAIL_CASES_PASSWORD_SUFFIX` | `---END PASSWORD---` | No | Closing marker for the password in the email body | +| Password strip whitespace | `email_cases.password_strip_whitespace` | `EMAIL_CASES_PASSWORD_STRIP_WHITESPACE` | `false` | No | Strip all spaces, tabs, and newlines from extracted passwords | + +The connector scans the email body for text between these markers and uses the extracted passwords to decrypt attachments. Multiple passwords can be embedded in a single email. + +If `EMAIL_CASES_PASSWORD_STRIP_WHITESPACE` is `true`, all spaces, tabs, and newlines are removed from the extracted password. This is useful when HTML rendering or email line wrapping inserts whitespace within the password between the markers. + +**Example email body:** +``` +Please find the malware sample attached. + +The password is: ---BEGIN PASSWORD---infected123---END PASSWORD--- +``` + +--- + +### Case Defaults + +| Parameter | config.yml key | Docker env var | Default | Mandatory | Description | +|-----------|----------------|----------------|---------|-----------|-------------| +| Default severity | `email_cases.default_severity` | `EMAIL_CASES_DEFAULT_SEVERITY` | `medium` | No | Default case severity (e.g. `low`, `medium`, `high`, `critical`) | +| Default priority | `email_cases.default_priority` | `EMAIL_CASES_DEFAULT_PRIORITY` | `P3` | No | Default case priority (e.g. `P1`, `P2`, `P3`, `P4`) | +| Case prefix | `email_cases.case_prefix` | `EMAIL_CASES_CASE_PREFIX` | *empty* | No | Optional prefix prepended to case names (e.g. `[EMAIL] `) | + +Severity and priority values that don't exist in OpenCTI are **auto-created** as vocabulary entries at connector startup. + +--- + +### Labels + +| Parameter | config.yml key | Docker env var | Default | Mandatory | Description | +|-----------|----------------|----------------|---------|-----------|-------------| +| Labels | `email_cases.labels` | `EMAIL_CASES_LABELS` | *empty* | No | Comma-separated labels always added to every case | + +Labels are **auto-created** in OpenCTI if they don't exist. + +**Example:** +``` +EMAIL_CASES_LABELS=NCSC UK,Email Alert,SOC Triage +``` + +--- + +### Subject Rules + +| Variable | Description | Default | +|----------|-------------|---------| +| `EMAIL_CASES_SUBJECT_RULES` | JSON array of subject-based rules (see below) | `[]` | + +Subject rules let you **conditionally** set case properties based on the email subject. Rules are evaluated on every new case creation. Multiple rules can match the same email — labels and response types are merged; the first matching `severity`, `priority`, and `case_template` win. + +#### Rule structure + +Each rule requires two fields: + +| Field | Required | Description | +|-------|----------|-------------| +| `match_type` | Yes | How to match: `exact`, `contains`, `starts_with`, or `regex` | +| `value` | Yes | The string or regex pattern to match against | + +And can optionally include: + +| Field | Optional | Description | +|-------|----------|-------------| +| `labels` | Yes | List of label names to add to the case | +| `response_types` | Yes | List of incident response types (auto-created if missing) | +| `severity` | Yes | Override the default severity for this case (auto-created if missing) | +| `priority` | Yes | Override the default priority for this case (auto-created if missing) | +| `case_template` | Yes | Name of an existing OpenCTI case template to apply | + +#### Match types + +| Match type | Description | Case-sensitive | +|------------|-------------|---------------| +| `exact` | Subject must equal the value exactly | Yes | +| `contains` | Subject must contain the value | No | +| `starts_with` | Subject must start with the value | No | +| `regex` | Subject must match the regular expression | Per regex flags | + +#### Examples + +**Single rule — escalate ransomware alerts:** +```json +[ + { + "match_type": "contains", + "value": "Ransomware", + "labels": ["Ransomware"], + "response_types": ["ransomware"], + "severity": "critical", + "priority": "P1", + "case_template": "Ransomware Playbook" + } +] +``` + +**Multiple rules — different behavior per subject pattern:** +```json +[ + { + "match_type": "contains", + "value": "Threat Alert", + "labels": ["Threat Alert"], + "response_types": ["ransomware"], + "severity": "high", + "priority": "P1" + }, + { + "match_type": "starts_with", + "value": "INC-", + "labels": ["Incident"], + "response_types": ["data-leak"], + "case_template": "Incident Response Playbook" + }, + { + "match_type": "regex", + "value": "CVE-\\d{4}-\\d+", + "labels": ["Vulnerability", "CVE"], + "severity": "high" + }, + { + "match_type": "exact", + "value": "Weekly Threat Report", + "labels": ["Weekly Report"], + "severity": "low", + "priority": "P4" + } +] +``` + +**As an environment variable (Docker):** +```yaml +- EMAIL_CASES_SUBJECT_RULES=[{"match_type":"contains","value":"Threat Alert","labels":["Threat Alert"],"severity":"critical","priority":"P1"},{"match_type":"starts_with","value":"INC-","labels":["Incident"]}] +``` + +--- + +### Sender Rules + +| Variable | Description | Default | +|----------|-------------|---------| +| `EMAIL_CASES_SENDER_RULES` | JSON array of sender-based rules (see below) | `[]` | + +Sender rules let you set case properties based on who sent the email. Each rule matches on the sender email address (case-insensitive). + +#### Rule structure + +| Field | Required | Description | +|-------|----------|-------------| +| `sender` | Yes | The sender email address to match (case-insensitive) | +| `author` | No | Organization name to set as case author. **Auto-created** as an Organization identity if it doesn't exist. | +| `marking` | No | Marking definition to apply (e.g. `TLP:GREEN`, `TLP:AMBER`, `TLP:RED`). Must already exist in OpenCTI — TLP markings are built-in. | +| `assignees` | No | List of OpenCTI user emails to assign to the case. Users must already exist in OpenCTI. | +| `participants` | No | List of OpenCTI user emails to add as participants. Users must already exist in OpenCTI. | + +#### Example + +```json +[ + { + "sender": "alerts@ncsc.gov.uk", + "author": "NCSC UK", + "marking": "TLP:AMBER", + "assignees": ["analyst@company.com"], + "participants": ["soc-team@company.com", "manager@company.com"] + }, + { + "sender": "noreply@security-vendor.com", + "author": "Security Vendor", + "marking": "TLP:GREEN" + } +] +``` + +**As an environment variable (Docker):** +```yaml +- EMAIL_CASES_SENDER_RULES=[{"sender":"alerts@ncsc.gov.uk","author":"NCSC UK","marking":"TLP:AMBER","assignees":["analyst@company.com"]}] +``` + +> **Note:** If an assignee or participant email is not found in OpenCTI, a warning is logged and the case is created without that user. + +> **Note:** If a marking definition name doesn't match any existing marking in OpenCTI, a warning is logged and the case is created without the marking. + +--- + +### Import Settings + +| Parameter | config.yml key | Docker env var | Default | Mandatory | Description | +|-----------|----------------|----------------|---------|-----------|-------------| +| Import interval (**deprecated**) | `email_cases.import_interval` | `EMAIL_CASES_IMPORT_INTERVAL` | `300` | No | **DEPRECATED** — ignored when `CONNECTOR_DURATION_PERIOD` is set; retained only for backward compatibility. | +| Max emails per cycle | `email_cases.max_emails_per_cycle` | `EMAIL_CASES_MAX_EMAILS_PER_CYCLE` | `50` | No | Max emails processed per cycle | +| TLS verify | `email_cases.tls_verify` | `EMAIL_CASES_TLS_VERIFY` | `true` | No | Verify TLS certificates | + +#### Polling cadence + +`CONNECTOR_DURATION_PERIOD` (config.yml key `connector.duration_period`) is **the** control for how +often the connector polls. It is the standard `connectors-sdk` scheduling field and accepts an +ISO-8601 duration (e.g. `PT5M` for five minutes, `PT1H` for one hour); the default is `PT5M`. + +The legacy `EMAIL_CASES_IMPORT_INTERVAL` (seconds) is **DEPRECATED**: it is ignored when +`CONNECTOR_DURATION_PERIOD` is set and is retained only for backward compatibility. New +deployments should set the cadence exclusively through `CONNECTOR_DURATION_PERIOD`. + +--- + +### Attachment Settings + +| Parameter | config.yml key | Docker env var | Default | Mandatory | Description | +|-----------|----------------|----------------|---------|-----------|-------------| +| Max attachment size (MB) | `email_cases.max_attachment_size_mb` | `EMAIL_CASES_MAX_ATTACHMENT_SIZE_MB` | `25` | No | Max attachment size in MB (larger files are skipped) | +| Store attachments in OpenCTI | `email_cases.attachment_store_in_opencti` | `EMAIL_CASES_ATTACHMENT_STORE_IN_OPENCTI` | `true` | No | Upload attachments as files on the case | + +--- + +## Auto-Creation Behavior + +The connector automatically creates missing entities at startup and during processing: + +| Entity | When created | How | +|--------|-------------|-----| +| **Labels** | On first use | Created via OpenCTI Label API | +| **Severity values** | At startup | Created as `case_severity_ov` vocabulary entries | +| **Priority values** | At startup | Created as `case_priority_ov` vocabulary entries | +| **Response type values** | At startup | Created as `incident_response_types_ov` vocabulary entries | +| **Author identity** | On first use | Created as Organization identity via OpenCTI Identity API | +| **Marking definitions** | Never | Must already exist in OpenCTI (TLP markings are built-in) | +| **Users (assignees/participants)** | Never | Must already exist as OpenCTI users | +| **Case templates** | Never | Must already exist in OpenCTI | + +--- + +## Supported Attachment Types + +| Format | Extension | Password Support | Library | +|--------|-----------|-----------------|---------| +| ZIP archive | `.zip` | Yes | `zipfile` (stdlib) | +| 7-Zip archive | `.7z` | Yes | `py7zr` | +| RAR archive | `.rar` | No (extract only) | `bsdtar` (libarchive-tools) | +| Excel spreadsheet | `.xlsx` | Yes | `msoffcrypto-tool` | +| PDF document | `.pdf` | Yes | `pikepdf` | +| CSV | `.csv` | No | `csv` (stdlib) | +| Text | `.txt` | No | stdlib | +| Email | `.eml` | No | `email` (stdlib) | +| Outlook message | `.msg` | No | — (uploaded as-is) | + +### Attachment behavior + +- **Original attachments** are always uploaded to the case as-is (including encrypted ones) +- **Extracted/decrypted content** is uploaded alongside the original when extraction succeeds +- For archives, each inner file is also uploaded individually +- If decryption fails (wrong password, corrupted file), the original is still uploaded and a warning is logged + +### Nested encryption + +The connector handles nested encryption scenarios: +- An unprotected archive (zip/7z/rar) containing encrypted inner files (xlsx/pdf) +- An encrypted archive containing further encrypted content +- Maximum nesting depth: 3 levels + +### Error handling + +Each attachment is processed independently — if one fails, the rest continue: +- Corrupted archives: logged and skipped, original file still uploaded +- Wrong passwords: all configured passwords are tried, failure logged at debug level +- Missing libraries (py7zr, pikepdf, msoffcrypto-tool): logged as warning, file passed through as-is +- Oversized files: skipped with metadata only +- File I/O errors: caught and logged with filename context + +--- + +## Unicode and Non-English Support + +The connector handles non-English content throughout: + +- **Email subjects** — MIME encoded-word decoding (RFC 2047) correctly handles Arabic, Chinese, Japanese, Cyrillic, and other scripts +- **Email bodies** — UTF-8 and other charset encodings decoded properly; HTML entities decoded +- **Attachment filenames** — Non-ASCII filenames (e.g. `تقرير.xlsx`, `報告.pdf`) preserved through the processing pipeline +- **Subject normalization** — Strips localized reply/forward prefixes beyond English: + - German: `AW:`, `WG:` + - French: `TR:` + - Spanish: `RV:` + - Italian: `I:`, `R:` + - Portuguese: `ENC:`, `RES:` + - Norwegian/Swedish: `SV:`, `VS:` + - Dutch: `Doorst:` +- **HTML content** — `html.escape()` preserves Unicode characters, only escaping HTML special characters +- **Labels and rules** — Unicode label names and rule values work correctly + +--- + +## Extending with Custom Handlers + +To add support for a custom file type, subclass `BaseAttachmentHandler`: + +```python +from attachment_handler.base import BaseAttachmentHandler, ExtractedFile +from connector.utils import compute_file_hashes + +class MyCustomHandler(BaseAttachmentHandler): + def supported_extensions(self) -> list[str]: + return [".custom"] + + def extract(self, file_path, passwords=None) -> list[ExtractedFile]: + with open(file_path, "rb") as f: + content = f.read() + return [ExtractedFile( + filename="parsed.txt", + content=content, + content_type="text/plain", + hashes=compute_file_hashes(content), + )] +``` + +Register it in `attachment_handler/registry.py`: +```python +self.register(MyCustomHandler()) +``` + +--- + +## Case Content Format + +Each email is rendered as an HTML block in the case's **Content** tab: + +- Emails are visually separated by horizontal rules +- Metadata (date, sender, recipients) is shown at the top +- Email body is displayed in a blockquote +- Attachments are listed with filenames +- Replies and original emails are labeled accordingly +- If passwords were extracted, a note indicates how many were found + +The case **Description** uses plain text (sender, subject, first received date). + +--- + +## Behavior + +### Persistence model — why the OpenCTI API, not STIX bundles + +This connector writes to OpenCTI through the live pycti API (`case_incident.create`, +`stix_domain_object.add_file`, `update_field`) rather than emitting STIX 2.1 bundles via +`send_stix2_bundle`. This is a deliberate design decision required by the connector's behavior: + +- Attachment files are uploaded to the case's **Files** tab — binary attachments have no STIX + representation (a STIX Artifact SCO is a different, graph-level object). +- Email content is written to the case **Content** tab as HTML and **appended** on each thread + reply — a read-modify-append that STIX bundles (which upsert, not append) cannot express. +- **Case templates**, **assignees**, and **participants** reference OpenCTI-internal objects + (templates, platform users) that have no STIX equivalent. +- `send_stix2_bundle` is asynchronous (queued to the worker) and cannot return the case's internal + ID needed for the immediately-following file uploads and content appends; a hybrid design would + race. + +Idempotency is preserved via deterministic STIX IDs (`pycti.CaseIncident.generate_id`, +`pycti.Identity.generate_id`), so re-runs upsert the same objects rather than creating duplicates. + +### State management and deduplication + +- **State management** — The connector persists its progress in the connector state: the + `last_run` timestamp, the set of already-processed message IDs, and the thread→case map that + links email threads to their Case-Incident. This lets each polling cycle resume where the last + one left off and route thread replies to the correct existing case. +- **Deduplication** — If the connector state is reset (or lost), existing cases are found by name + before creation, so a reset does not produce duplicate cases. + +## Troubleshooting + +| Problem | Solution | +|---------|----------| +| IMAP connection refused | Check host, port, and SSL settings. Ensure firewall allows outbound on port 993 (or your configured port) | +| `OAuth2 token error` (Graph) | Verify `tenant_id`, `client_id`, `client_secret`. Ensure the `Mail.Read` application permission is granted and admin consent is given | +| Gmail auth failed | Verify service account credentials JSON path and that domain-wide delegation is enabled for the `gmail.readonly` scope | +| EWS autodiscover failed | Set `EMAIL_CASES_EWS_SERVER` explicitly to the full EWS endpoint URL | +| No emails matched | Check `EMAIL_CASES_SENDER_ADDRESS` (must match the `From` header exactly) and `EMAIL_CASES_SUBJECT_FILTERS` syntax | +| Password not extracted | Verify the prefix/suffix markers match the email body exactly (including whitespace). Check connector logs for extraction count | +| Attachment too large | Increase `EMAIL_CASES_MAX_ATTACHMENT_SIZE_MB` or check the file size | +| RAR extraction failed | Ensure `unrar`/`bsdtar` is installed in the Docker image (included by default). Check logs for "bsdtar not installed" warning | +| 7z extraction failed | Check logs for "py7zr not installed" warning. Ensure `py7zr` is in requirements.txt | +| xlsx/pdf decryption failed | Check logs for "All password attempts failed" warning. Verify password markers in email body | +| Duplicate cases after restart | The connector deduplicates by case name — existing cases are found and reused. If the `case_prefix` changed, duplicates may occur | +| Labels not appearing | Verify `EMAIL_CASES_LABELS` is a comma-separated string and `EMAIL_CASES_SUBJECT_RULES` is valid JSON. Check connector logs for label creation errors | +| Severity/priority not set | Values are auto-created at startup. Check logs for "Created vocabulary entry" or "Failed to create vocabulary entry" | +| Response type not set | Values are auto-created at startup. Check logs for vocabulary creation messages | +| Case template not applied | Verify the template name in `case_template` exactly matches the name in OpenCTI (Settings > Customization > Case templates) | +| Author not set | Check logs for "Creating Identity" or "Failed to resolve identity". The author is created as an Organization | +| Marking not applied | Marking definitions must already exist in OpenCTI. Check logs for "Marking definition not found". TLP markings (`TLP:WHITE`, `TLP:GREEN`, `TLP:AMBER`, `TLP:RED`) are built-in | +| Assignee/participant not found | Users must exist in OpenCTI and be matched by email. Check logs for "User not found for assignee/participant" | +| Non-English subjects garbled | Check the email server's charset encoding. The connector uses RFC 2047 decoding and falls back to UTF-8 | + +--- + +## Debugging + +Set `CONNECTOR_LOG_LEVEL=debug` to emit per-email fetch traces — each fetched email is logged with +its subject, sender, date, and `thread_id` — which makes it easy to see exactly which emails were +picked up and how they were grouped into threads. All logs are emitted as structured records via +`helper.connector_logger`, so message text and key/value fields are logged separately (e.g. +`helper.connector_logger.info("msg", {"key": val})`) and can be filtered or parsed downstream. + +--- + +## Requirements + +- OpenCTI Platform >= 7.260715.0 +- Docker (recommended) or Python 3.12+ diff --git a/external-import/email-cases-importer/VERSION b/external-import/email-cases-importer/VERSION new file mode 100644 index 00000000000..af0b7ddbffd --- /dev/null +++ b/external-import/email-cases-importer/VERSION @@ -0,0 +1 @@ +1.0.6 diff --git a/external-import/email-cases-importer/__metadata__/connector_manifest.json b/external-import/email-cases-importer/__metadata__/connector_manifest.json new file mode 100644 index 00000000000..49cf40f2048 --- /dev/null +++ b/external-import/email-cases-importer/__metadata__/connector_manifest.json @@ -0,0 +1,19 @@ +{ + "title": "Email Cases Importer", + "slug": "email-cases-importer", + "description": "Polls email inboxes (IMAP, Microsoft Graph, Gmail, EWS) for emails from specific senders matching configurable subject filters. Creates Incident Response Cases in OpenCTI, grouping email thread replies into the same case. Extracts and decrypts attachments (7z, zip, rar, xlsx, pdf) using passwords found in email bodies. Supports extensible attachment handlers.", + "short_description": "Email to Incident Response Case connector with multi-protocol support", + "logo": "logo.png", + "use_cases": ["Incident Response", "Threat Intelligence"], + "verified": false, + "last_verified_date": null, + "playbook_supported": false, + "max_confidence_level": 50, + "support_version": ">=7.260715.0", + "subscription_link": null, + "source_code": "https://github.com/OpenCTI-Platform/connectors/tree/master/external-import/email-cases-importer", + "manager_supported": false, + "container_version": "1.0.6", + "container_image": "opencti/connector-email-cases-importer", + "container_type": "EXTERNAL_IMPORT" +} diff --git a/external-import/email-cases-importer/__metadata__/logo.png b/external-import/email-cases-importer/__metadata__/logo.png new file mode 100644 index 0000000000000000000000000000000000000000..50cb72214ce698328f0a08fd3de2ed57a3e6e9ed GIT binary patch literal 22952 zcmV*fKv2JlP)vc_9gbKu8EFVEdm!NO&ZV5MJ^^ zLJ0{V(@6*kV9OYrUP7~R0UK=DmSwf=yEAkDzjJ41wUSn{w7ZgarTKn{)vi~wvorUc zd(L+ngNTb9VGlT#Fs<8n*EW>_(J_#mqoM2L!B{nbabRpDbbTZwRte67 z0DR!^f-xrmH^5|ocll^Q*8mm=cmkY7!4L)z0{C7q)(9PYAhGS>*a=`e7&7a5w=;ZvQ)72e|p8bMtrevBxu#0LF|*snhT2V{1SvL7x!=K+0h!bp3Mx zn;LpzOSti1|7Bn7-(?LumTc3tYcWdK<)DHkJe8FS7w0e zM}o5>089fg0WOaR4woT}B!F59P*p&u9};RI+3njp2NQRf9tq@Xfl0nuqB<3KVI0r)m>z8*|^8=SulS^fa$jo99;LRbtPjCD3mZ1f{+ z1uzhxUAzVo>Y#TtfjPsYjR!%Jj|JD}f%Bum`Bb=lZYT~zu+^mTqiZ@i{Vk`vM&d~K zj`dBj3yrj*ptGUhxi}ThWylJH;)M7fq$pBJAQ6jzu?=ACb#UorNYZOqSiYs}wW^a) zSJUZQ+2{}03ZTD*egVj@Zr{}&Yp4dKIpF%q0Gk8gV0Z(H_%{-C!?f1GxwVHI!p;Uu z$a92o{`9>K#Q;c<6-ihzJ@Aj{7~OT3Ensv9zn!B7XetKzg#D3sQs zDt1`AS51x9!IXWn@epV@1)QG-hGXFJl?ij1gq_S~vgtX!pOd3qdcxiprC_)zx=?OR zlHqhiCSQeIbZu?qg`v672E(UcEX?2bUFwOZ z8Q}U^06P<0KN{YkY%KB^`R$rvF2iKmhUk+Wp<^+CW8n?RhVL7twXT}JyEJ5X z_`35mV@eup6emjXr?103&6V zE5qTS{EK&>skdR#s7Jco_hF3Ts{lIh1ds1I8<2ktw|ADv(4!q+gz!pJ@U0tbIVr3c zH=SqsT^QG${;ebxfy+zv2|yyT23qo`SUmdQ8R5fWTZA+`6#$WXRwdqwJ+t6YmP2-& zE>?RYLJFbi1z)5O(phHQ#B`pm^@}L~?evemKDhOJ`-e#Mlkv7}A5-*-!0nff5^4Q0 zB;(7mWX$`+nI9Nl3V>q7%S5sWNA2!0P?Vd&a21>$8Ie{>E2RZrbb>Es$RR-^fwLrC z6aky!M{lAx%M3n+A)pwKL50DuFnDC+M`3_hX|M04<3?4E6bE7&7e8T@Bb4HZXdG=R zjwpSo*3IWhO!0>VlhKT~;46Ewo zblUC{d%p%IErZKjjc|*(D2jvJvvzh`i6lT;rl8ZKkTf9Zk}%FKW1>&OWS@))UKta; z62`b?jCL`UIV1!e3|^U~1+%Z9&yJ`ZZ7N4=O2@9a@v|+e;j^fYPa+yVi)g5i>j))v zP~D@E#@lW#kg_3IdZTER0@pQ>ZtN?k1g@>nw41Sb;B9Aq+MwQg@duF-*J1 z8jggd+zyv_0pd~0B%`c7qZD@#A$1mbvD=z=M`75{D`8qd!C~bJ4l7e|aDaqe#z>bW z{jN1&tq6*dG&o%f z9Bzsj{2lSwjks{^dZYRfY^YlULsv`e_+??rXG3Q}>g>K-ujPVlL|~=@F|Gfs2&o*K zLmJ*{PGRld6kcme;p317ofHB=rg|TdMU^&g1`pimrsOS?k&H*c)#X??@^0O?3+Zpx`D3~+CffGkLF}uovg937of-W}H zJcYYZu;ez==g+1X=sSNLQn9u%g_R9StZPi+bK+agYhYOigQ^TTxdn<0Uy#y^*CH9Y z8W)WJ&rs$D27CSVAH%=6R>p-V{uoUDF=RzSJVI%^@<3REJYQj;;L`psBW_1{A^Qc!B^s&{eyztkUsfamvc`{p6kY_`H3xLrSUht_Bn zlQb(5j;K&@&NvsoRP7SKXTniRv zd&ZjRSVSaYJn|eut>46#Cw*!Vgf$)8yp=H6&?EX9uYjcd1`bCM(a?bI`qsi0yEpxh z0~3NW&K>8%`QzOPjch*Y8QeI)?tD7M4k;tWLgKA%Ah&P@pOFx_ic@#u7TR1ll#QT zm`X<+2j&EI4T{s~Uzmvh4;nxHo~SFxOeiZ+l=d>TZb0p4Gobh%fX6=*;U>!69}tG$ z+PN(=e}qrQg%jMkVt)^gC~r6Mhx1O~SEMr;JGlw3?M>mHPh(iIJ%LuL%M$K^k<2t8 zq#@T0QaMm534$=*b{Q_5xTVyi1gvEJeWQD>j@q5)!{PoT6h{CtGWjKCzyzNpTL%s( zQ}DIP9$Y%fjcT_+7+dD8E%dp=67I}v_<2;ry<6kB=aU#dZ&L?D5vcH%5kc61c=8Hd zFzVS-t|urx1wd6?R^PYB_WT4+?~jo*JMH!}(brk<52;Xa-Bd3wn&3{y>B$(#Bmr#{ z29`p#tb&#l#{*kqxNCC^o0?UTS3t8!1|$WU>lz#`*=X&ayb+5={;E_Hhe~UiT49*G zKJa7g*n8m(ED|Qau1jG3Cvs_W&ouQ9sdV7_gS@z8q8o1WV@W5_drIqqmL9AW3X3bC zROS1(#_{WoQEYBWiD-)=4**%nxh}N4uR=m3bT8h2@#|Q=Bw1>q!qQorpF zi7ey;5G4p!Du{&Y5NSRiU!KxXO0@*6q=f(B`lI3YJ_f}(9+6g6w2SU1kWh=<`6NfV zM-n^!wFi1pZZi6|;1{PNXh`U|G1d-1`sfsb@We&&+^|jvxw!g{XJNEs1N)5jE}QT{nAhb>)SLe{VAbn5T7b;cYg~m-*1tKa_Ax!EA`v1pX3O*B;0bS57$rirWcFN>X&xZfEt|qazhlq*bqTH z#i2O*(Hh{o4uuE^MMo@p1uhu-Krt%;R@BWqYIolRU+`AMLmD)#Uzq;PUH|m)Zrm|5 zAgb}KxN@$|G;9sQCJuZFs#Q3CStZ)>HV}0%NeLE=I$LTfU1WWB$9?lmII} z!modYUzmNGX!MVz(|x$*5TCdR_O3sCVBPDa4*cl-FmB%v5t2YD#ah280omv;5Ni1j z&L8vZVp0Odp#c8yrbAR$)$Y6rfvQ^(ZdSp0!G&L#{7D@X0}B3hOb{oHa;E#_*}MMm zf>mQcpo+u)dLx8}s0PKwjH=H5ums#b>OTDqoIm#W#i9gQ@x=LeT!UccouYv?1ROi z1XyvH{q>i^6MO*4s0NMl`U@xgq~&R*C*A3r4)x*I8U8Npu#Mpj)l>rXc>TK}+_fPJ z<}hX*6f}d7h(X1ni>jqq_#9lY-!p@)1O{&i#!~Z(KlJ%&aC+}UDsG1Q3Mc#|;JV7; zm4Ls_E)&8pnrqs^ZzBVWDd@CbetSd!zdtqznbwKM3%f~B7+Y11AGiMzJhXkbkO3mU z2ML3-2U-qnn-r5#?*sK z09K0c!#jS8V8ud&n^T4C-zS7W!SU@weYp4NV4A_TjZ5|(=U4|M( zS_w!BLQN?IMqG(U>aRCQK|WxlARAE9@8Ru>;jgF_Q+g=TrvRUyZr7FI_`zX5{Cv9K zZq{L=U)q*S47mF35bjlAsO-xW{$(@$SbB)huG_cKFW?!~p*Or2#vLC-3}UcwNhNpLUp(wRb9DdPir$A-Bw(#2+C@`yM1PA{6Fh9O?sIRb`0ENI)2^}u? zNGRGvV&=sFkb?cI0Ep0_*k+8~1%LT;k>*>V*4>8BuM4x^-uZ0|q!Lg`2^{XnH_75m z@B%6UiE0H@1S;lW+SFeT1Su%>Z)q=6@ksq;2vlAn*vAj+;?=)Cj4v+`qZL@>ss;T)|f#I7Q|!cIusW@G(nBb#NzSw{WWx?@D>31Eabb8 z*5dqrIGlc{NnQ})H{<+gOmO4RvuxqFF?e8Rl_}w&iWf6FZ@B5)fwRnUVk$JmB$++y7N!=5rkighG#A9O+A(`Pp z8WQv5oH86#PF=wZqy#KJz+ZVP9^QG4F$A~pu0Ua30IQ&3Mg1%|Jg-7isiaD3*Ohsc z^r{^D`DDCuQYH5D81>qAl8=pIfM$jVH-HxHB3!YUl75_kbdN}C&?%20Kj9aqmJWDihR5fW3*on>^4PFZ43N#pz*N6c zc`8xEIkTO7ULdzwO#%+L2T=ZKgw^S=g$oT9P64cdWMnSw`Vj(Ehl@go0)+eoDV$HU7@5Ie;YQ6v4H5LQBM4JRsdQ`FPtk7ZBr4boQK+7*9aM_Eu7p+g){x>#CFYq z(|r>nEwuH^g$noGG{cVz$Ghx$ej6pB1imrZgRdV56XyzbU0i;#X~zMVKHW7IlegFja#R;r})~iQOuCd9DzFz58${lE1OxfWz$#K<0Ofr~pOUib4U@a>MjrvHMB{D(4m=+fR^w8ppVRj0cYi8eMw_y@8<( z8@=SLAf}V!;p56s<(9y;0&rF$D$t_ASMepRs9$JU-1$cZ@>2k`>wgH2+WL`@e@9j-Na^#sGV9XcbU~(Kt9fos9eUeX_aPB%u#3QMLV*GYEU5c=C z@(_OdL#_jUr-XZF86ACWzu!hFgPD&miQ)dEgK)?N$PXYwkmBNQuNSKJ^Lz=x+!TP- zt>n>$)8Q*S8PPTkEI;B;v|mbb+BtwHY9^}O_V_d?`&3|(f zX~Gm9{DpW*gy4WEfTgQwZl8|5hg=AM`QeC1DCjTeZTh<;+$eyoiTu8YVfyP>HrCu8|?9enp?@K;PRCi>(a_OnWSXO42> z+Xs5>A|D&W!WH=OG%pUQP(+1meoBC>fUpi<`BA|9bA=T^y#)IJsc(ZlYBW5{U10dW z*!}Zu@-7g#$d*`2GsSDO{x*iM6~KWX z9qdDePZsGkd0GK9UW|n6yW$z;>J&r)kR~CGvSP(#7j2vW zHijW2+6V!Ra!L5^L0<5@`vx%T9K_*poFW?YP_7Pz2KRLyz5f9D8aSv|dF~Aft>&Hw zRyuISB)7N#c9Rbq!`&75)&X8r2W4o~Hz1z_a<0McmjU)o_G%MjVcCKeC`H!u7}G#5vh32v40W& zOmN(Dd=OU-Qo)aPW1}j^jqiu?j~xk=QO6;Zkg+j5A(`tioX~OYL0()t#oOWa{)QzB zw{3{xrnlRmxTM_2j?GMW7@XE=RSWxX0(}(RoghLlB zW6JdWYA4p7TrudvpBWOI=#_EKer}N~lQT(f%n{JwE3bs)Izzlxm%Vvtb_GzkOgD0h z+4)dYbk$g1gA*MRuA1yAnw=+WEh7 zpbyW_ufQ0WF>BHiezPO}pn*R7TL*Z=iRRTjl)fzW>D0Lq4XDXpA0TCo{ge4$yK@?3 zc^2Z4oHPH#?yqtz7~{nBpdy&|Vh}2e2jH1aNlbU(t|J5Z!?9&hxDmFr6P1Pr1QD$R zn#OVKkpcYaXrlz(y6*&QgBq+QYMlr1QnN-n^Id-u;t>W(nvcgC_Y;|ciD2P+p;V1iE;<(lL# z+9+KJ1+Q{Yj@*j5<+$l!qxnqceYadwdYSxi43Z@vm^J`cPITufFpLS;-5d0P^V2)H z_WMu(M)r`-7aHb&?ukE?+Bh~S5H4Pli*|E$Cstz7IAh3cCeNdp>maFfoU^6{4}O+Nzdq2gWMt8PE|k*-luwZm zx$abDq^pa>^SZ7oMe1Vy6I1wD<3W&>BMkFDw<*8kz$(Y0ac=k&BlK4U*1rIq&x(yv zoK)A0ZBZ>P^qqNuaUMzdiPs)t^l&Q{7sf{4rc2jUJMr>Kl{hL3*{7PyV0%o*7go36 z=`Ar4A|5QzC)3&+;gWD>wNsp6UT#1lLLU4aJh^AADC+B!8Z2cs|26Hje9eEMJr#zF zC%DD)Vv-x-_qipkX-MMu7n<>6LozM&op}Kg`2U$1z#oqb!fBe4gKf#!=mXg|)r)@- z+2`&^_7PO``D;xn%z3d1FEu2^y3mRsEtJmS5|SOWYA`p5AtD-lK`&DJ7lasgN)7I) z04x)MvC|0&}X2e1Vh3+W7*-2w}wuvX9}>c|DVTbWeY>K7tF^wjxHEx8$Z4 zf$(Q$2D)Y79C8ElDi9(!fMi)a0r%_)bgmH0IBt#cdn!5)-ZDRQmBxMLq;~|V;*fC8 zI9KNnDLPp9oKi3E*%(31s%Cr|)zZ^lEm$r>wOhi!PH5)^2sSTJB$<6lj>&@TtH60Z zkbRVD|K-=(aLYR($POmT+loZ?)3DBY;eKwx!sQVVkf0`YxP7A$RF6(i^-lkzg&Olw z*1rH*rrkfcQA1|zpUi&|`!4|j;g&z>mhfsr3bS8m!t=X($qSGYQTd=|kqu{b$?WR| z*>`eXGybwADwMvszeOhe$dH+1D;?NBXqE}Hd};$+fFzwHo{wyI?b{W=f(7P)K{gM` zgkd`5a^mzqo#Tu#&JH2AQUKKe$WDe`aUG|wZpLpmbff(lIX$zu1hmB7G) z>{DSbCv<$}K(CS6*9)?5Eh%?*9>q2T=mxkX;ESV;Xh0rr023j7js@vZzX>TF{u3MB zjwxXBRHWkOH} zB~H&&ilAr`1z zN6Fv1~XojJQNX&wO6@kDdE#REt{o$(nun*q1a z15KkQf_+y2rUp1Ds9>^BE?NJ-JJ4>=Gk5{Y7uXW%o);iC`Z=2yC>+_Ra{TPb0Pa0H z2#3@?+4soy1Zq|`V`~YKeLcaF#6yA#_Vdf)TI3{GqDe@K!|AGonmk0jwgQrb8Z!Za zYezv5rn{bd_8-M9l@2@OP(mgz@S4aMXvV*G_l_5EnY@5qTHJTC&qrk6+;aSIA7tN8 z*GF){%Pl2G_VvcHCQj|>awWg4K$1|GmkxTkcs7z~5i2*C@gqUCKlw~1V8CqC{2$7& zc!7q5hSOFz&9oe@duHy@JEx6wz`-b3*2WCDVL$XD+ zE4d#^5(z=T4DqZ6nklEpjRD6*sH6aLi2oa%$WiT;FooK#yf9{_rl;eg&?RQ0n|6=Lwx@u3Q4OZj1ZLk&*C@WQ9zKQgbp#Zpi z5S$`)H>b-e&PC@qj7W+i=>0cnynusv0ljBlz_bo7pJ?y`)T(N@*AZlreUJ2->}y42 z2$6mJg7qDzmnq_6EwnnEM9EiuP0cS8XU;rZRwy5z{+!NdY#U-2FVVASrBg%@^2{&~fJKX8h{op7RARpXA0W;spX`z5pA- zWM7KoCr9j?>{}r+`-TVE*Apz)aI8ndDC!kPbpg4p52_3#$sdbVoRW*x(EzU$od;R2 z1R%{clhX6ehTyu*pBIoBZhR{ws?(zT;036T;I#2WufKpr_IV}X;kjjE6n8eV&wB5z z8zNZpa?7wH`}P$Hpp3xr9@3aOt_yAz1+fum+A~sutX4yk-NL5GWBTv@Uc>w!rV6^( zbp%wMM!dkL9(aLK#0$(T$M>cWoqPd{?3?6Mgq3ste(r9`zLd`Kl{eb(!?(ji@P{MW zmkri6pw85^pmOdFpbG}247xs20;F+JCP z@$=ZROk-!NrrZ%RD`m(U(n~G4ErS=ZObgBOVFMWX~V z$v#SPKBltwWZ!Vh?8^m~+eh+76Rvai0VotGqJF55%y2+%S%BsWrIVp5KpTbN1sHC6 zs|}aDX4Da6DuIr?0Obo*;LP#v_WlB_SjfI|jLVYQXAX?E$-X?{Pb+{NHGq}A%TZ2I z0B^qidr2c%@^7nw!terr-4eyD7n;SCA_C9G3%DdaGp`)qA4Fat8`Le6b5vO6_(>9U_kWf z1q3h9kNpKYX7*KP&FrJ|%REQxoN3VI-)0l-#39*rBYlRo$&wq8zrZnJEasiVYXtyo z?gso1Xs)KfV;XG$3NeMXS&N5AQu5ycmYf19dj7z+ zxCm!oIw`=41sK^~G)u~*RGpdfUSZTja95*FI2rvw$zF_?C6D^Z4E30PxN z$LTM%U|UE-m;-0M)PkRHh-B?AAjW7}yugW-m>SHF7qG}aYF6^MIc2!zknEX#)_ZT; z5XEJ$w&JF@Lb!WNOnfd0eWnti^SgG62R}W+FO;7hMeZG}%jhZ4rduQ_B*R$vRgSSf zS&R|0V&`&Zx-0`iD#w@BwBo(JDZ!qLS$Q(UkKPX9qE}lH?o>z6fft}Uf|Dw7)&x&Z zynsdaO`y!a6Dx3G7P8M$SoHdpZ?@s4x7$Q!ABFj@exnV4`7EB^`8%KAj}G?X#u+{% zp~*G_1{?)Z0QP;$;lh*fYh0vt(#}tj^`dJzx{_+^H_pHMD3y=DrQ7$0Dge&@g1iWj!WMZG}h%WU->e5c9iFYt7Ic3yy+*2K?e z_=O2it>yJp_xS>unSGZ`@nU67Iri(JjQ8ypD&uWN-Oi3>yjK1KO`kY_Z7W`EF!~>4 zRy+zThy(w7WB?aW@gNoFHmN*dn=YqYBO|`IE z^J6;_n7^_KpM=z`nSH{Ik$@!UIA=`@UTsQsQ36&gxd)CZ!&wvENZIof_laaak$cWL z%>@QyG5UWz-(YdvTxqr|(7W0Ru5&yxrwk{Ja&{4Zivzg&oseiB=5VnNzH>H+5TRK| zAb5e-S`k(aVl5jlFtc32+LJ19)`W~Yg3e^$?B0=mi`KLttQn?$cS=eqG?@vm;>?#? z@P3<`wp7yV#^feyj}KzOIA>PnWCLkYr8<$k1&(tu8!%xE1<5gwkG1PIe>k@+9Iet#hZ+btByVpm=&c83=Z@-rryug+iPFP92fZ8oDQ0bHeFYv>| zOkRLKKV_Ej64^JQcVu60d-e$nLT1=2CMUJeP)w%|o8N(~ji=_60X&le5w3T3fzuisN_31aavEw-D^iWDd&``1yt?ZhJo> z1i81u-yJV7`^9EKzV*~!;AhkQB9Gq#;MkE)tl0zQZR6+K|#5wk*XP>Zc6owrU z4X3Vd5yL(@C;`(IpxP43hnP<1VTTCQ;2HNKFW!wYF0?ii3IJgv*_`EkL<&<`H_^@s z=n9Ka`^Av~d~>o#+`w$Y|JyAw{OFyKVEglFfo<^u%}E{Su4xgo^g8nbR(}C{PI>%q zol=E=&o37(r@JHj-f2-W_r+$c6l7mlQnJ@dz`?M&RmEv9wV=rq@XS00tJ0C8Ba;J) zkt}5IE@_a(O)ccCAwPYGNz8z?ks4?-p7i}l7RyWirX2~$4iWlF==kAb0et@;Z(1f;=ww=Qaae~iD4~UR{>!F%apz2b zn%%eVndJujyCI1S*R_g?!g?VJyn5gT{zLf!FJ$Khq%33~J^t65ZTS9MA#>_+#|poK zC;_*G=eNgk!K-Z@svZfJwKB6z!80e6!zVLviZ~2YCBV%`Y8^+Bq|9znxq! z975a-jiVu9oQkam+H3mH+utK*-&q2cDf}dWYxkza&MI_gp0gf$%?qTu@&cI*zGbB} zBy^mzx*7L~%syjx=#TTWl)w{PV`B8TMKVcS33Nzq1lJWwa9y%`O+apXenp z43z+32gzDqX#k{UNsCJL)Dfs&j0?^yCz#{2s6ENSsOi@+XN(Jv&n`>nwP)7!)8eN7 z|EI5RK{FNSP*WXxSq^>+)Cq;^D%#DsDPGKA<6X$INu3fk3o_>f@baG zk;R3VfKMWZ0vL|jn}V!La!f5(#PpsDN7u4mtEQh~-=9!NpFCszg#Zb-Wby)Mt!YX3 z?Y8m*1_MTjP{I%14vDbjHxKY+RRY&c@t{rB@#D8c;{C&+9z`;*f!J(wC5$QKmukvW z$yTUnkdTlY0M-Oq$?1qzB!FB1i|iT(_o7Qp|9HPF5?844WoO~1YeK&E8EabbVUtnT zJJ`Z+K}6pK!_SPg-a!uKrt2Xb-S-l3bm%4E7jD;3Z%fBb3xGJI&q)E~8o3uCK$D!5!!DAvOsO|G`f(VtV93$Ok)|yZ5DEwx=1rQO5+KN>> z5(cR@46aQf(B4ac?X#D_-A7O_0dG-_Nh>t0WwRx$K}{Kvo7`Lg#vuu7|5JKuU9L?= zW>9X&6pLQtqAN}2$qrRmHS?l!ZC-how%FEgO4sGfueagpZDvjXP`@w*Mu%PkFYlw5 zz|ldRJ;^g8`8Z5)b4u;7`1G`BgY4E;yAbHDm2tA0cJktEn`IU zP}pDM8|kTWnRIC*l5yJBa+(ofL;@K8(~=U;huF36TGOv{JaBv&zEo|DzRp(D|GoFZ z_``-M92qULY~+T0^%7tmdI^-_*wKz5*GoV$P5$>u0Z`g_Zif++FXCYim2Vcu6)!v3 zCrHGif_lxNF^KT_C5H9Xvf9in8)|D__?%|bjNcy}#6{zcn*MBMvp-oM!R?gH(u;0B zHVWHIfMzQEeO@`HS5hnVp;1Q6&Gpw1*TrBM`cgR&fb>U1_Ul?B+{(6g1@PX;-IU7% z#hE)1C;_`;y2x^~K5MAJBAnEuj$a)Wz*Unw#-<6DGWeg(G2HTQ*e3k?0m8bXUIJ}a zF9AJkFM$zGipo@Aa+y3NdI{*}_oIx>a6&ifx*Wim4u_L|*@AOMhGWI0(EqWi4M@n)0Z2Ixbup~o#V1NiP#Z`Ot$^!(3T|6r9^|`q zuiI-VfT7mVy0{ApJAHT`QEQ&{wJi(&Hjb4|aEEM>&K=p`__%7Mq{mcwa=a7(F- zSU0~6vH$fEU8HR1NC0v;sFz!E4}6aG;%sNQ-52FEuB2mQ_{=tKY$hai@PV^E8cSsbI0QnW6%=F*i~P5)6N zop^d~IXq^(wR25BRX(2jQZshPG=uajr7t-|VKo%F^}R5Du_2muAd1yX;7)4OW`+kD ziWgRs*2aX6*C@J{-$o-r6(r)z;+e_)nHE5evEwW3RYcoTU{o)dTS5@+hV@|$uT#jP zgm?g}rvKmyqo%*y4CQqeelZy1rDkjk4Q@jZ8$I+AcqfG4ZHc9Y-hx$8c+CMG{OkxS zBQEanpd~xRGquKairMAf13*?J#KIv++FRna#cAFi5Ig`l!?~kAfv#`Jl?O;4P~&)Z zXA(IdWRPn5CkA9ZcS1Qvxw1F(XiagPxuyl{n^lpJVhew9A+wjjx4P*iU|9w~JlKct z&+wH*FM+@BOoEwRfOGc$oizIi@8Z1b1}x>m|L+*`TV?V9OkNF#i#!HBcQ=6HAG?wr zgs9ksf0U@{ufRla_L}~fS=0Y!1J(4~Wxd4;tCs+|W|!@wm%uNqUIJ7>SY+$eBGqZ# zQR{fF7-QISJeZLH#Mnylyry05+Z8}MAAmg%O=UUr0A@IFeT$0!w5VO$P8So1;G+bQ zGN*)R=U0g7IGtUKKGGWHUuAb%=#pAv@$;Ck7j7!8#tVPZV!BkWtf1 z4v)m}^t^H$Rjy>M>8A^H@#}4P_OtkacFDDoAB;{_MlXSL*R=N7OW^)vj9voqIOW+F zj}jm}x+7tfpXYuEU9_P}MC;L%GW`D~?IPKJTr&U3H#jTU2##0a@)*8DPM>IK0UCML zId!%eAY%GvJowSM6ams!! z=)?&OvJ%M5{(HJUDHdiP`;HEmho%A7;i_>_QTv1TLBJgqi{Ts4CHaT2Idbd}WL@|T z%_+RPH`QS(Z81O$y3jcuo?V91$2hx`^;)t18{P}!j~gOF`0cXZ((WbDl+jC|b3+jh zqn7~XeM}qW6y2K#`<%HM``;1D)BL9-KO*=!|A%;P)&68v0Cnj)mgf*@O+liM&M_Kb zO8s9yiyP;G;sKfdw;fxCh2vaZ%6ctJVA)3z+(9+{Zd>?=S}%be@lIvLFn*_cfq%?5 zTD)1W4;E-iKDaF*1ShZRKUt9xX$wJx;s4i&;nN*Hr}H0CV|b3*o$KHU9FMr@^x5Z5 zK|-_Yz$lM|O{Z0f9`9CTgTV&Xkyp~R*^D?p!BPfyY>MKBH>lsdP52EZ2;wiraag4T zD{CrHWHiwy?9_H(0R3 zlYiP8H&TCdJ`$P#4jIw5FeLmloB5wz0Z>*TMFTDzwGo>7Dm*^AJUI>i6NAPShPAs> zqH?ZS$iU1SOxoWXih04yEn*?iqM+Jwa$)SBp?3|ERITn{`1faVoU^tC3DX1^?4@bR z#_ByuVWrSul-zS@m!%wlVlnJQ7~soy5%_MsVwNe>yBUn4piRItGjU z%fkXNj2TJ(n|K^KGV3?X{&lHVc=>W2e`%( zx);_zcH+dn$E(qb$|i+7zXE!;R-!t_y6rCWB1Q91g*gn zAVu&uTwLA^&a=k;GtqluPjUPf$rvRj=S~KiT)@^A7612XOnkU)1=tv_(9PZdj!o^m ze;##+RPrk$7D@u@-Ral%fqe@z5eSqSxNyYlh{s=o$4?YR-c|s0Zf&NP+BO$pV;Dj= z`%UbK>9})a6p|xf-ya&h0UG@A4=ma5V_~^>D*VZY-tFdxnnbCL+fk7EY@i{e;kJ*X z9f~n**cdM0X7pj%`UpZXn)s8aJ4`b^sO`5T{db@Ktk>@(E8x%-44C&O-2TInhzb`b zr*k!bTnu>slqyX2%SD$RXv0Q-!Rj{kZi|YepKlU{XZf`t;pF?vh=x{U$(WjKWxw5_ z_x?Q92P|2l0qj?zCw6Wr0mkW?IoGiy$BplYg)*>>02{*%H@p`XbuxL^A5g5{*g6Xg82UHuxQk?eck;DeT?ao6;Nkb0l+db zgDFoPCZYmPhFd?3;3KnbvmF(%G1Q=$jb~^+!CfCmAUpF6`w?hbm`+0j*e$v2{)E0K z^(@b*cd9{=3`RNSmXI_ot$g%1trA8 z|H9%iFB;|hecSyBxn=bc%SdU$lKcasVvyyp9UTm7cP4PhX4@68F*L(|%hpHm{+?tZ zM#@tjJwqx%>4QJaDBs7xC;-wlq;VFFT!mQZ8Tf)SICas>9Y$0@!p-l6@uBGo*iiu+ zL&fx8w>O1ferTBfdAqn`_b;b;e}BPwquv%-e>v>_$g45W;>Cs`BRg(HB9;(I{)Rb_ z6NK5%fOv}Ive(;?Dw0NxHf-byYwc3u+ZAuL31v_K)1M~&N{EI-kexS+9DomVKV3{8i!r0iP7Q*y53jQhX zuO)94Rk|yvfp71!{<0xI=bg0AGBC_-4xmN-2#TW`YJzHHItG?ETHF--=2_PZ%+DP3O#(G zvK2h_J-9rS*^qZ~kmw;O0av`%hV3*R#SRVHC_%q}LrB9n-fT1a`xolqG{2nVIxN^f zF39*igkKyk?-LS)AdwK`{)TY#^MxP;t-b+`Q5_e)YWhw#A7G;dz|BV*?S1|$tq3Kl zOuqoREhJOEav8Cdn<(Z8dHUuF{#7hVu*#B;dFc_U*fOu3V;Z~rFF6xCzePo6A21=BnNfy zqg)bxw>~On3XoY<06)rxjebGOH2taN-Cs9Hjo$u+u>Gm%j*9Q})OT^t$ohPVzf2U6 zTH{Pa1;kMs*1_#N2JtYJ56k(UARUyx?h7ZDWBv%IO$yp5TGH?7oe7-%QVSHBiI7+U zNPolhkF-3GCDmUnl<9AwfCGAnj96iC0-S#pYAOL)$$7kZ=llQ_9E;YrVspeG1??mu z8^uQYz1yPVl69?ur=unx`3t}3^2?BjwZY;3S|Lq;3x%D~MGZq%tx~XL%=<{jehhCg zuR4K_ZNwNFV>-@S(}Jd?F^y8?-5vtTgu zZF^>!S}umw^+{a#TAL7jt9sCeje&%2E;=Il7Ori@JA0C1Z5BxQbvWG=^4kH$dzGm3 zud6HQ&TpZAhV+VoBGf~0+-HbKzXpdpzftEIpbJhH`teWV_{y7YX;;8j0yYK;y7@@E z{{^qL;>8_tC~m1RjK8S#lc`NGgRWhMg=Kq1vK>wA?g#YWlx`veS8)Xwjeiu;mb(zB zP{4JyV2rDq;oglgT>Vaaj)1KMZ1gv{`9JbLFMquak5hQMaKdj?`uWQh#6tg1)cKPP z6_EAsjQ(|>iM1r+VlpIb-Se^t4My7v!UqsUUO+;}cc=UDtHb>1mP6Q22GYJTl_Z?SOuX*gOrtzh4d8f!Qpr zIVxzaRN>5i z4n@%+DLP`|8eF*Fx^~8Ud>{TY1z-gWX&9CO!`odpwq0)@<}w`L-8wGjS(_WL`0^gw(_`mxJ->j0_M4Rx~V95Fg5rM!x`$e;y*ODi|xA5|98ju48_+3y;nz!w9Ew z!|aA4Hu?h9Tr<><;+&UT@fKC~6{@($0!76Fl?oy)f5)QHU&7K=3YOJq`PTRsp9|2L z6kIYr2|aZl;?XT|`zSnEFmp%b3AiP!-j&4smCg7#WYiAWRfIPBTGjV!g(7R}9y2$9RHL$`z@CqQEBt$*T$L>Kg@g-=fCOF*GM5JJs*Gd-ps71xR zl}&iDA(>VJcCLVp?!Zm6pT^xhRi8l3%4TefXrjGEL4_Z>2A5YxGO-=u*0b@|X(7>C zZ-9lLFnHdvmD~K#j(PC-{taDoB9&yKl&U}w2|^l&0){(}3g87o$rVvO z|M%YwiA5q6uCT`dZFC|OF0~@kT!TyYe|Ml5|7;jsF|Y`a6+6y`)Bgn2lnyN=i8ROp zfsRB1&r=-VJjjdR91##2-sS~tSg_VYMC178TOmBKIR?qej27O7CH!>J8dz46h=)$Y z1>;xsSCLO|G5GtSr34;wO-0BWNa&bX?ZSP>1Tons+nIyI z9Na23XAr|^E`6=dEb1vR(=R?u=a3!L(;tY1&%*`do*Zb#zc>{D!MYH&_1}Q6{BFd; z8g#v&N+2B}NN|kwO8CR9AkH1VTXx7yH_B>5~z`|d2luaN04 zBi4E;&L96^F$h1Qc$R%y3GBK7zOvg93#-sIxzI|$;svOr>-vMexb-kUT#|9q?NH(H zA^H@vj{k;t!}#N-sCYp(^G5nb_?=!E(bmgx{@DAAMfi(L0d!UZ{_@*J;ZQ$V0#=%^ zp3pICloP){Du|<~SkWd3ht00PrZI)DzuAV5no^Km#=O3QQ}87GlEUHgNQkz51Lu$Z zeK84t@hN~VmO$`!#3LFswV=eH1rZudakvzQrPF=5@gScumB?Hu#dmRRrN1=-`dxZ}1bs+qxw>^#UF1nrHLUFF~v=Z3;4S0QbBN5Y~QG?;!M~-y|QY#|N zljN8?#))4a5x}epBQeoVBOZFY{&nw!@b;c0n3D|4eySItP@n9Ogz2A%UWW6>K8QcO z>A+R9QYbPMrvm5@9jLzyZttI=suHvmbxJ9?63CoK=u{ZKcd!pXnC68~ZXf1olZ2&( z%=^}y;`o0bMsVlGD0ID_h`uzq2E|3(RRZzI1;xVlm!1M3SQqZ0U1!7Veh51EAQdlI zLy^v~ItwAGV_KyHKbzrCM+)p_Bc%aRK%cSe({Rq4-w)$cDmm%TUEfOYr3^pLTM!MM zhl|FqEN0;^eg)7$3GA2$x94%ljuD7Q3R*yv)fzDKhR+%A#!sgEFr6xx&7={VA1H1= zpYTRg3b%X^!E-wjfDAFHqhISmbX|ons6b79hG_U4T(sXi#V-7%p#TWhh5XCbL*NfS z0jFnbp{ocx!!r5xB!}C<@Qo=R;Tlwn{-(AfC|1GW8r5*?`Y8VNNer6GMVoFb;ivyj zf~%tEWjyjWA}tGX$>eQV$`vdxZs9K-1wgP;CLh>53jWH6;q@;-TN7R2g69f!=N63e z$@unEFTOd&i*m{*wiQ9q2!2CC$DNyE_}!)`T4FljWc@!-mU9l8{2i#05ovt}&3i5u zg*}6q+dp&_0KvMj#Hfzg|9{~PT!~1F4k{$q1YB_aH!rvL~vB!LExQ0e4~J>P}H`76<5v>(aAE;R+Tx>R6NnJg*{NfC^o0xxq{ zBG_Gs3X9IywLYffo=;-<{U3@H5BI+@-mVCYE+js;g!A@uc!nj{PB|*o~%zGkswoF61n>WDEPXFAqFZW#3Iijyz?u#^pKsU!1NDY1(3-L zP(QW9X50p+`#VsRKq}GiESc4sSfyRYU zx+odd~5I}lw=>q*A9AsPKTmW+MQ=-)^4`U*R$doVDB)*WY9 z1*7fwgP%-5)p;!u;!UA}X)B&Kn!ZA{^b9SJ)WQPooABaa*U~kJc z_^$(Ymr8uUw-{>ctfN`>SmSk&m7hX!_z(>Zs5Q{pDiDj%s6Lu$btZPgOfq$VPX$ZZ^_)&Ci&_=&=(LhdzuUtkd(F{$y z5$9FiGgM6e&M38Y)q9s9-GO-53^-iBh08l1u{Jc%( zR_ia9kcc)y(|#mk`vfAFmk(ak&i=;GRsfv|!iTm`h0F6RI6Y^HNI^0_C~iS#WD1ig zSRrW;W`f}^RJbKf@hg~7Mn5tR4JsnvV5~>#Vkz`yx3+%od=ALLsvy#CM@+|-u!@gD z8s2MF#ZJFDtf7^{b)?wnphVWOG1@PLO@~gD@-VfzGfzpA|i4wta5}>T%_FYO6Vd3Z(CHu)`*7BqB=g0 zX=sS+qL+fPjhm(one^2CK11;ol_XgtpEpP(-v`jtA7atSXNJSBKRgvcyEQ;Nl8)Lf zWl+j*1lVgq{w1CdLvueHYz1H;GuBI`oindp46qyEa33vpj#yZSXf9KH z9D$8Aj3ge;B~hm@%EVJ^!PxCMx8ljH3V)cxRsfxm=?jt~IO3Xx;PSU2D|1ENX*6UM zhZI5h&_<_l9<_)gRzH=X9(^wY>Tj@c)ZeqZd^QB^1>7AvlR}T}J{6L3HFSOkyaDP0 z186B)bc32uG^`;KcxbwZ)5zJ6g;U^o29oq!oLf<6zQ-2)9$+hgp21MdB`gu)L4zcE ztocYt_$r_;f!kk&WP%7oat(}H9@{DynBAT%Ft?A1$W1)58ypV5|}hsMtUw6maNHoO6!Slh`MpeA*2P{@%9i(f2CW7fF zg0a(~>vQ3Bje)~O$ti|HAnSmHfcjtzWecH$z=NjeN^rU%h`eN+s`0i%l3xPXo`uHh zuw=xJF0R*Np*Zc`zCh^r7TUuK#=>mp25Rag)Ya%%#I^yrSNz=jX*mLH7E;NRpzAeY zYz905A7lqv21X29l-7wh)AV0yS_wkf871Xd!YV^?GB{l_WW`W^v2X~<*t&UqMDIh=TCz}R6d9r2o!&t_h^Foa zV;=zgRWRpT@Wi_~Z`95X#}s5<9kc!Vfq|_62Fpzt35FVxz1Gp3eXgYv;lwmBX(l+1 z1TY=IWJuBo!GzP?6H(McJ@itBVxX2nH27ZZk{v2|jOKduKl)tr*Y3lU+%V3Kw-^F# zR2lOQ$D~&zssWZEc5@PT%95mfC_0~bnBLa_&Od=9y$7AY1F&~cE^WXUDqFi;k2=WM z18$r50}opP3^pj;_|RpnTZKUQdrJjUd?HfGDPU|W7@GplCxPqxL6WMV>*a7bB`D60 zvQ6PgGg+?v(fr88G1-3&?D048mQByGpJ z6)j!AFS$mLY=JKh*b1PSEd)Y$N`X-OffLTrBaqZbL1JT}Yt;a&1~3MURe|$L02N@& z55NNs4;UQaaCg|%)1Qqv7*Bz-I2fV;4+CfhV@=TcUNC7VxV96rv=ch72WJf!IjXVS j$vD>TPOx`)6eIpWdpe{)Dxn4700000NkvXXu0mjfkn!gU literal 0 HcmV?d00001 diff --git a/external-import/email-cases-importer/config.yml.sample b/external-import/email-cases-importer/config.yml.sample new file mode 100644 index 00000000000..9728c3fbec6 --- /dev/null +++ b/external-import/email-cases-importer/config.yml.sample @@ -0,0 +1,212 @@ +opencti: + url: 'http://localhost:8080' + token: 'ChangeMe' + +connector: + type: 'EXTERNAL_IMPORT' + id: 'ChangeMe' # Valid UUIDv4 + name: 'Email Cases Importer' + scope: 'Email Cases Importer' + log_level: 'info' + duration_period: 'PT5M' + +email_cases: + # ── Protocol ─────────────────────────────────────────────────────────── + # Supported: imap, microsoft_graph, gmail, ews + protocol: 'imap' + + # ── IMAP settings ────────────────────────────────────────────────────── + imap_host: 'mail.example.com' + imap_port: 993 + imap_username: 'user@example.com' + imap_password: 'ChangeMe' + imap_folder: 'INBOX' + imap_use_ssl: true + + # ── Microsoft Graph settings (Office 365) ────────────────────────────── + # Uncomment and fill in if protocol is 'microsoft_graph' + # graph_tenant_id: 'ChangeMe' + # graph_client_id: 'ChangeMe' + # graph_client_secret: 'ChangeMe' + # graph_user_id: 'user@company.com' + + # ── Gmail settings ───────────────────────────────────────────────────── + # Uncomment and fill in if protocol is 'gmail' + # gmail_credentials_file: '/path/to/service-account-credentials.json' + # gmail_user_id: 'me' + + # ── EWS settings (on-premise Exchange) ───────────────────────────────── + # Uncomment and fill in if protocol is 'ews' + # ews_server: 'https://exchange.company.com/ews/exchange.asmx' + # ews_username: 'DOMAIN\user' + # ews_password: 'ChangeMe' + # ews_auth_type: 'NTLM' # NTLM or OAuth2 + + # ── Email filtering ──────────────────────────────────────────────────── + # Only emails from this sender are processed. + sender_address: 'alerts@security-vendor.com' + + # JSON array of subject filters. An email is processed if its subject + # matches ANY filter. Supported types: exact, contains, regex. + # + # Examples: + # exact — subject must equal the value exactly (case-sensitive) + # contains — subject must contain the value (case-sensitive) + # regex — subject must match the regular expression + # + # Use '[]' (empty array) to accept ANY subject (no subject filtering). + subject_filters: >- + [ + {"type": "contains", "value": "Security Alert"}, + {"type": "regex", "value": "INC-\\d+"}, + {"type": "exact", "value": "Weekly Threat Report"} + ] + + # ── Thread tracking ──────────────────────────────────────────────────── + # How replies are grouped into the same IR case. + # provider_thread_id — uses the email provider's native thread/conversation ID (most reliable) + # message_headers — uses In-Reply-To and References headers (works across providers) + # subject_matching — strips RE:/FW: prefixes and matches on base subject + thread_tracking_strategy: 'provider_thread_id' + + # ── First-run starting date (optional) ──────────────────────────────── + # Used ONLY on the very first fetch cycle, when the connector has no prior + # state. Later cycles resume from the last run timestamp stored in state. + # Formats: 'YYYY-MM-DD' or 'YYYY-MM-DDTHH:MM:SSZ'. + # Leave empty to let the email server return the most recent emails (bounded + # by max_emails_per_cycle). + # start_date: '2026-04-01' + + # ── Password extraction ──────────────────────────────────────────────── + # Passwords embedded in the email body between these markers are + # extracted and used to decrypt encrypted attachments. + # + # Example email body: + # Please find the report attached. + # The password is: ---BEGIN PASSWORD---MyP@ssw0rd---END PASSWORD--- + password_prefix: '---BEGIN PASSWORD---' + password_suffix: '---END PASSWORD---' + + # Strip all spaces, tabs, and newlines from extracted passwords. + # Enable this if your email provider wraps long lines or if the HTML + # rendering inserts whitespace within the password. + # Example: "---BEGIN PASSWORD---My Pass\n word---END PASSWORD---" + # with strip_whitespace=false → "My Pass\n word" + # with strip_whitespace=true → "MyPassword" + password_strip_whitespace: false + + # ── Display ─────────────────────────────────────────────────────────── + # Show sender and recipient display names in case content. + # true → "NCSC UK Contact " + # false → "contact@ncsc.gov.uk" + display_sender_names: true + + # ── Timeout ────────────────────────────────────────────────────────── + # Maximum seconds for a single email fetch cycle. + # Prevents the connector from hanging on unresponsive mail servers. + email_fetch_timeout: 120 + + # ── Case defaults ────────────────────────────────────────────────────── + # These are used unless overridden by a subject rule. + # Values that don't exist in OpenCTI are auto-created as vocabulary entries. + default_severity: 'medium' # e.g. low, medium, high, critical + default_priority: 'P3' # e.g. P1, P2, P3, P4 + case_prefix: '' # Optional prefix prepended to case names + + # ── Labels ───────────────────────────────────────────────────────────── + # Comma-separated labels always applied to every created case. + # Labels are auto-created in OpenCTI if they don't exist. + # Leave empty to skip. + labels: 'NCSC UK,Email Alert' + + # ── Subject rules ────────────────────────────────────────────────────── + # JSON array of rules that conditionally set case properties based on + # the email subject. Multiple rules can match — labels and response_types + # are merged; first matching severity/priority/case_template wins. + # + # Each rule requires: + # match_type — how to match: exact, contains, starts_with, regex + # value — the string or pattern to match against + # + # Each rule can optionally set: + # labels — list of additional label names + # response_types — list of incident response types (auto-created if missing) + # severity — override default severity (auto-created if missing) + # priority — override default priority (auto-created if missing) + # case_template — name of an OpenCTI case template to apply + # + # Match types: + # exact — subject must equal the value exactly (case-sensitive) + # contains — subject must contain the value (case-insensitive) + # starts_with — subject must start with the value (case-insensitive) + # regex — subject must match the regular expression + subject_rules: >- + [ + { + "match_type": "contains", + "value": "Threat Alert", + "labels": ["Threat Alert"], + "response_types": ["ransomware"], + "severity": "critical", + "priority": "P1", + "case_template": "Ransomware Playbook" + }, + { + "match_type": "starts_with", + "value": "INC-", + "labels": ["Incident"], + "response_types": ["data-leak"] + }, + { + "match_type": "regex", + "value": "CVE-\\d{4}-\\d+", + "labels": ["Vulnerability"], + "severity": "high" + }, + { + "match_type": "exact", + "value": "Weekly Threat Report", + "labels": ["Weekly Report"], + "severity": "low", + "priority": "P4" + } + ] + + # ── Sender rules ─────────────────────────────────────────────────────── + # JSON array of rules that set case properties based on the sender email. + # Each rule matches on the sender address and can set: + # author — organization name to set as case author (auto-created) + # marking — marking definition to apply (e.g. "TLP:GREEN", "TLP:AMBER") + # assignees — list of OpenCTI user emails to assign to the case + # participants — list of OpenCTI user emails to add as participants + # + # The author identity is auto-created as an Organization if it doesn't exist. + # Marking definitions must already exist in OpenCTI (TLP markings are built-in). + # Assignees and participants must be existing OpenCTI users (matched by email). + sender_rules: >- + [ + { + "sender": "alerts@ncsc.gov.uk", + "author": "NCSC UK", + "marking": "TLP:AMBER", + "assignees": ["analyst@company.com"], + "participants": ["soc-team@company.com"] + }, + { + "sender": "noreply@security-vendor.com", + "author": "Security Vendor", + "marking": "TLP:GREEN" + } + ] + + # ── Import settings ──────────────────────────────────────────────────── + # Polling cadence is controlled by `connector.duration_period` (ISO-8601, + # e.g. 'PT5M') above. `import_interval` is DEPRECATED: ignored when a valid + # duration_period is set; kept only for backward compatibility. + # import_interval: 300 # (deprecated) seconds between polling cycles + max_emails_per_cycle: 50 # Max emails processed per cycle + tls_verify: true # Verify TLS certificates + + # ── Attachment settings ──────────────────────────────────────────────── + max_attachment_size_mb: 25 # Max attachment size in MB + attachment_store_in_opencti: true diff --git a/external-import/email-cases-importer/docker-compose.yml b/external-import/email-cases-importer/docker-compose.yml new file mode 100644 index 00000000000..ef5bebfe93b --- /dev/null +++ b/external-import/email-cases-importer/docker-compose.yml @@ -0,0 +1,82 @@ +services: + connector-email-cases-importer: + build: + context: . + dockerfile: Dockerfile + environment: + - OPENCTI_URL=http://opencti:8080 + - OPENCTI_TOKEN=${OPENCTI_ADMIN_TOKEN} + - CONNECTOR_ID=${CONNECTOR_EMAIL_CASES_ID} + - CONNECTOR_NAME=Email Cases Importer + - CONNECTOR_SCOPE=Email Cases Importer + - CONNECTOR_LOG_LEVEL=info + - CONNECTOR_DURATION_PERIOD=PT5M + # Protocol: imap, microsoft_graph, gmail, ews + - EMAIL_CASES_PROTOCOL=imap + # IMAP settings + - EMAIL_CASES_IMAP_HOST=${EMAIL_CASES_IMAP_HOST} + - EMAIL_CASES_IMAP_PORT=993 + - EMAIL_CASES_IMAP_USERNAME=${EMAIL_CASES_IMAP_USERNAME} + - EMAIL_CASES_IMAP_PASSWORD=${EMAIL_CASES_IMAP_PASSWORD} + - EMAIL_CASES_IMAP_FOLDER=INBOX + - EMAIL_CASES_IMAP_USE_SSL=true + # Microsoft Graph settings (uncomment if using microsoft_graph) + # - EMAIL_CASES_GRAPH_TENANT_ID=${EMAIL_CASES_GRAPH_TENANT_ID} + # - EMAIL_CASES_GRAPH_CLIENT_ID=${EMAIL_CASES_GRAPH_CLIENT_ID} + # - EMAIL_CASES_GRAPH_CLIENT_SECRET=${EMAIL_CASES_GRAPH_CLIENT_SECRET} + # - EMAIL_CASES_GRAPH_USER_ID=${EMAIL_CASES_GRAPH_USER_ID} + # Gmail settings (uncomment if using gmail) + # - EMAIL_CASES_GMAIL_CREDENTIALS_FILE=/run/secrets/gmail_creds.json + # - EMAIL_CASES_GMAIL_USER_ID=me + # EWS settings (uncomment if using ews) + # - EMAIL_CASES_EWS_SERVER=${EMAIL_CASES_EWS_SERVER} + # - EMAIL_CASES_EWS_USERNAME=${EMAIL_CASES_EWS_USERNAME} + # - EMAIL_CASES_EWS_PASSWORD=${EMAIL_CASES_EWS_PASSWORD} + # - EMAIL_CASES_EWS_AUTH_TYPE=NTLM + # Email filtering + - EMAIL_CASES_SENDER_ADDRESS=${EMAIL_CASES_SENDER_ADDRESS} + # JSON array. Use '[]' to accept any subject (no subject filtering). + - EMAIL_CASES_SUBJECT_FILTERS=${EMAIL_CASES_SUBJECT_FILTERS} + # Thread tracking: provider_thread_id, message_headers, subject_matching + - EMAIL_CASES_THREAD_TRACKING_STRATEGY=provider_thread_id + # Optional first-run floor: ISO-8601 or YYYY-MM-DD. Only used on the first + # cycle (no prior state). Leave empty for "most recent" behavior. + # - EMAIL_CASES_START_DATE=2026-04-01 + # Password extraction markers + - EMAIL_CASES_PASSWORD_PREFIX=---BEGIN PASSWORD--- + - EMAIL_CASES_PASSWORD_SUFFIX=---END PASSWORD--- + - EMAIL_CASES_PASSWORD_STRIP_WHITESPACE=false + # Display names in case content (e.g. "NCSC UK ") + - EMAIL_CASES_DISPLAY_SENDER_NAMES=true + # Timeout for email fetch operations (seconds) + - EMAIL_CASES_EMAIL_FETCH_TIMEOUT=120 + # Case defaults + - EMAIL_CASES_DEFAULT_SEVERITY=medium + - EMAIL_CASES_DEFAULT_PRIORITY=P3 + - EMAIL_CASES_CASE_PREFIX= + # Labels (comma-separated, auto-created if missing) + - EMAIL_CASES_LABELS= + # Subject rules (JSON array, see README for format) + - EMAIL_CASES_SUBJECT_RULES=[] + # Sender rules (JSON array, see README for format) + - EMAIL_CASES_SENDER_RULES=[] + # Import settings + # Polling cadence is controlled by CONNECTOR_DURATION_PERIOD (above). + # EMAIL_CASES_IMPORT_INTERVAL is DEPRECATED — ignored when + # CONNECTOR_DURATION_PERIOD is set; kept only for backward compatibility. + # - EMAIL_CASES_IMPORT_INTERVAL=300 + - EMAIL_CASES_MAX_EMAILS_PER_CYCLE=50 + - EMAIL_CASES_TLS_VERIFY=true + # Attachment settings + - EMAIL_CASES_MAX_ATTACHMENT_SIZE_MB=25 + - EMAIL_CASES_ATTACHMENT_STORE_IN_OPENCTI=true + restart: always + cap_drop: [ALL] + security_opt: [no-new-privileges:true] + read_only: true + tmpfs: [/tmp] + mem_limit: 512m + pids_limit: 100 + depends_on: + opencti: + condition: service_healthy diff --git a/external-import/email-cases-importer/entrypoint.sh b/external-import/email-cases-importer/entrypoint.sh new file mode 100644 index 00000000000..bf0ea0bd79c --- /dev/null +++ b/external-import/email-cases-importer/entrypoint.sh @@ -0,0 +1,2 @@ +#!/bin/sh +exec python main.py diff --git a/external-import/email-cases-importer/pyproject.toml b/external-import/email-cases-importer/pyproject.toml new file mode 100644 index 00000000000..e47ae82070a --- /dev/null +++ b/external-import/email-cases-importer/pyproject.toml @@ -0,0 +1,28 @@ +[tool.black] +line-length = 88 +target-version = ["py311", "py312"] +include = '\.pyi?$' +extend-exclude = ''' +/( + \.git + | \.venv + | venv + | __pycache__ +)/ +''' + +[tool.isort] +profile = "black" + +[tool.flake8] +max-line-length = 88 +extend-ignore = ["E", "W"] +exclude = [".git", "__pycache__", ".venv", "venv", "build", "dist"] + +[tool.pytest.ini_options] +testpaths = ["tests"] +pythonpath = ["src"] +addopts = "-ra --strict-markers" +filterwarnings = [ + "ignore::DeprecationWarning", +] diff --git a/external-import/email-cases-importer/scripts/test_mailbox_connection.py b/external-import/email-cases-importer/scripts/test_mailbox_connection.py new file mode 100644 index 00000000000..8f6822e39ee --- /dev/null +++ b/external-import/email-cases-importer/scripts/test_mailbox_connection.py @@ -0,0 +1,492 @@ +#!/usr/bin/env python3 +"""Standalone mailbox connectivity tester for the Email Cases connector. + +Exercises the exact `email_client.*` code paths the connector uses, without +pulling in OpenCTI, pycti, or the connectors-sdk. Use this to validate mailbox +credentials and network reachability before deploying the connector. + +Examples +-------- +IMAP: + python scripts/test_mailbox_connection.py imap \\ + --host imap.example.com --username u --password p \\ + --sender alerts@example.com + +Microsoft Graph (Office 365): + python scripts/test_mailbox_connection.py microsoft_graph \\ + --tenant-id T --client-id C --client-secret S \\ + --user-id u@company.com --sender alerts@example.com + +Gmail: + python scripts/test_mailbox_connection.py gmail \\ + --credentials-file /path/creds.json --sender alerts@example.com + +EWS (on-premise Exchange): + python scripts/test_mailbox_connection.py ews \\ + --server https://exchange.local/EWS/Exchange.asmx \\ + --username 'DOMAIN\\u' --password p --sender alerts@example.com + +EWS with Autodiscover (omit --server; username must be the SMTP address): + python scripts/test_mailbox_connection.py ews \\ + --username alice@contoso.com --password p --sender alerts@example.com + +Env var fallback +---------------- +Any CLI flag can be omitted if the corresponding `EMAIL_CASES_*` env var is +set (same names as the connector's docker-compose.yml). CLI flags win. + +Exit codes +---------- +0 connection succeeded and sender fetch returned without error +1 connection or fetch failed +2 CLI / config error (missing required argument) +""" + +from __future__ import annotations + +import argparse +import os +import sys +import traceback +from datetime import datetime, timezone +from pathlib import Path + +# Make `src/` importable so we can reuse the connector's email_client code +# as-is. The script lives at /scripts/, and src sits at /src. +THIS_DIR = Path(__file__).resolve().parent +SRC_DIR = THIS_DIR.parent / "src" +if str(SRC_DIR) not in sys.path: + sys.path.insert(0, str(SRC_DIR)) + + +# --------------------------------------------------------------------------- +# Helpers +# --------------------------------------------------------------------------- + + +def _env(name: str, default: str = "") -> str: + """Read an env var, returning default when missing or empty.""" + val = os.environ.get(name, "") + return val if val else default + + +def _resolve(cli_value, env_name: str, default=""): + """CLI value takes precedence; otherwise fall back to env var.""" + if cli_value not in (None, ""): + return cli_value + return _env(env_name, default) + + +def _require(value, flag: str, env_name: str): + if not value: + print( + f"ERROR: missing required value. Pass {flag} or set {env_name}.", + file=sys.stderr, + ) + sys.exit(2) + return value + + +def _fmt_since(raw: str | None): + if not raw: + return None + try: + # Accept both "YYYY-MM-DD" and full ISO 8601 + if len(raw) == 10: + return datetime.strptime(raw, "%Y-%m-%d").replace(tzinfo=timezone.utc) + return datetime.fromisoformat(raw.replace("Z", "+00:00")) + except ValueError as exc: + print(f"ERROR: --since must be ISO-8601 or YYYY-MM-DD ({exc})", file=sys.stderr) + sys.exit(2) + + +def _print_header(title: str): + print() + print("=" * 70) + print(title) + print("=" * 70) + + +def _print_sample(messages, limit: int = 3): + if not messages: + print("No emails matched the sender filter.") + return + print(f"Showing the first {min(limit, len(messages))} of {len(messages)}:") + print() + for idx, msg in enumerate(messages[:limit], start=1): + print(f" [{idx}] subject : {msg.subject!r}") + print(f" sender : {msg.sender}") + print(f" date : {msg.date.isoformat() if msg.date else '-'}") + print(f" thread_id : {msg.thread_id or '-'}") + print(f" attachments : {len(msg.attachments)}") + for att in msg.attachments[:3]: + print( + f" - {att.filename} " + f"({att.content_type}, {att.size} bytes)" + ) + if len(msg.attachments) > 3: + print(f" ... +{len(msg.attachments) - 3} more") + print() + + +# --------------------------------------------------------------------------- +# Per-protocol client builders +# --------------------------------------------------------------------------- + + +def build_imap_client(args): + from email_client.imap_client import ImapClient + + host = _require( + _resolve(args.host, "EMAIL_CASES_IMAP_HOST"), + "--host", + "EMAIL_CASES_IMAP_HOST", + ) + username = _require( + _resolve(args.username, "EMAIL_CASES_IMAP_USERNAME"), + "--username", + "EMAIL_CASES_IMAP_USERNAME", + ) + password = _require( + _resolve(args.password, "EMAIL_CASES_IMAP_PASSWORD"), + "--password", + "EMAIL_CASES_IMAP_PASSWORD", + ) + port = int(_resolve(args.port, "EMAIL_CASES_IMAP_PORT", "993")) + folder = _resolve(args.folder, "EMAIL_CASES_IMAP_FOLDER", "INBOX") + use_ssl = _resolve(args.use_ssl, "EMAIL_CASES_IMAP_USE_SSL", "true") + use_ssl_bool = str(use_ssl).strip().lower() in ("1", "true", "yes", "on") + tls_verify = _resolve(args.tls_verify, "EMAIL_CASES_TLS_VERIFY", "true") + tls_verify_bool = str(tls_verify).strip().lower() in ("1", "true", "yes", "on") + + print( + f"IMAP host={host}:{port} folder={folder} ssl={use_ssl_bool} tls_verify={tls_verify_bool}" + ) + return ImapClient( + host=host, + port=port, + username=username, + password=password, + folder=folder, + use_ssl=use_ssl_bool, + tls_verify=tls_verify_bool, + ) + + +def build_graph_client(args): + try: + from email_client.graph_client import GraphClient + except ImportError as exc: + print( + f"ERROR: could not import the Microsoft Graph client " + f"(email_client.graph_client), which uses only 'requests': {exc}", + file=sys.stderr, + ) + sys.exit(1) + + tenant_id = _require( + _resolve(args.tenant_id, "EMAIL_CASES_GRAPH_TENANT_ID"), + "--tenant-id", + "EMAIL_CASES_GRAPH_TENANT_ID", + ) + client_id = _require( + _resolve(args.client_id, "EMAIL_CASES_GRAPH_CLIENT_ID"), + "--client-id", + "EMAIL_CASES_GRAPH_CLIENT_ID", + ) + client_secret = _require( + _resolve(args.client_secret, "EMAIL_CASES_GRAPH_CLIENT_SECRET"), + "--client-secret", + "EMAIL_CASES_GRAPH_CLIENT_SECRET", + ) + user_id = _require( + _resolve(args.user_id, "EMAIL_CASES_GRAPH_USER_ID"), + "--user-id", + "EMAIL_CASES_GRAPH_USER_ID", + ) + tls_verify = _resolve(args.tls_verify, "EMAIL_CASES_TLS_VERIFY", "true") + tls_verify_bool = str(tls_verify).strip().lower() in ("1", "true", "yes", "on") + + print(f"Graph tenant={tenant_id[:8]}... user={user_id}") + return GraphClient( + tenant_id=tenant_id, + client_id=client_id, + client_secret=client_secret, + user_id=user_id, + tls_verify=tls_verify_bool, + ) + + +def build_gmail_client(args): + try: + from email_client.gmail_client import GmailClient + except ImportError as exc: + print( + f"ERROR: Gmail client requires 'google-auth'. Install: " + f"pip install google-auth ({exc})", + file=sys.stderr, + ) + sys.exit(1) + + credentials_file = _require( + _resolve(args.credentials_file, "EMAIL_CASES_GMAIL_CREDENTIALS_FILE"), + "--credentials-file", + "EMAIL_CASES_GMAIL_CREDENTIALS_FILE", + ) + if not Path(credentials_file).is_file(): + print( + f"ERROR: credentials file not found: {credentials_file}", + file=sys.stderr, + ) + sys.exit(2) + user_id = _resolve(args.user_id, "EMAIL_CASES_GMAIL_USER_ID", "me") + tls_verify = _resolve(args.tls_verify, "EMAIL_CASES_TLS_VERIFY", "true") + tls_verify_bool = str(tls_verify).strip().lower() in ("1", "true", "yes", "on") + + print(f"Gmail user={user_id} creds={credentials_file}") + return GmailClient( + credentials_file=credentials_file, + user_id=user_id, + tls_verify=tls_verify_bool, + ) + + +def build_ews_client(args): + try: + from email_client.ews_client import EwsClient + except ImportError as exc: + print( + f"ERROR: EWS client requires 'exchangelib'. Install: " + f"pip install exchangelib ({exc})", + file=sys.stderr, + ) + sys.exit(1) + + # --server is OPTIONAL: when omitted, the connector's EwsClient falls back + # to exchangelib's Autodiscover using the primary SMTP address. Username + # must therefore be an SMTP address in autodiscover mode. + server = _resolve(args.server, "EMAIL_CASES_EWS_SERVER") + username = _require( + _resolve(args.username, "EMAIL_CASES_EWS_USERNAME"), + "--username", + "EMAIL_CASES_EWS_USERNAME", + ) + password = _require( + _resolve(args.password, "EMAIL_CASES_EWS_PASSWORD"), + "--password", + "EMAIL_CASES_EWS_PASSWORD", + ) + auth_type = _resolve(args.auth_type, "EMAIL_CASES_EWS_AUTH_TYPE", "NTLM") + tls_verify = _resolve(args.tls_verify, "EMAIL_CASES_TLS_VERIFY", "true") + tls_verify_bool = str(tls_verify).strip().lower() in ("1", "true", "yes", "on") + + if server: + print(f"EWS server={server} user={username} auth={auth_type}") + else: + if "@" not in username: + print( + "WARN: --server omitted and --username does not look like an " + "SMTP address. Autodiscover needs the primary SMTP address as " + "the username (e.g. user@contoso.com).", + file=sys.stderr, + ) + print(f"EWS server=(Autodiscover) user={username} auth={auth_type}") + return EwsClient( + server=server, # empty string triggers autodiscover in EwsClient + username=username, + password=password, + auth_type=auth_type, + tls_verify=tls_verify_bool, + ) + + +BUILDERS = { + "imap": build_imap_client, + "microsoft_graph": build_graph_client, + "gmail": build_gmail_client, + "ews": build_ews_client, +} + + +# --------------------------------------------------------------------------- +# Runner +# --------------------------------------------------------------------------- + + +def run_test(args) -> int: + # Resolve and validate the cross-cutting args BEFORE we print anything or + # build the client, so errors surface cleanly at the top. + sender = _require( + _resolve(args.sender, "EMAIL_CASES_SENDER_ADDRESS"), + "--sender", + "EMAIL_CASES_SENDER_ADDRESS", + ) + since = _fmt_since(args.since) + max_results = int(args.max_results) + + _print_header(f"Mailbox connectivity test -- protocol: {args.protocol}") + + try: + client = BUILDERS[args.protocol](args) + except SystemExit: + raise + except Exception: + print("ERROR: failed to build client.") + traceback.print_exc() + return 1 + + # Step 1: connect + _print_header("Step 1/3 -- connect") + try: + client.connect() + except Exception: + print("FAIL: connect() raised.") + traceback.print_exc() + return 1 + print("OK: connected.") + + # Step 2: fetch (with disconnect in finally so we always clean up) + try: + _print_header("Step 2/3 -- fetch emails") + print(f" sender = {sender}") + print(f" since = {since.isoformat() if since else '(none)'}") + print(f" max_results = {max_results}") + try: + messages = client.fetch_emails( + sender=sender, since=since, max_results=max_results + ) + except Exception: + print("FAIL: fetch_emails() raised.") + traceback.print_exc() + return 1 + print(f"OK: received {len(messages)} message(s).") + + # Step 3: show a small sample + _print_header("Step 3/3 -- sample") + _print_sample(messages, limit=min(3, max_results)) + + finally: + try: + client.disconnect() + print("Disconnected.") + except Exception: + # Swallow -- we already reported the real problem (if any) above. + pass + + _print_header("RESULT: PASS") + return 0 + + +# --------------------------------------------------------------------------- +# CLI +# --------------------------------------------------------------------------- + + +def build_parser() -> argparse.ArgumentParser: + p = argparse.ArgumentParser( + prog="test_mailbox_connection", + description=( + "Test connectivity to a mailbox using the same email_client code " + "paths the Email Cases connector uses. Runs outside OpenCTI." + ), + formatter_class=argparse.RawDescriptionHelpFormatter, + ) + p.add_argument( + "protocol", + choices=sorted(BUILDERS.keys()), + help="Which protocol to test.", + ) + + # Common (all protocols) + p.add_argument( + "--sender", default=None, help="Sender email to filter on (required)." + ) + p.add_argument( + "--since", + default=None, + help="Only fetch emails received after this date (ISO-8601 or YYYY-MM-DD).", + ) + p.add_argument( + "--max-results", + type=int, + default=5, + help="Maximum emails to fetch (default: 5).", + ) + p.add_argument( + "--tls-verify", + default=None, + help="Verify TLS certs (true/false). Default true.", + ) + + # IMAP / EWS shared + p.add_argument("--username", default=None, help="[imap, ews] Username.") + p.add_argument("--password", default=None, help="[imap, ews] Password.") + + # IMAP + p.add_argument("--host", default=None, help="[imap] IMAP server hostname.") + p.add_argument("--port", default=None, help="[imap] IMAP port (default 993).") + p.add_argument( + "--folder", default=None, help="[imap] Folder to monitor (default INBOX)." + ) + p.add_argument( + "--use-ssl", + default=None, + help="[imap] Use SSL/TLS (true/false). Default true.", + ) + + # Microsoft Graph + p.add_argument( + "--tenant-id", default=None, help="[microsoft_graph] Azure AD tenant ID." + ) + p.add_argument( + "--client-id", default=None, help="[microsoft_graph] Azure AD client ID." + ) + p.add_argument( + "--client-secret", + default=None, + help="[microsoft_graph] Azure AD client secret.", + ) + p.add_argument( + "--user-id", + default=None, + help="[microsoft_graph, gmail] Mailbox user ID or UPN.", + ) + + # Gmail + p.add_argument( + "--credentials-file", + default=None, + help="[gmail] Path to Google service account credentials JSON.", + ) + + # EWS + p.add_argument( + "--server", + default=None, + help=( + "[ews] Exchange server URL. Omit to use Autodiscover " + "(in that case --username must be the SMTP address)." + ), + ) + p.add_argument( + "--auth-type", + default=None, + choices=["NTLM", "OAuth2", None], + help="[ews] NTLM or OAuth2 (default NTLM).", + ) + + return p + + +def main(argv=None) -> int: + parser = build_parser() + args = parser.parse_args(argv) + try: + return run_test(args) + except KeyboardInterrupt: + print("\nInterrupted.", file=sys.stderr) + return 1 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/external-import/email-cases-importer/src/attachment_handler/__init__.py b/external-import/email-cases-importer/src/attachment_handler/__init__.py new file mode 100644 index 00000000000..da985ffa193 --- /dev/null +++ b/external-import/email-cases-importer/src/attachment_handler/__init__.py @@ -0,0 +1,8 @@ +from attachment_handler.base import BaseAttachmentHandler, ExtractedFile +from attachment_handler.registry import HandlerRegistry + +__all__ = [ + "BaseAttachmentHandler", + "ExtractedFile", + "HandlerRegistry", +] diff --git a/external-import/email-cases-importer/src/attachment_handler/archive_handler.py b/external-import/email-cases-importer/src/attachment_handler/archive_handler.py new file mode 100644 index 00000000000..146712485f4 --- /dev/null +++ b/external-import/email-cases-importer/src/attachment_handler/archive_handler.py @@ -0,0 +1,213 @@ +import logging +import os +import tempfile +import zipfile + +from attachment_handler.base import BaseAttachmentHandler, ExtractedFile +from connector.utils import compute_file_hashes + +logger = logging.getLogger(__name__) + +MAX_EXTRACT_DEPTH = 3 +MAX_EXTRACTED_SIZE = 100 * 1024 * 1024 # 100 MB + + +class ArchiveHandler(BaseAttachmentHandler): + """Handler for archive files: zip, 7z, rar.""" + + def supported_extensions(self) -> list[str]: + return [".zip", ".7z", ".rar"] + + def extract( + self, + file_path: str, + passwords: list[str] | None = None, + ) -> list[ExtractedFile]: + lower = file_path.lower() + if lower.endswith(".zip"): + return self._extract_zip(file_path, passwords) + if lower.endswith(".7z"): + return self._extract_7z(file_path, passwords) + if lower.endswith(".rar"): + return self._extract_rar(file_path, passwords) + return [] + + def _extract_zip( + self, + file_path: str, + passwords: list[str] | None, + depth: int = 0, + ) -> list[ExtractedFile]: + if depth >= MAX_EXTRACT_DEPTH: + return [] + + results = [] + total_size = 0 + + try: + with zipfile.ZipFile(file_path, "r") as zf: + # Try without password first, then each password + pwd_list = [None] + [p.encode() for p in (passwords or [])] + used_pwd = None + + for pwd in pwd_list: + try: + # Do NOT call zf.testzip() here: for an encrypted ZIP it + # raises before the password is applied, so no password + # attempt would ever succeed. Reading the first entry + # with the candidate password is the real validation. + if pwd: + zf.setpassword(pwd) + if zf.namelist(): + zf.read(zf.namelist()[0], pwd=pwd) + used_pwd = pwd + break + except (RuntimeError, zipfile.BadZipFile): + continue + + for info in zf.infolist(): + if info.is_dir(): + continue + if total_size + info.file_size > MAX_EXTRACTED_SIZE: + break + + try: + content = zf.read(info.filename, pwd=used_pwd) + except RuntimeError: + # Try each password individually for this file + content = None + for pwd in [p.encode() for p in (passwords or [])]: + try: + content = zf.read(info.filename, pwd=pwd) + break + except RuntimeError: + continue + if content is None: + continue + + total_size += len(content) + hashes = compute_file_hashes(content) + + extracted = ExtractedFile( + filename=os.path.basename(info.filename), + content=content, + content_type="application/octet-stream", + hashes=hashes, + was_encrypted=used_pwd is not None, + ) + results.append(extracted) + + except zipfile.BadZipFile as e: + logger.warning("Bad ZIP file: %s — %s", file_path, e) + return [] + except Exception as e: + logger.warning("ZIP extraction failed: %s — %s", file_path, e) + return [] + + return results + + def _extract_7z( + self, + file_path: str, + passwords: list[str] | None, + ) -> list[ExtractedFile]: + try: + import py7zr + except ImportError: + logger.warning("py7zr not installed — cannot extract 7z archives") + return [] + + results = [] + total_size = 0 + pwd_list = [None] + (passwords or []) + + for pwd in pwd_list: + try: + with py7zr.SevenZipFile(file_path, mode="r", password=pwd) as archive: + with tempfile.TemporaryDirectory() as tmpdir: + archive.extractall(path=tmpdir) + for root, _, files in os.walk(tmpdir): + for fname in files: + fpath = os.path.join(root, fname) + # Path traversal protection + real_path = os.path.realpath(fpath) + if not real_path.startswith( + os.path.realpath(tmpdir) + os.sep + ): + continue + with open(fpath, "rb") as fh: + content = fh.read() + total_size += len(content) + if total_size > MAX_EXTRACTED_SIZE: + break + hashes = compute_file_hashes(content) + results.append( + ExtractedFile( + filename=os.path.basename(fname), + content=content, + content_type="application/octet-stream", + hashes=hashes, + was_encrypted=pwd is not None, + ) + ) + break # Success + except Exception as e: + logger.debug("7z password attempt failed: %s — %s", file_path, e) + continue + + return results + + def _extract_rar( + self, + file_path: str, + passwords: list[str] | None, + ) -> list[ExtractedFile]: + """Extract RAR archive using bsdtar (libarchive-tools). + + Falls back gracefully if bsdtar is not installed. + """ + import shutil + import subprocess + + if not shutil.which("bsdtar"): + logger.warning("bsdtar not installed — cannot extract RAR archives") + return [] + + results = [] + total_size = 0 + + with tempfile.TemporaryDirectory() as tmpdir: + # bsdtar doesn't support RAR passwords, try extraction as-is + cmd = ["bsdtar", "-x", "-C", tmpdir, "-f", file_path] + try: + subprocess.run(cmd, capture_output=True, timeout=60, check=True) + except subprocess.CalledProcessError as e: + logger.warning("RAR extraction failed: %s — %s", file_path, e.stderr) + return [] + except subprocess.TimeoutExpired: + logger.warning("RAR extraction timed out: %s", file_path) + return [] + + for root, _, files in os.walk(tmpdir): + for fname in files: + fpath = os.path.join(root, fname) + real_path = os.path.realpath(fpath) + if not real_path.startswith(os.path.realpath(tmpdir) + os.sep): + continue + with open(fpath, "rb") as fh: + content = fh.read() + total_size += len(content) + if total_size > MAX_EXTRACTED_SIZE: + break + hashes = compute_file_hashes(content) + results.append( + ExtractedFile( + filename=os.path.basename(fname), + content=content, + content_type="application/octet-stream", + hashes=hashes, + was_encrypted=False, + ) + ) + + return results diff --git a/external-import/email-cases-importer/src/attachment_handler/base.py b/external-import/email-cases-importer/src/attachment_handler/base.py new file mode 100644 index 00000000000..fd2ac59397c --- /dev/null +++ b/external-import/email-cases-importer/src/attachment_handler/base.py @@ -0,0 +1,58 @@ +from abc import ABC, abstractmethod +from dataclasses import dataclass, field + + +@dataclass +class ExtractedFile: + """Represents an extracted/processed file from an attachment.""" + + filename: str + content: bytes + content_type: str + hashes: dict[str, str] = field(default_factory=dict) + metadata: dict = field(default_factory=dict) + was_encrypted: bool = False + inner_files: list["ExtractedFile"] = field(default_factory=list) + + +class BaseAttachmentHandler(ABC): + """Abstract base class for attachment handlers. + + Subclass this to add custom attachment parsing. Register your handler + with the HandlerRegistry for the file extensions you want to handle. + + Example: + class MyCustomHandler(BaseAttachmentHandler): + def supported_extensions(self) -> list[str]: + return [".custom"] + + def extract(self, file_path, password=None): + # Parse your file and return ExtractedFile objects + ... + return [ExtractedFile(filename="...", content=b"...", ...)] + """ + + @abstractmethod + def supported_extensions(self) -> list[str]: + """Return list of file extensions this handler supports (e.g., ['.zip', '.7z']).""" + + @abstractmethod + def extract( + self, + file_path: str, + passwords: list[str] | None = None, + ) -> list[ExtractedFile]: + """Extract/process the file, optionally using provided passwords. + + Args: + file_path: Path to the file to process. + passwords: Optional list of passwords to try for decryption. + + Returns: + List of ExtractedFile objects representing extracted content. + """ + + def can_handle(self, filename: str) -> bool: + """Check if this handler can process the given filename.""" + lower = filename.lower() + return any(lower.endswith(ext) for ext in self.supported_extensions()) diff --git a/external-import/email-cases-importer/src/attachment_handler/document_handler.py b/external-import/email-cases-importer/src/attachment_handler/document_handler.py new file mode 100644 index 00000000000..aae329dcf8b --- /dev/null +++ b/external-import/email-cases-importer/src/attachment_handler/document_handler.py @@ -0,0 +1,126 @@ +import io +import logging + +from attachment_handler.base import BaseAttachmentHandler, ExtractedFile +from connector.utils import compute_file_hashes + +logger = logging.getLogger(__name__) + + +class DocumentHandler(BaseAttachmentHandler): + """Handler for document files: xlsx, pdf (with password support).""" + + def supported_extensions(self) -> list[str]: + return [".xlsx", ".pdf"] + + def extract( + self, + file_path: str, + passwords: list[str] | None = None, + ) -> list[ExtractedFile]: + lower = file_path.lower() + if lower.endswith(".xlsx"): + return self._handle_xlsx(file_path, passwords) + if lower.endswith(".pdf"): + return self._handle_pdf(file_path, passwords) + return [] + + def _handle_xlsx( + self, + file_path: str, + passwords: list[str] | None, + ) -> list[ExtractedFile]: + with open(file_path, "rb") as fh: + content = fh.read() + was_encrypted = False + + # Try to decrypt if passwords provided + if passwords: + decrypted = self._decrypt_office(file_path, passwords) + if decrypted is not None: + content = decrypted + was_encrypted = True + + hashes = compute_file_hashes(content) + return [ + ExtractedFile( + filename=file_path.rsplit("/", 1)[-1].rsplit("\\", 1)[-1], + content=content, + content_type="application/vnd.openxmlformats-officedocument.spreadsheetml.sheet", + hashes=hashes, + was_encrypted=was_encrypted, + ) + ] + + def _decrypt_office(self, file_path: str, passwords: list[str]) -> bytes | None: + """Attempt to decrypt an Office file using msoffcrypto-tool.""" + try: + import msoffcrypto + except ImportError: + logger.warning( + "msoffcrypto-tool not installed — cannot decrypt Office files" + ) + return None + + for pwd in passwords: + try: + with open(file_path, "rb") as f: + office_file = msoffcrypto.OfficeFile(f) + if not office_file.is_encrypted(): + return None + office_file.load_key(password=pwd) + output = io.BytesIO() + office_file.decrypt(output) + return output.getvalue() + except Exception as e: + logger.debug("Office decrypt attempt failed: %s — %s", file_path, e) + continue + logger.warning("All password attempts failed for Office file: %s", file_path) + return None + + def _handle_pdf( + self, + file_path: str, + passwords: list[str] | None, + ) -> list[ExtractedFile]: + with open(file_path, "rb") as fh: + content = fh.read() + was_encrypted = False + + if passwords: + decrypted = self._decrypt_pdf(file_path, passwords) + if decrypted is not None: + content = decrypted + was_encrypted = True + + hashes = compute_file_hashes(content) + return [ + ExtractedFile( + filename=file_path.rsplit("/", 1)[-1].rsplit("\\", 1)[-1], + content=content, + content_type="application/pdf", + hashes=hashes, + was_encrypted=was_encrypted, + ) + ] + + def _decrypt_pdf(self, file_path: str, passwords: list[str]) -> bytes | None: + """Attempt to decrypt a PDF using pikepdf.""" + try: + import pikepdf + except ImportError: + logger.warning("pikepdf not installed — cannot decrypt PDF files") + return None + + for pwd in passwords: + try: + pdf = pikepdf.open(file_path, password=pwd) + output = io.BytesIO() + pdf.save(output) + pdf.close() + return output.getvalue() + except Exception as e: + logger.debug("PDF decrypt attempt failed: %s — %s", file_path, e) + continue + logger.warning("All password attempts failed for PDF: %s", file_path) + return None diff --git a/external-import/email-cases-importer/src/attachment_handler/passthrough_handler.py b/external-import/email-cases-importer/src/attachment_handler/passthrough_handler.py new file mode 100644 index 00000000000..d202c800850 --- /dev/null +++ b/external-import/email-cases-importer/src/attachment_handler/passthrough_handler.py @@ -0,0 +1,41 @@ +from attachment_handler.base import BaseAttachmentHandler, ExtractedFile +from connector.utils import compute_file_hashes + + +class PassthroughHandler(BaseAttachmentHandler): + """Handler for files that don't need decryption: csv, txt, eml, msg.""" + + def supported_extensions(self) -> list[str]: + return [".csv", ".txt", ".eml", ".msg", ".json", ".xml", ".html"] + + def extract( + self, + file_path: str, + passwords: list[str] | None = None, + ) -> list[ExtractedFile]: + with open(file_path, "rb") as fh: + content = fh.read() + hashes = compute_file_hashes(content) + filename = file_path.rsplit("/", 1)[-1].rsplit("\\", 1)[-1] + + ext_to_mime = { + ".csv": "text/csv", + ".txt": "text/plain", + ".eml": "message/rfc822", + ".msg": "application/vnd.ms-outlook", + ".json": "application/json", + ".xml": "application/xml", + ".html": "text/html", + } + ext = "." + filename.rsplit(".", 1)[-1].lower() if "." in filename else "" + content_type = ext_to_mime.get(ext, "application/octet-stream") + + return [ + ExtractedFile( + filename=filename, + content=content, + content_type=content_type, + hashes=hashes, + was_encrypted=False, + ) + ] diff --git a/external-import/email-cases-importer/src/attachment_handler/registry.py b/external-import/email-cases-importer/src/attachment_handler/registry.py new file mode 100644 index 00000000000..f1a46565a14 --- /dev/null +++ b/external-import/email-cases-importer/src/attachment_handler/registry.py @@ -0,0 +1,198 @@ +import os +import tempfile + +from attachment_handler.archive_handler import ArchiveHandler +from attachment_handler.base import BaseAttachmentHandler, ExtractedFile +from attachment_handler.document_handler import DocumentHandler +from attachment_handler.passthrough_handler import PassthroughHandler +from connector.utils import compute_file_hashes + + +class HandlerRegistry: + """Registry mapping file extensions to attachment handlers. + + Supports adding custom handlers at runtime for extensibility. + """ + + def __init__(self): + self._handlers: dict[str, BaseAttachmentHandler] = {} + self._register_defaults() + + def _register_defaults(self) -> None: + """Register built-in handlers.""" + for handler in [ArchiveHandler(), DocumentHandler(), PassthroughHandler()]: + self.register(handler) + + def register(self, handler: BaseAttachmentHandler) -> None: + """Register a handler for its supported extensions.""" + for ext in handler.supported_extensions(): + self._handlers[ext.lower()] = handler + + def get_handler(self, filename: str) -> BaseAttachmentHandler | None: + """Get the appropriate handler for a filename.""" + if "." not in filename: + return None + ext = "." + filename.rsplit(".", 1)[-1].lower() + return self._handlers.get(ext) + + def process_attachment( + self, + filename: str, + content: bytes, + passwords: list[str] | None = None, + max_size_mb: int = 25, + ) -> list[ExtractedFile]: + """Process an attachment through the appropriate handler. + + For archives, recursively processes inner files (trying passwords + on nested encrypted content). + """ + if len(content) > max_size_mb * 1024 * 1024: + # Return file metadata without processing + return [ + ExtractedFile( + filename=filename, + content=b"", # Don't store oversized content + content_type="application/octet-stream", + hashes=compute_file_hashes(content), + metadata={"skipped": "exceeds_max_size"}, + ) + ] + + handler = self.get_handler(filename) + if handler is None: + # No handler — return raw file with hashes + return [ + ExtractedFile( + filename=filename, + content=content, + content_type="application/octet-stream", + hashes=compute_file_hashes(content), + ) + ] + + # Write to temp file for handler processing + tmp_path = None + try: + suffix = "." + filename.rsplit(".", 1)[-1] if "." in filename else "" + with tempfile.NamedTemporaryFile(delete=False, suffix=suffix) as tmp: + tmp.write(content) + tmp_path = tmp.name + + extracted = handler.extract(tmp_path, passwords) + + # For non-archives, preserve the original filename + if not isinstance(handler, ArchiveHandler): + for ef in extracted: + ef.filename = filename + + # For archives: recursively process inner files + if isinstance(handler, ArchiveHandler): + all_results = [] + for ef in extracted: + inner_handler = self.get_handler(ef.filename) + if inner_handler: + if isinstance(inner_handler, ArchiveHandler): + # Nested archive (e.g., encrypted zip inside + # unprotected zip) — recurse with depth tracking + inner_results = self._process_nested_archive( + ef, passwords, max_size_mb, depth=1 + ) + ef.inner_files = inner_results + else: + # Document/passthrough inside archive + inner_results = self._process_inner_file( + ef, passwords, max_size_mb + ) + ef.inner_files = inner_results + all_results.append(ef) + return all_results + + return extracted + + finally: + if tmp_path and os.path.exists(tmp_path): + os.unlink(tmp_path) + + def _process_nested_archive( + self, + extracted_file: ExtractedFile, + passwords: list[str] | None, + max_size_mb: int, + depth: int, + ) -> list[ExtractedFile]: + """Recursively process a nested archive (archive inside archive).""" + if depth >= 3: # Max nesting depth + return [] + + tmp_path = None + try: + suffix = ( + "." + extracted_file.filename.rsplit(".", 1)[-1] + if "." in extracted_file.filename + else "" + ) + with tempfile.NamedTemporaryFile(delete=False, suffix=suffix) as tmp: + tmp.write(extracted_file.content) + tmp_path = tmp.name + + handler = self.get_handler(extracted_file.filename) + if handler is None: + return [] + + inner_extracted = handler.extract(tmp_path, passwords) + + # Process files inside this nested archive too + all_results = [] + for ef in inner_extracted: + inner_handler = self.get_handler(ef.filename) + if inner_handler: + if isinstance(inner_handler, ArchiveHandler): + ef.inner_files = self._process_nested_archive( + ef, passwords, max_size_mb, depth + 1 + ) + else: + ef.inner_files = self._process_inner_file( + ef, passwords, max_size_mb + ) + all_results.append(ef) + return all_results + except Exception: + return [] + finally: + if tmp_path and os.path.exists(tmp_path): + os.unlink(tmp_path) + + def _process_inner_file( + self, + extracted_file: ExtractedFile, + passwords: list[str] | None, + max_size_mb: int, + ) -> list[ExtractedFile]: + """Process an inner file extracted from an archive.""" + handler = self.get_handler(extracted_file.filename) + if handler is None: + return [] + + tmp_path = None + try: + suffix = ( + "." + extracted_file.filename.rsplit(".", 1)[-1] + if "." in extracted_file.filename + else "" + ) + with tempfile.NamedTemporaryFile(delete=False, suffix=suffix) as tmp: + tmp.write(extracted_file.content) + tmp_path = tmp.name + + results = handler.extract(tmp_path, passwords) + # Preserve original filename for non-archive handlers + if not isinstance(handler, ArchiveHandler): + for ef in results: + ef.filename = extracted_file.filename + return results + except Exception: + return [] + finally: + if tmp_path and os.path.exists(tmp_path): + os.unlink(tmp_path) diff --git a/external-import/email-cases-importer/src/connector/__init__.py b/external-import/email-cases-importer/src/connector/__init__.py new file mode 100644 index 00000000000..55dce0dccd6 --- /dev/null +++ b/external-import/email-cases-importer/src/connector/__init__.py @@ -0,0 +1,8 @@ +from connector.connector import EmailCasesConnector +from connector.settings import ConnectorSettings, EmailCasesConfig + +__all__ = [ + "ConnectorSettings", + "EmailCasesConfig", + "EmailCasesConnector", +] diff --git a/external-import/email-cases-importer/src/connector/connector.py b/external-import/email-cases-importer/src/connector/connector.py new file mode 100644 index 00000000000..62130ed8006 --- /dev/null +++ b/external-import/email-cases-importer/src/connector/connector.py @@ -0,0 +1,978 @@ +import re +import threading +import time +from datetime import datetime, timezone + +import pycti +from pycti import OpenCTIConnectorHelper + +from attachment_handler.registry import HandlerRegistry +from connector.converter_to_stix import ConverterToStix +from connector.utils import ( + collapse_blank_lines, + extract_passwords, + matches_subject_filter, + normalize_subject, + sanitize_html, +) +from email_client.base import EmailMessage +from email_client.factory import create_email_client + + +class EmailCasesConnector: + def __init__(self, config, helper: OpenCTIConnectorHelper): + self.config = config + self.helper = helper + self.converter = ConverterToStix(helper, config) + self.handler_registry = HandlerRegistry() + self._interval = self._resolve_poll_interval() + self._subject_filters = config.email_cases.get_parsed_subject_filters() + self._sender = config.email_cases.sender_address + self._password_prefix = config.email_cases.password_prefix + self._password_suffix = config.email_cases.password_suffix + self._password_strip_ws = config.email_cases.password_strip_whitespace + self._thread_strategy = config.email_cases.thread_tracking_strategy + self._max_att_size = config.email_cases.max_attachment_size_mb + self._max_emails = config.email_cases.max_emails_per_cycle + self._static_labels = config.email_cases.get_parsed_labels() + self._subject_rules = config.email_cases.get_parsed_subject_rules() + self._sender_rules = config.email_cases.get_parsed_sender_rules() + self._display_names = config.email_cases.display_sender_names + self._fetch_timeout = config.email_cases.email_fetch_timeout + self._label_cache: dict[str, str] = {} # label name -> OpenCTI ID + self._entity_cache: dict[str, str] = {} # cache for resolved entity IDs + # OpenCTI internal id of the connector's own author identity, created at + # startup (see _ensure_connector_author) and used as the default case + # author when no sender-rule author applies. + self._connector_author_id: str | None = None + + def _ensure_connector_author(self) -> None: + """Create/resolve the connector's own author Identity in OpenCTI and + cache its internal id. + + Used as the default ``createdBy`` for cases with no sender-rule author. + The converter's STIX id references an Identity that is never created in + the platform, so resolve a real internal id here instead. + """ + connector_cfg = getattr(self.config, "connector", None) + author_name = getattr(connector_cfg, "name", None) or "Email Cases Importer" + self._connector_author_id = self._resolve_identity_id(author_name) + + def _resolve_poll_interval(self) -> int: + """Seconds between poll cycles. + + ``CONNECTOR_DURATION_PERIOD`` (ISO-8601, e.g. ``PT5M``) is the single + source of truth for the polling cadence, consistent with the OpenCTI + connectors-sdk (it is a ``datetime.timedelta`` on the connector config). + ``EMAIL_CASES_IMPORT_INTERVAL`` is retained only as a deprecated + fallback for existing deployments and is ignored whenever a valid + duration period is configured. + """ + connector_cfg = getattr(self.config, "connector", None) + period = getattr(connector_cfg, "duration_period", None) + try: + if period is not None: + seconds = int(period.total_seconds()) + if seconds > 0: + return seconds + except (AttributeError, TypeError, ValueError): + pass + return int(self.config.email_cases.import_interval) + + def run(self): + # Startup: test email connection (with timeout) + try: + self._test_connection_with_timeout() + except Exception as exc: + self.helper.connector_logger.error( + "[CONNECTOR] Email connection failed — stopping", + {"error": str(exc)}, + ) + exit(1) + + # Startup: ensure vocabulary values exist + self._ensure_vocabularies() + + # Startup: ensure the connector's author identity exists in OpenCTI + self._ensure_connector_author() + + self.helper.connector_logger.info( + "[CONNECTOR] Starting email import loop", + { + "interval_seconds": self._interval, + "sender": self._sender, + "protocol": self.config.email_cases.protocol, + }, + ) + while True: + try: + self._import_emails() + except (KeyboardInterrupt, SystemExit): + self.helper.connector_logger.info("[CONNECTOR] Stopping") + break + except Exception as e: + self.helper.connector_logger.error( + "[CONNECTOR] Import cycle failed", {"error": str(e)} + ) + time.sleep(self._interval) + + def _import_emails(self): + state = self.helper.get_state() or {} + last_run = state.get("last_run") + thread_map = state.get("thread_map", {}) + processed_ids = set(state.get("processed_message_ids", [])) + + # Resolve `since`: prefer prior state; otherwise fall back to configured + # start_date (first-run backfill floor); otherwise None (server returns + # most recent emails). + since = None + since_source = "none" + if last_run: + since = datetime.fromisoformat(last_run.replace("Z", "+00:00")) + since_source = "last_run" + else: + configured_start = self.config.email_cases.get_parsed_start_date() + if configured_start: + since = configured_start + since_source = "start_date" + + self.helper.connector_logger.info( + "[CONNECTOR] Import cycle starting", + { + "since": since.isoformat() if since else None, + "since_source": since_source, + "sender_filter": self._sender, + "subject_filters_count": len(self._subject_filters), + "max_emails_per_cycle": self._max_emails, + }, + ) + + friendly_name = f"Email IR import @ {datetime.now(timezone.utc).strftime('%Y-%m-%dT%H:%M:%SZ')}" + work_id = self.helper.api.work.initiate_work( + self.helper.connect_id, friendly_name + ) + + try: + client = create_email_client(self.config) + with client: + emails = self._fetch_with_timeout(client, since) + + # Per-email debug trace: subjects + senders + dates. Only emitted + # when CONNECTOR_LOG_LEVEL=debug so production output stays quiet. + for idx, e in enumerate(emails, start=1): + self.helper.connector_logger.debug( + "[CONNECTOR] Fetched email", + { + "idx": idx, + "subject": e.subject, + "sender": e.sender, + "date": e.date.isoformat() if e.date else None, + "thread_id": e.thread_id, + "attachments": len(e.attachments), + }, + ) + + # Strict sender sanity check — protocol-level filters can be + # loose (IMAP SEARCH FROM is substring, Gmail q="from:" is fuzzy), + # so cross-check against the configured sender here. + sender_lower = (self._sender or "").lower().strip() + sender_exact = [ + e + for e in emails + if (e.sender or "").lower().strip() == sender_lower + ] + + # Subject filter stage — applied to the sender-verified list so + # that off-sender messages surfaced by a fuzzy protocol filter + # are dropped here instead of being processed. + subject_matched = [ + e + for e in sender_exact + if matches_subject_filter(e.subject, self._subject_filters) + ] + + # Dedup stage — drop anything we've already imported + new_matched = [ + e for e in subject_matched if e.message_id not in processed_ids + ] + already_processed = len(subject_matched) - len(new_matched) + + self.helper.connector_logger.info( + "[CONNECTOR] Fetch/filter breakdown", + { + "fetched": len(emails), + "exact_sender_match": len(sender_exact), + "other_senders": len(emails) - len(sender_exact), + "subject_matched": len(subject_matched), + "subject_skipped": len(sender_exact) - len(subject_matched), + "already_processed": already_processed, + "to_process": len(new_matched), + }, + ) + + # Debug subject list of what will actually be processed + for e in new_matched: + self.helper.connector_logger.debug( + "[CONNECTOR] Will process email", + { + "subject": e.subject, + "sender": e.sender, + "date": e.date.isoformat() if e.date else None, + "message_id": e.message_id, + }, + ) + + matched = new_matched + + if not matched: + self.helper.api.work.to_processed(work_id, "No new matching emails") + self.helper.set_state( + { + "last_run": datetime.now(timezone.utc).strftime( + "%Y-%m-%dT%H:%M:%SZ" + ), + "thread_map": thread_map, + "processed_message_ids": list(processed_ids), + } + ) + return + + cases_created = 0 + cases_updated = 0 + emails_processed = 0 + emails_failed = 0 + + for email_msg in matched: + try: + is_new = self._process_email(email_msg, thread_map, client) + processed_ids.add(email_msg.message_id) + emails_processed += 1 + if is_new: + cases_created += 1 + self.helper.connector_logger.debug( + "[CONNECTOR] Created new case", + { + "subject": email_msg.subject, + "message_id": email_msg.message_id, + }, + ) + else: + cases_updated += 1 + self.helper.connector_logger.debug( + "[CONNECTOR] Updated existing case", + { + "subject": email_msg.subject, + "message_id": email_msg.message_id, + }, + ) + except Exception as e: + emails_failed += 1 + self.helper.connector_logger.error( + "[CONNECTOR] Failed to process email", + {"subject": email_msg.subject, "error": str(e)}, + ) + + # Total-failure guard (audit #5): if there were emails to process + # but every one failed, do NOT report the Work as successful or + # advance the state watermark. Raising here routes to the handler + # below, which marks the Work in_error; set_state is skipped so the + # same emails are retried next cycle instead of being silently lost. + if emails_processed == 0 and emails_failed > 0: + raise RuntimeError( + f"All {emails_failed} matching email(s) failed to process; " + "no cases created this cycle" + ) + + msg = ( + f"Processed {emails_processed} emails " + f"({cases_created} new cases, {cases_updated} updates, " + f"{emails_failed} failed)" + ) + self.helper.connector_logger.info( + "[CONNECTOR] Import done", + { + "emails_processed": emails_processed, + "cases_created": cases_created, + "cases_updated": cases_updated, + "emails_failed": emails_failed, + "msg": msg, + }, + ) + self.helper.api.work.to_processed(work_id, msg) + self.helper.set_state( + { + "last_run": datetime.now(timezone.utc).strftime( + "%Y-%m-%dT%H:%M:%SZ" + ), + "thread_map": thread_map, + "processed_message_ids": list(processed_ids), + } + ) + + except Exception as e: + self.helper.connector_logger.error( + "[CONNECTOR] Import failed", {"error": str(e)} + ) + self.helper.api.work.to_processed(work_id, str(e), in_error=True) + raise + + def _process_email( + self, + email_msg: EmailMessage, + thread_map: dict, + client, + ) -> bool: + """Process a single email: create/update case, upload attachments. + + Returns True if a new case was created. + """ + thread_id = self._resolve_thread_id(email_msg, client) + normalized_subject = normalize_subject(email_msg.subject) + is_new_case = thread_id not in thread_map + + # Extract passwords from email body. Normalize blank-line runs so + # nested-
HTML emails and plain-text emails with padded signatures + # both render cleanly in the case content. + body_text = email_msg.body_plain or sanitize_html(email_msg.body_html) or "" + body_text = collapse_blank_lines(body_text) + passwords = extract_passwords( + body_text, + self._password_prefix, + self._password_suffix, + strip_whitespace=self._password_strip_ws, + ) + if passwords: + self.helper.connector_logger.info( + "[CONNECTOR] Passwords extracted from email body", + {"count": len(passwords), "subject": email_msg.subject}, + ) + + # Process attachments (decrypt/extract). Keep originals plus any + # extracted/decrypted content, de-duplicated by (filename, bytes): a + # passthrough handler can surface an inner file that is byte-identical to + # its parent (e.g. an unencrypted CSV inside a plain zip), which would + # otherwise be uploaded twice. + attachment_files = [] + attachment_names = [] + seen_attachments: set[tuple[str, bytes]] = set() + + def _add_attachment(fn: str, data: bytes, ctype: str | None) -> None: + key = (fn, data) + if key in seen_attachments: + return + seen_attachments.add(key) + attachment_files.append((fn, data, ctype)) + attachment_names.append(fn) + + for att in email_msg.attachments: + # Always include the original attachment as-is + _add_attachment(att.filename, att.content, att.content_type) + + # Try to extract/decrypt — add results alongside the original + try: + extracted = self.handler_registry.process_attachment( + filename=att.filename, + content=att.content, + passwords=passwords, + max_size_mb=self._max_att_size, + ) + for ef in extracted: + _add_attachment(ef.filename, ef.content, ef.content_type) + for inner in ef.inner_files: + _add_attachment( + inner.filename, inner.content, inner.content_type + ) + except Exception as e: + self.helper.connector_logger.warning( + "[CONNECTOR] Attachment processing failed", + {"filename": att.filename, "error": str(e)}, + ) + + # Clear display names if feature is disabled + if not self._display_names: + email_msg.sender_display = "" + email_msg.recipients_display = [] + + # Build content block for this email + is_reply = not is_new_case or bool(email_msg.in_reply_to) + content_block = self.converter.format_email_content_block( + email_msg=email_msg, + body_text=body_text, + attachment_names=attachment_names, + is_reply=is_reply, + passwords_found=len(passwords), + ) + + # Resolve labels and subject rules + all_labels = list(self._static_labels) + rule_result = self._match_subject_rules(email_msg.subject) + all_labels.extend(rule_result["labels"]) + # Deduplicate labels preserving order + seen = set() + unique_labels = [] + for lbl in all_labels: + if lbl not in seen: + seen.add(lbl) + unique_labels.append(lbl) + label_ids = self._resolve_label_ids(unique_labels) if unique_labels else [] + response_types = rule_result["response_types"] or None + case_template = rule_result["case_template"] + rule_severity = rule_result["severity"] + rule_priority = rule_result["priority"] + + # Resolve sender rules + sender_result = self._match_sender_rules(email_msg.sender) + author_id = None + if sender_result["author"]: + author_id = self._resolve_identity_id(sender_result["author"]) + marking_id = None + if sender_result["marking"]: + marking_id = self._resolve_marking_id(sender_result["marking"]) + assignee_ids = ( + self._resolve_member_ids(sender_result["assignees"]) + if sender_result["assignees"] + else None + ) + participant_ids = ( + self._resolve_member_ids(sender_result["participants"]) + if sender_result["participants"] + else None + ) + + if is_new_case: + # Check if a case with this name already exists (e.g. after state reset) + existing_id = self._find_existing_case(normalized_subject) + if existing_id: + case_id = existing_id + self._append_to_case(case_id, content_block) + thread_map[thread_id] = case_id + self.helper.connector_logger.info( + "[CONNECTOR] Found existing case, appending", + {"case_id": case_id, "subject": normalized_subject}, + ) + else: + case_id = self._create_case( + email_msg, + normalized_subject, + content_block, + label_ids=label_ids, + response_types=response_types, + severity=rule_severity, + priority=rule_priority, + author_id=author_id, + marking_id=marking_id, + assignee_ids=assignee_ids, + participant_ids=participant_ids, + ) + if case_template: + self._apply_case_template(case_id, case_template) + thread_map[thread_id] = case_id + self.helper.connector_logger.info( + "[CONNECTOR] Created new case", + {"case_id": case_id, "subject": normalized_subject}, + ) + else: + case_id = thread_map[thread_id] + self._append_to_case(case_id, content_block) + self.helper.connector_logger.info( + "[CONNECTOR] Updated existing case", + {"case_id": case_id, "subject": email_msg.subject}, + ) + + # Upload attachments to the case + for filename, content, content_type in attachment_files: + try: + self.helper.api.stix_domain_object.add_file( + id=case_id, + file_name=filename, + data=content, + mime_type=content_type or "application/octet-stream", + no_trigger_import=True, + ) + self.helper.connector_logger.info( + "[CONNECTOR] Uploaded attachment", + {"filename": filename, "case_id": case_id}, + ) + except Exception as e: + self.helper.connector_logger.warning( + "[CONNECTOR] Failed to upload attachment", + {"filename": filename, "error": str(e)}, + ) + + return is_new_case + + def _resolve_label_ids(self, label_names: list[str]) -> list[str]: + """Resolve label names to OpenCTI IDs, creating labels if needed.""" + ids = [] + for name in label_names: + if name in self._label_cache: + ids.append(self._label_cache[name]) + continue + try: + result = self.helper.api.label.create(value=name) + label_id = result["id"] + self._label_cache[name] = label_id + ids.append(label_id) + except Exception as e: + self.helper.connector_logger.warning( + "[CONNECTOR] Failed to create/find label", + {"label": name, "error": str(e)}, + ) + return ids + + def _resolve_identity_id(self, name: str) -> str | None: + """Resolve an organization identity by name, creating if needed.""" + cache_key = f"identity:{name}" + if cache_key in self._entity_cache: + return self._entity_cache[cache_key] + try: + result = self.helper.api.identity.create( + type="Organization", + name=name, + ) + identity_id = result["id"] + self._entity_cache[cache_key] = identity_id + return identity_id + except Exception as e: + self.helper.connector_logger.warning( + "[CONNECTOR] Failed to resolve identity", + {"name": name, "error": str(e)}, + ) + return None + + def _resolve_marking_id(self, marking_name: str) -> str | None: + """Resolve a marking definition by name (e.g. 'TLP:AMBER').""" + cache_key = f"marking:{marking_name}" + if cache_key in self._entity_cache: + return self._entity_cache[cache_key] + try: + result = self.helper.api.marking_definition.read( + filters={ + "mode": "and", + "filters": [{"key": "definition", "values": [marking_name]}], + "filterGroups": [], + } + ) + if result: + self._entity_cache[cache_key] = result["id"] + return result["id"] + self.helper.connector_logger.warning( + "[CONNECTOR] Marking definition not found", + {"name": marking_name}, + ) + except Exception as e: + self.helper.connector_logger.warning( + "[CONNECTOR] Failed to resolve marking", + {"name": marking_name, "error": str(e)}, + ) + return None + + def _resolve_member_ids(self, emails: list[str]) -> list[str]: + """Resolve user emails to OpenCTI user IDs.""" + ids = [] + for email in emails: + cache_key = f"member:{email}" + if cache_key in self._entity_cache: + ids.append(self._entity_cache[cache_key]) + continue + try: + result = self.helper.api.query( + """ + query Users($search: String) { + users(search: $search) { + edges { node { id name user_email } } + } + } + """, + {"search": email}, + ) + edges = result.get("data", {}).get("users", {}).get("edges", []) + for edge in edges: + node = edge["node"] + if node.get("user_email", "").lower() == email.lower(): + self._entity_cache[cache_key] = node["id"] + ids.append(node["id"]) + break + else: + self.helper.connector_logger.warning( + "[CONNECTOR] User not found for assignee/participant", + {"email": email}, + ) + except Exception as e: + self.helper.connector_logger.warning( + "[CONNECTOR] Failed to resolve user", + {"email": email, "error": str(e)}, + ) + return ids + + def _match_subject_rules(self, subject: str) -> dict: + """Match subject against configured rules. Returns merged result.""" + result = { + "labels": [], + "response_types": [], + "case_template": None, + "severity": None, + "priority": None, + } + for rule in self._subject_rules: + match_type = rule["match_type"] + value = rule["value"] + matched = False + if match_type == "exact" and subject == value: + matched = True + elif match_type == "contains" and value.lower() in subject.lower(): + matched = True + elif match_type == "starts_with" and subject.lower().startswith( + value.lower() + ): + matched = True + elif match_type == "regex": + try: + if re.search(value, subject): + matched = True + except re.error: + pass + if matched: + result["labels"].extend(rule.get("labels", [])) + result["response_types"].extend(rule.get("response_types", [])) + if rule.get("case_template") and not result["case_template"]: + result["case_template"] = rule["case_template"] + if rule.get("severity") and not result["severity"]: + result["severity"] = rule["severity"] + if rule.get("priority") and not result["priority"]: + result["priority"] = rule["priority"] + return result + + def _match_sender_rules(self, sender: str) -> dict: + """Match sender against configured sender rules. Returns merged result.""" + result = { + "author": None, + "marking": None, + "assignees": [], + "participants": [], + } + for rule in self._sender_rules: + if rule["sender"].lower() == sender.lower(): + if rule.get("author") and not result["author"]: + result["author"] = rule["author"] + if rule.get("marking") and not result["marking"]: + result["marking"] = rule["marking"] + result["assignees"].extend(rule.get("assignees", [])) + result["participants"].extend(rule.get("participants", [])) + return result + + def _find_case_template_id(self, template_name: str) -> str | None: + """Look up a case template by name. Returns ID or None.""" + query = """ + query CaseTemplates($search: String) { + caseTemplates(search: $search) { + edges { + node { + id + name + } + } + } + } + """ + try: + result = self.helper.api.query(query, {"search": template_name}) + edges = result.get("data", {}).get("caseTemplates", {}).get("edges", []) + for edge in edges: + if edge["node"]["name"] == template_name: + return edge["node"]["id"] + except Exception as e: + self.helper.connector_logger.warning( + "[CONNECTOR] Failed to find case template", + {"name": template_name, "error": str(e)}, + ) + return None + + def _apply_case_template(self, case_id: str, template_name: str): + """Apply a case template to a case via GraphQL mutation.""" + template_id = self._find_case_template_id(template_name) + if not template_id: + self.helper.connector_logger.warning( + "[CONNECTOR] Case template not found", + {"name": template_name}, + ) + return + mutation = """ + mutation ApplyTemplate($id: ID!, $caseTemplatesId: [ID]!) { + caseSetTemplate(id: $id, caseTemplatesId: $caseTemplatesId) { + id + } + } + """ + try: + self.helper.api.query( + mutation, + { + "id": case_id, + "caseTemplatesId": [template_id], + }, + ) + self.helper.connector_logger.info( + "[CONNECTOR] Applied case template", + {"template": template_name, "case_id": case_id}, + ) + except Exception as e: + self.helper.connector_logger.warning( + "[CONNECTOR] Failed to apply case template", + {"template": template_name, "error": str(e)}, + ) + + def _create_case( + self, + email_msg: EmailMessage, + normalized_subject: str, + content: str, + label_ids: list[str] | None = None, + response_types: list[str] | None = None, + severity: str | None = None, + priority: str | None = None, + author_id: str | None = None, + marking_id: str | None = None, + assignee_ids: list[str] | None = None, + participant_ids: list[str] | None = None, + ) -> str: + """Create a new Case-Incident via the OpenCTI API. Returns internal ID.""" + prefix = self.config.email_cases.case_prefix + name = f"{prefix}{normalized_subject}" if prefix else normalized_subject + description = self.converter.format_case_description(email_msg) + + # Deterministic STIX id via the pycti generator, keyed on the case name + # and the first email's received time. This makes re-runs upsert the same + # Case-Incident instead of creating duplicates. The `created` value passed + # to create() MUST match the generate_id seed, so derive both from one + # string. + created = email_msg.date or datetime.now(timezone.utc) + created_iso = created.strftime("%Y-%m-%dT%H:%M:%SZ") + case_stix_id = pycti.CaseIncident.generate_id(name=name, created=created_iso) + + kwargs = dict( + stix_id=case_stix_id, + name=name, + created=created_iso, + description=description, + content=content, + severity=severity or self.config.email_cases.default_severity, + priority=priority or self.config.email_cases.default_priority, + createdBy=author_id + or self._connector_author_id + or self.converter.identity_id, + ) + if label_ids: + kwargs["objectLabel"] = label_ids + if response_types: + kwargs["response_types"] = response_types + if marking_id: + kwargs["objectMarking"] = [marking_id] + if assignee_ids: + kwargs["objectAssignee"] = assignee_ids + if participant_ids: + kwargs["objectParticipant"] = participant_ids + + result = self.helper.api.case_incident.create(**kwargs) + + return result["id"] + + def _append_to_case(self, case_id: str, new_content_block: str): + """Append a new email content block to an existing case's Content tab.""" + query = """ + query CaseIncidentContent($id: String!) { + caseIncident(id: $id) { + content + } + } + """ + result = self.helper.api.query(query, {"id": case_id}) + existing_content = "" + if result and result.get("data", {}).get("caseIncident"): + existing_content = result["data"]["caseIncident"].get("content") or "" + + updated_content = existing_content + "\n" + new_content_block + + self.helper.api.stix_domain_object.update_field( + id=case_id, + input={"key": "content", "value": [updated_content]}, + ) + + def _find_existing_case(self, normalized_subject: str) -> str | None: + """Look up an existing Case-Incident by name. Returns internal ID or None.""" + prefix = self.config.email_cases.case_prefix + name = f"{prefix}{normalized_subject}" if prefix else normalized_subject + try: + result = self.helper.api.case_incident.list( + filters={ + "mode": "and", + "filters": [{"key": "name", "values": [name]}], + "filterGroups": [], + }, + first=1, + ) + if result and len(result) > 0: + return result[0]["id"] + except Exception as e: + self.helper.connector_logger.warning( + "[CONNECTOR] Failed to look up existing case", + {"name": name, "error": str(e)}, + ) + return None + + def _resolve_thread_id(self, email_msg: EmailMessage, client) -> str: + """Resolve thread ID based on configured strategy.""" + if self._thread_strategy == "provider_thread_id": + tid = client.get_thread_id(email_msg) + if tid: + return tid + return f"subject:{normalize_subject(email_msg.subject)}" + + if self._thread_strategy == "message_headers": + if email_msg.in_reply_to: + return email_msg.in_reply_to + if email_msg.references: + return email_msg.references[0] + return email_msg.message_id + + if self._thread_strategy == "subject_matching": + return f"subject:{normalize_subject(email_msg.subject)}" + + # Default fallback + return client.get_thread_id(email_msg) or email_msg.message_id + + def _ensure_vocabularies(self): + """Ensure all configured vocabulary values exist in OpenCTI. + + Creates missing severity, priority, and response_types entries + so they are not silently dropped on case creation. + """ + all_severities = [self.config.email_cases.default_severity] + all_priorities = [self.config.email_cases.default_priority] + all_response_types = [] + for rule in self._subject_rules: + all_response_types.extend(rule.get("response_types", [])) + if rule.get("severity"): + all_severities.append(rule["severity"]) + if rule.get("priority"): + all_priorities.append(rule["priority"]) + + checks = [ + ("case_severity_ov", all_severities), + ("case_priority_ov", all_priorities), + ] + if all_response_types: + checks.append(("incident_response_types_ov", all_response_types)) + + for category, values in checks: + existing = self._get_vocabulary_values(category) + for value in values: + if value not in existing: + self._create_vocabulary_entry(category, value) + + def _get_vocabulary_values(self, category: str) -> set[str]: + """Fetch existing vocabulary values for a category.""" + query = """ + query Vocabularies($category: VocabularyCategory!) { + vocabularies(category: $category) { + edges { node { name } } + } + } + """ + try: + result = self.helper.api.query(query, {"category": category}) + edges = result.get("data", {}).get("vocabularies", {}).get("edges", []) + return {e["node"]["name"] for e in edges} + except Exception as e: + self.helper.connector_logger.warning( + "[CONNECTOR] Failed to fetch vocabularies", + {"category": category, "error": str(e)}, + ) + return set() + + def _create_vocabulary_entry(self, category: str, value: str): + """Create a vocabulary entry if it doesn't exist.""" + mutation = """ + mutation VocabularyAdd($input: VocabularyAddInput!) { + vocabularyAdd(input: $input) { id name } + } + """ + try: + self.helper.api.query( + mutation, + { + "input": {"name": value, "category": category}, + }, + ) + self.helper.connector_logger.info( + "[CONNECTOR] Created vocabulary entry", + {"category": category, "value": value}, + ) + except Exception as e: + self.helper.connector_logger.warning( + "[CONNECTOR] Failed to create vocabulary entry", + {"category": category, "value": value, "error": str(e)}, + ) + + def _test_connection_with_timeout(self): + """Test email connection with a timeout.""" + error_holder: list[Exception] = [] + + def _do_connect(): + try: + c = create_email_client(self.config) + with c: + pass + except Exception as exc: + error_holder.append(exc) + + thread = threading.Thread(target=_do_connect, daemon=True) + thread.start() + thread.join(timeout=self._fetch_timeout) + + if thread.is_alive(): + raise TimeoutError( + f"Email connection test did not complete within {self._fetch_timeout}s" + ) + if error_holder: + raise error_holder[0] + + self.helper.connector_logger.info( + "[CONNECTOR] Email connection verified", + {"protocol": self.config.email_cases.protocol}, + ) + + def _fetch_with_timeout(self, client, since) -> list[EmailMessage]: + """Fetch emails with a timeout to prevent the connector from hanging.""" + result_holder: list[EmailMessage] = [] + error_holder: list[Exception] = [] + + def _do_fetch(): + try: + result_holder.extend( + client.fetch_emails( + sender=self._sender, + since=since, + max_results=self._max_emails, + ) + ) + except Exception as exc: + error_holder.append(exc) + + thread = threading.Thread(target=_do_fetch, daemon=True) + thread.start() + thread.join(timeout=self._fetch_timeout) + + if thread.is_alive(): + self.helper.connector_logger.error( + "[CONNECTOR] Email fetch timed out", + {"timeout_seconds": self._fetch_timeout}, + ) + raise TimeoutError( + f"Email fetch did not complete within {self._fetch_timeout}s" + ) + + if error_holder: + raise error_holder[0] + + return result_holder diff --git a/external-import/email-cases-importer/src/connector/converter_to_stix.py b/external-import/email-cases-importer/src/connector/converter_to_stix.py new file mode 100644 index 00000000000..b07c2ccab43 --- /dev/null +++ b/external-import/email-cases-importer/src/connector/converter_to_stix.py @@ -0,0 +1,107 @@ +import html + +import pycti +import stix2 + +from email_client.base import EmailMessage + + +def _html_ascii(text: str) -> str: + """HTML-escape ``text`` and encode every non-ASCII character as a numeric + character reference, yielding a pure-ASCII payload. + + The OpenCTI case ``content`` field double-encodes raw non-ASCII bytes on + write (a literal 'é' round-trips as 'é'). Emitting 'é' instead keeps + the wire payload ASCII while rendering identically in the Content tab, so + unicode email subjects/bodies (e.g. Arabic, CJK, accented Latin) display + correctly. + """ + return html.escape(text).encode("ascii", "xmlcharrefreplace").decode("ascii") + + +class ConverterToStix: + """Creates the connector identity and formats case content.""" + + def __init__(self, helper, config): + self._helper = helper + self._config = config + self._identity_id = self._create_connector_identity() + + def _create_connector_identity(self) -> str: + identity = stix2.Identity( + id=pycti.Identity.generate_id( + name="Email Cases Importer", identity_class="system" + ), + name="Email Cases Importer", + identity_class="system", + allow_custom=True, + custom_properties={"x_opencti_reliability": "A - Completely reliable"}, + ) + self._connector_identity = identity + return identity.id + + def format_case_description(self, email_msg: EmailMessage) -> str: + """Build a plain-text description for the IR case.""" + date_str = email_msg.date.strftime("%Y-%m-%d %H:%M:%S UTC") + return ( + f"Incident Response Case\n\n" + f"Sender: {email_msg.sender_display or email_msg.sender}\n" + f"Subject: {email_msg.subject}\n" + f"First received: {date_str}" + ) + + def format_email_content_block( + self, + email_msg: EmailMessage, + body_text: str, + attachment_names: list[str], + is_reply: bool = False, + passwords_found: int = 0, + ) -> str: + """Format a single email as an HTML block for the case Content tab.""" + date_str = email_msg.date.strftime("%Y-%m-%d %H:%M:%S UTC") + sender = _html_ascii(email_msg.sender_display or email_msg.sender) + recipients = _html_ascii( + ", ".join(email_msg.recipients_display or email_msg.recipients) + if (email_msg.recipients_display or email_msg.recipients) + else "(none)" + ) + + label = "Reply" if is_reply else "Original" + subject = _html_ascii(email_msg.subject) + + parts = [ + "
", + f"

{subject}

", + f"

" + f"Date: {date_str} — {label}
" + f"From: {sender}
" + f"To: {recipients}" + f"

", + ] + + if body_text: + escaped_body = _html_ascii(body_text.strip()).replace("\n", "
") + parts.append(f"
{escaped_body}
") + + if passwords_found > 0: + parts.append( + f"
{passwords_found} password(s) extracted " + f"from email body and used to decrypt attachments.
" + ) + + if attachment_names: + files_html = ", ".join( + f"{_html_ascii(n)}" for n in attachment_names + ) + parts.append(f"

Attachments: {files_html}

") + + return "\n".join(parts) + + @property + def identity_id(self) -> str: + return self._identity_id + + @property + def connector_identity(self): + return self._connector_identity diff --git a/external-import/email-cases-importer/src/connector/settings.py b/external-import/email-cases-importer/src/connector/settings.py new file mode 100644 index 00000000000..08b3fe86f67 --- /dev/null +++ b/external-import/email-cases-importer/src/connector/settings.py @@ -0,0 +1,287 @@ +import json +from datetime import datetime, timezone +from typing import Literal + +from connectors_sdk import ( + BaseConfigModel, + BaseConnectorSettings, + BaseExternalImportConnectorConfig, +) +from pydantic import Field, field_validator + + +class ExternalImportConnectorConfig(BaseExternalImportConnectorConfig): + name: str = Field(default="Email Cases Importer") + + +class SubjectFilter: + def __init__(self, filter_type: str, value: str): + self.filter_type = filter_type + self.value = value + + +class EmailCasesConfig(BaseConfigModel): + # Protocol selection + protocol: Literal["imap", "microsoft_graph", "gmail", "ews"] = Field( + default="imap", + description="Email protocol to use.", + ) + + # IMAP settings + imap_host: str = Field(default="", description="IMAP server hostname.") + imap_port: int = Field(default=993, description="IMAP server port.") + imap_username: str = Field(default="", description="IMAP username.") + imap_password: str = Field(default="", description="IMAP password.") + imap_folder: str = Field(default="INBOX", description="IMAP folder to monitor.") + imap_use_ssl: bool = Field(default=True, description="Use SSL/TLS for IMAP.") + + # Microsoft Graph settings + graph_tenant_id: str = Field(default="", description="Azure AD tenant ID.") + graph_client_id: str = Field(default="", description="Azure AD client ID.") + graph_client_secret: str = Field(default="", description="Azure AD client secret.") + graph_user_id: str = Field(default="", description="Mailbox user ID or UPN.") + + # Gmail settings + gmail_credentials_file: str = Field( + default="", description="Path to Google service account credentials JSON." + ) + gmail_user_id: str = Field( + default="me", description="Gmail user ID (default: 'me')." + ) + + # EWS settings + ews_server: str = Field(default="", description="Exchange server URL.") + ews_username: str = Field(default="", description="Exchange username.") + ews_password: str = Field(default="", description="Exchange password.") + ews_auth_type: Literal["NTLM", "OAuth2"] = Field( + default="NTLM", description="EWS auth type." + ) + + # Email filtering + sender_address: str = Field(description="Sender email address to monitor.") + subject_filters: str = Field( + description=( + 'JSON array of subject filters. An empty array ("[]") means "accept ' + 'any subject" (no subject-level filtering). ' + 'Example: [{"type":"exact","value":"Security Alert"},' + '{"type":"regex","value":"INC-\\\\d+"}]' + ), + ) + + # Thread tracking + thread_tracking_strategy: Literal[ + "provider_thread_id", "message_headers", "subject_matching" + ] = Field( + default="provider_thread_id", + description="Thread tracking strategy.", + ) + + # First-run starting date + start_date: str = Field( + default="", + description=( + "Optional ISO 8601 starting date used ONLY on the first fetch cycle " + "(when no prior state exists). Later cycles resume from the last run " + "timestamp stored in state. Formats accepted: 'YYYY-MM-DD' or " + "'YYYY-MM-DDTHH:MM:SSZ'. Example: '2026-04-01'. Leave empty to let " + "the email server return the most recent N emails on first run." + ), + ) + + @field_validator("start_date") + @classmethod + def validate_start_date(cls, v: str) -> str: + if not v: + return v + try: + if len(v) == 10: + datetime.strptime(v, "%Y-%m-%d") + else: + datetime.fromisoformat(v.replace("Z", "+00:00")) + except ValueError as exc: + raise ValueError( + f"start_date must be ISO-8601 or YYYY-MM-DD: {exc}" + ) from exc + return v + + def get_parsed_start_date(self) -> datetime | None: + if not self.start_date: + return None + if len(self.start_date) == 10: + return datetime.strptime(self.start_date, "%Y-%m-%d").replace( + tzinfo=timezone.utc + ) + return datetime.fromisoformat(self.start_date.replace("Z", "+00:00")) + + # Password extraction + password_prefix: str = Field( + default="---BEGIN PASSWORD---", + description="Prefix marker for password in email body.", + ) + password_suffix: str = Field( + default="---END PASSWORD---", + description="Suffix marker for password in email body.", + ) + password_strip_whitespace: bool = Field( + default=False, + description=( + "Strip all spaces, tabs, and newlines from extracted passwords. " + "Useful when HTML rendering or email line wrapping inserts whitespace " + "within the password between the prefix and suffix markers." + ), + ) + + # Display + display_sender_names: bool = Field( + default=True, + description=( + "Show sender and recipient display names in case content " + "(e.g. 'NCSC UK ' instead of just 'contact@ncsc.gov.uk')." + ), + ) + + # Timeouts + email_fetch_timeout: int = Field( + default=120, + description=( + "Timeout in seconds for a single email fetch cycle. " + "Prevents the connector from hanging on unresponsive mail servers." + ), + ) + + # Case defaults + default_severity: str = Field( + default="medium", description="Default severity for created cases." + ) + default_priority: str = Field( + default="P3", description="Default priority for created cases." + ) + case_prefix: str = Field(default="", description="Optional prefix for case names.") + + # Labels and subject rules + labels: str = Field( + default="", + description="Comma-separated labels always added to cases (e.g. 'NCSC UK,Email Alert').", + ) + subject_rules: str = Field( + default="[]", + description=( + "JSON array of subject-based rules. Each rule can add labels, set " + "response_types, severity, priority, and apply a case_template. Example: " + '[{"match_type":"contains","value":"Threat Alert",' + '"labels":["Threat Alert"],"response_types":["ransomware"],' + '"severity":"critical","priority":"P1",' + '"case_template":"My Template"}]' + ), + ) + sender_rules: str = Field( + default="[]", + description=( + "JSON array of sender-based rules. Each rule matches on sender email " + "and can set author, marking, assignees, and participants. Example: " + '[{"sender":"alerts@ncsc.gov.uk","author":"NCSC UK",' + '"marking":"TLP:AMBER","assignees":["analyst@company.com"],' + '"participants":["soc-team@company.com"]}]' + ), + ) + + @field_validator("subject_rules") + @classmethod + def validate_subject_rules(cls, v: str) -> str: + try: + rules = json.loads(v) + if not isinstance(rules, list): + raise ValueError("subject_rules must be a JSON array") + for r in rules: + if not isinstance(r, dict): + raise ValueError("Each rule must be a JSON object") + if "match_type" not in r or "value" not in r: + raise ValueError("Each rule must have 'match_type' and 'value'") + if r["match_type"] not in ("exact", "contains", "starts_with", "regex"): + raise ValueError( + f"match_type must be exact, contains, starts_with, or regex, got: {r['match_type']}" + ) + except json.JSONDecodeError as exc: + raise ValueError(f"subject_rules must be valid JSON: {exc}") from exc + return v + + def get_parsed_labels(self) -> list[str]: + if not self.labels: + return [] + return [label.strip() for label in self.labels.split(",") if label.strip()] + + def get_parsed_subject_rules(self) -> list[dict]: + return json.loads(self.subject_rules) + + @field_validator("sender_rules") + @classmethod + def validate_sender_rules(cls, v: str) -> str: + try: + rules = json.loads(v) + if not isinstance(rules, list): + raise ValueError("sender_rules must be a JSON array") + for r in rules: + if not isinstance(r, dict): + raise ValueError("Each rule must be a JSON object") + if "sender" not in r: + raise ValueError("Each sender rule must have 'sender'") + except json.JSONDecodeError as exc: + raise ValueError(f"sender_rules must be valid JSON: {exc}") from exc + return v + + def get_parsed_sender_rules(self) -> list[dict]: + return json.loads(self.sender_rules) + + # Import settings + import_interval: int = Field( + default=300, + description=( + "DEPRECATED — use CONNECTOR_DURATION_PERIOD (ISO-8601, e.g. PT5M), " + "the standard connectors-sdk polling field. Seconds between email " + "polling cycles; retained only as a fallback for existing " + "deployments and ignored when a valid CONNECTOR_DURATION_PERIOD is " + "set." + ), + ) + max_emails_per_cycle: int = Field( + default=50, description="Maximum emails to process per import cycle." + ) + tls_verify: bool = Field(default=True, description="Verify TLS certificates.") + + # Attachment handling + max_attachment_size_mb: int = Field( + default=25, description="Maximum attachment size in MB." + ) + attachment_store_in_opencti: bool = Field( + default=True, description="Upload attachments as Artifacts to OpenCTI." + ) + + @field_validator("subject_filters") + @classmethod + def validate_subject_filters(cls, v: str) -> str: + try: + filters = json.loads(v) + if not isinstance(filters, list): + raise ValueError("subject_filters must be a JSON array") + for f in filters: + if not isinstance(f, dict): + raise ValueError("Each filter must be a JSON object") + if "type" not in f or "value" not in f: + raise ValueError("Each filter must have 'type' and 'value'") + if f["type"] not in ("exact", "contains", "regex"): + raise ValueError( + f"Filter type must be exact, contains, or regex, got: {f['type']}" + ) + except json.JSONDecodeError as exc: + raise ValueError(f"subject_filters must be valid JSON: {exc}") from exc + return v + + def get_parsed_subject_filters(self) -> list[dict]: + return json.loads(self.subject_filters) + + +class ConnectorSettings(BaseConnectorSettings): + connector: ExternalImportConnectorConfig = Field( + default_factory=ExternalImportConnectorConfig + ) + email_cases: EmailCasesConfig = Field(default_factory=EmailCasesConfig) diff --git a/external-import/email-cases-importer/src/connector/utils.py b/external-import/email-cases-importer/src/connector/utils.py new file mode 100644 index 00000000000..abb7cc32c71 --- /dev/null +++ b/external-import/email-cases-importer/src/connector/utils.py @@ -0,0 +1,124 @@ +import hashlib +import html +import re + + +def extract_passwords( + body: str, + prefix: str, + suffix: str, + strip_whitespace: bool = False, +) -> list[str]: + """Extract passwords from email body using configurable prefix/suffix markers. + + If strip_whitespace is True, all spaces, tabs, and newlines are removed from + the extracted password (useful when HTML rendering or email wrapping inserts + whitespace within the password). + """ + if not prefix or not suffix: + return [] + # Decode HTML entities in the body first + decoded_body = html.unescape(body) + # Escape prefix/suffix for regex safety + pattern = re.escape(prefix) + r"(.+?)" + re.escape(suffix) + matches = re.findall(pattern, decoded_body, re.DOTALL) + results = [] + for m in matches: + pwd = m.strip() + if strip_whitespace: + pwd = re.sub(r"\s+", "", pwd) + if pwd: + results.append(pwd) + return results + + +def normalize_subject(subject: str) -> str: + """Strip RE:/FW:/FWD: prefixes and normalize whitespace for thread matching. + + Handles common localized prefixes: + EN: RE, FW, FWD DE: AW, WG FR: TR ES: RV NL: Doorst + IT: I, R PT: ENC, RES NO/SV: SV, VS + """ + if not subject: + return "" + # Covers English + common localized reply/forward prefixes + pattern = r"^(RE|FW|FWD|AW|WG|TR|RV|I|R|ENC|RES|SV|VS|Doorst)\s*:\s*" + cleaned = re.sub(pattern, "", subject.strip(), flags=re.IGNORECASE) + # Recursively strip in case of multiple prefixes (RE: FW: RE: ...) + while cleaned != subject: + subject = cleaned + cleaned = re.sub(pattern, "", subject.strip(), flags=re.IGNORECASE) + return cleaned.strip() + + +def collapse_blank_lines(text: str) -> str: + """Collapse runs of 3+ consecutive newlines down to 2 (one blank line). + + Outlook and gateway-rewritten HTML emails often translate to plain text + with dozens of consecutive blank lines (nested
/

/
). This keeps + paragraph breaks (a single blank line) but removes the visual explosion. + """ + if not text: + return text + # Normalize line endings first so \r\n and \r don't defeat the regex + text = text.replace("\r\n", "\n").replace("\r", "\n") + # Treat whitespace-only lines as blank + text = re.sub(r"[ \t]+\n", "\n", text) + # 3+ newlines -> exactly 2 (one blank line between paragraphs) + return re.sub(r"\n{3,}", "\n\n", text).strip() + + +def sanitize_html(html_content: str) -> str: + """Convert HTML email body to plain text, preserving structure.""" + if not html_content: + return "" + # Remove style and script tags and their content + text = re.sub(r"]*>.*?", "", html_content, flags=re.DOTALL) + text = re.sub(r"]*>.*?", "", text, flags=re.DOTALL) + # Convert br and p tags to newlines + text = re.sub(r"", "\n", text, flags=re.IGNORECASE) + text = re.sub(r"

", "\n\n", text, flags=re.IGNORECASE) + text = re.sub(r"
", "\n", text, flags=re.IGNORECASE) + # Remove remaining HTML tags + text = re.sub(r"<[^>]+>", "", text) + # Decode HTML entities + text = html.unescape(text) + # Normalize whitespace (preserve newlines) + lines = text.split("\n") + lines = [re.sub(r"[ \t]+", " ", line).strip() for line in lines] + # Collapse blank-line explosions from nested
/
/

+ return collapse_blank_lines("\n".join(lines)) + + +def compute_file_hashes(data: bytes) -> dict[str, str]: + """Compute MD5, SHA-1, and SHA-256 hashes for file data.""" + return { + "MD5": hashlib.md5(data).hexdigest(), # noqa: S324 + "SHA-1": hashlib.sha1(data).hexdigest(), # noqa: S324 + "SHA-256": hashlib.sha256(data).hexdigest(), + } + + +def matches_subject_filter(subject: str, filters: list[dict]) -> bool: + """Check if a subject matches any of the configured filters. + + An empty filter list means "accept any subject". This is the intuitive + reading of "no filter configured"; the alternative (empty = match nothing) + silently drops every email and is a config footgun. + """ + if not filters: + return True + for f in filters: + filter_type = f.get("type", "") + value = f.get("value", "") + if filter_type == "exact" and subject == value: + return True + if filter_type == "contains" and value in subject: + return True + if filter_type == "regex": + try: + if re.search(value, subject): + return True + except re.error: + continue + return False diff --git a/external-import/email-cases-importer/src/email_client/__init__.py b/external-import/email-cases-importer/src/email_client/__init__.py new file mode 100644 index 00000000000..40e2b0900b8 --- /dev/null +++ b/external-import/email-cases-importer/src/email_client/__init__.py @@ -0,0 +1,9 @@ +from email_client.base import BaseEmailClient, EmailAttachment, EmailMessage +from email_client.factory import create_email_client + +__all__ = [ + "BaseEmailClient", + "EmailMessage", + "EmailAttachment", + "create_email_client", +] diff --git a/external-import/email-cases-importer/src/email_client/_http.py b/external-import/email-cases-importer/src/email_client/_http.py new file mode 100644 index 00000000000..32ef794f67c --- /dev/null +++ b/external-import/email-cases-importer/src/email_client/_http.py @@ -0,0 +1,67 @@ +"""Shared HTTP robustness helpers for the REST-based email clients. + +Centralizes two behaviors the OpenCTI maintainers require (see +OpenCTI-Platform/connectors#6164): + +* `parse_json` — tolerate a non-JSON 200 body (e.g. an HTML error page from a + proxy/WAF) by raising a clear, typed error with a short body preview, instead + of surfacing an opaque ``ValueError`` deep inside response parsing. +* `get_with_retry` — honor HTTP 429 ``Retry-After`` with bounded backoff so a + transient rate-limit doesn't fail the entire fetch cycle. + +Used by the Microsoft Graph and Gmail clients (both built on requests). +""" + +import time + +_DEFAULT_RETRY_WAIT = 2 +_MAX_RETRY_WAIT = 60 + + +def _safe_int(value, default): + """Best-effort int parse with a fallback (mirrors the #6164 helper).""" + try: + return int(value) + except (TypeError, ValueError): + return default + + +class EmailClientHTTPError(RuntimeError): + """Raised when an email REST API returns an unusable (non-JSON) response.""" + + +def parse_json(resp): + """Return ``resp.json()``, or raise EmailClientHTTPError on a non-JSON body. + + ``requests``' ``JSONDecodeError`` subclasses ``ValueError``, so a single + except covers both stdlib and requests JSON failures. + """ + try: + return resp.json() + except ValueError as exc: + content_type = resp.headers.get("Content-Type", "unknown") + preview = (resp.text or "")[:200] + raise EmailClientHTTPError( + f"Expected a JSON response but got Content-Type={content_type!r} " + f"(HTTP {resp.status_code}): {preview!r}" + ) from exc + + +def get_with_retry(session, url, *, max_retries=3, **kwargs): + """``session.get`` that retries on HTTP 429, honoring ``Retry-After``. + + ``Retry-After`` is read as an integer number of seconds (the form Graph and + Gmail use); a missing/non-integer value falls back to a safe default, and + the wait is capped to avoid an unbounded stall. + """ + resp = session.get(url, **kwargs) + attempts = 0 + while resp.status_code == 429 and attempts < max_retries: + wait = min( + _safe_int(resp.headers.get("Retry-After"), _DEFAULT_RETRY_WAIT), + _MAX_RETRY_WAIT, + ) + time.sleep(max(wait, 0)) + resp = session.get(url, **kwargs) + attempts += 1 + return resp diff --git a/external-import/email-cases-importer/src/email_client/base.py b/external-import/email-cases-importer/src/email_client/base.py new file mode 100644 index 00000000000..cc8861f35c2 --- /dev/null +++ b/external-import/email-cases-importer/src/email_client/base.py @@ -0,0 +1,66 @@ +from abc import ABC, abstractmethod +from dataclasses import dataclass, field +from datetime import datetime + + +@dataclass +class EmailAttachment: + """Represents an email attachment.""" + + filename: str + content_type: str + content: bytes + size: int + + +@dataclass +class EmailMessage: + """Normalized email message across all protocols.""" + + message_id: str + subject: str + sender: str + recipients: list[str] + date: datetime + body_plain: str + body_html: str + thread_id: str + sender_display: str = "" + recipients_display: list[str] = field(default_factory=list) + in_reply_to: str = "" + references: list[str] = field(default_factory=list) + attachments: list[EmailAttachment] = field(default_factory=list) + raw_headers: dict[str, str] = field(default_factory=dict) + + +class BaseEmailClient(ABC): + """Abstract base class for email protocol clients.""" + + @abstractmethod + def connect(self) -> None: + """Establish connection to the email server.""" + + @abstractmethod + def disconnect(self) -> None: + """Close connection to the email server.""" + + @abstractmethod + def fetch_emails( + self, + sender: str, + since: datetime | None = None, + max_results: int = 50, + ) -> list[EmailMessage]: + """Fetch emails from a specific sender, optionally since a given date.""" + + @abstractmethod + def get_thread_id(self, message: EmailMessage) -> str: + """Extract or compute a thread identifier for the message.""" + + def __enter__(self): + self.connect() + return self + + def __exit__(self, exc_type, exc_val, exc_tb): + self.disconnect() + return False diff --git a/external-import/email-cases-importer/src/email_client/ews_client.py b/external-import/email-cases-importer/src/email_client/ews_client.py new file mode 100644 index 00000000000..78464122876 --- /dev/null +++ b/external-import/email-cases-importer/src/email_client/ews_client.py @@ -0,0 +1,187 @@ +from datetime import datetime, timezone + +from email_client.base import BaseEmailClient, EmailAttachment, EmailMessage + + +class EwsClient(BaseEmailClient): + """Exchange Web Services (EWS) email client using exchangelib.""" + + def __init__( + self, + server: str, + username: str, + password: str, + auth_type: str = "NTLM", + tls_verify: bool = True, + ): + self._server = server + self._username = username + self._password = password + self._auth_type = auth_type + self._tls_verify = tls_verify + self._account = None + self._prev_http_adapter_cls = None + + def connect(self) -> None: + if self._auth_type == "OAuth2": + raise NotImplementedError( + "EWS OAuth2 authentication is not implemented. Set " + "EMAIL_CASES_EWS_AUTH_TYPE=NTLM (default), or use the " + "microsoft_graph protocol for OAuth2-based Office 365 access." + ) + + from exchangelib import ( + DELEGATE, + Account, + Configuration, + Credentials, + ) + from exchangelib.protocol import BaseProtocol, NoVerifyHTTPAdapter + + if not self._tls_verify: + # HTTP_ADAPTER_CLS is a process-global in exchangelib; remember the + # previous value so disconnect() can restore it rather than leaving + # TLS verification disabled for every exchangelib user in the process. + self._prev_http_adapter_cls = BaseProtocol.HTTP_ADAPTER_CLS + BaseProtocol.HTTP_ADAPTER_CLS = NoVerifyHTTPAdapter + + credentials = Credentials(self._username, self._password) + + if self._server: + config = Configuration( + server=self._server, + credentials=credentials, + ) + self._account = Account( + primary_smtp_address=self._username, + config=config, + autodiscover=False, + access_type=DELEGATE, + ) + else: + self._account = Account( + primary_smtp_address=self._username, + credentials=credentials, + autodiscover=True, + access_type=DELEGATE, + ) + + def disconnect(self) -> None: + if self._account and self._account.protocol: + self._account.protocol.close() + self._account = None + if self._prev_http_adapter_cls is not None: + from exchangelib.protocol import BaseProtocol + + BaseProtocol.HTTP_ADAPTER_CLS = self._prev_http_adapter_cls + self._prev_http_adapter_cls = None + + def fetch_emails( + self, + sender: str, + since: datetime | None = None, + max_results: int = 50, + ) -> list[EmailMessage]: + if not self._account: + raise RuntimeError("Not connected to Exchange server") + + # Message.sender is a single-valued Mailbox field — exchangelib does NOT + # accept nested lookups (`sender__email_address`) or the legacy + # `from_emailaddresses` path; both raise InvalidField. The supported form + # is a Mailbox object compared by equality; email_address is sufficient. + from exchangelib import Mailbox + + inbox = self._account.inbox + qs = inbox.filter(sender=Mailbox(email_address=sender)) + + if since: + # exchangelib accepts tz-aware datetime directly; no EWSDateTime + # coercion needed on 5.x. + qs = qs.filter(datetime_received__gte=since.astimezone(timezone.utc)) + + qs = qs.order_by("-datetime_received")[:max_results] + + messages = [] + for item in qs: + msg = self._convert_item(item) + if msg: + messages.append(msg) + + return messages + + def _convert_item(self, item) -> EmailMessage | None: + message_id = item.message_id or "" + subject = item.subject or "" + sender = "" + sender_display = "" + if item.sender: + sender = item.sender.email_address or "" + name = item.sender.name or "" + sender_display = f"{name} <{sender}>" if name else sender + + recipients = [] + recipients_display = [] + for recip_list in (item.to_recipients, item.cc_recipients): + if recip_list: + for r in recip_list: + if r.email_address: + recipients.append(r.email_address) + name = r.name or "" + recipients_display.append( + f"{name} <{r.email_address}>" if name else r.email_address + ) + + date = item.datetime_received + if date and date.tzinfo is None: + date = date.replace(tzinfo=timezone.utc) + elif not date: + date = datetime.now(timezone.utc) + + body_plain = "" + body_html = "" + if item.body: + if item.body.body_type == "HTML": + body_html = item.body + else: + body_plain = str(item.body) + + in_reply_to = item.in_reply_to or "" + thread_id = "" + if item.conversation_id: + thread_id = str(item.conversation_id.id) + if not thread_id: + thread_id = message_id + + attachments = [] + if item.has_attachments and item.attachments: + for att in item.attachments: + from exchangelib import FileAttachment + + if isinstance(att, FileAttachment) and att.content: + attachments.append( + EmailAttachment( + filename=att.name or "unknown", + content_type=att.content_type or "application/octet-stream", + content=att.content, + size=len(att.content), + ) + ) + + return EmailMessage( + message_id=message_id, + subject=subject, + sender=sender, + recipients=recipients, + date=date, + body_plain=body_plain, + body_html=body_html, + thread_id=thread_id, + sender_display=sender_display, + recipients_display=recipients_display, + in_reply_to=in_reply_to, + references=[], + attachments=attachments, + ) + + def get_thread_id(self, message: EmailMessage) -> str: + return message.thread_id diff --git a/external-import/email-cases-importer/src/email_client/factory.py b/external-import/email-cases-importer/src/email_client/factory.py new file mode 100644 index 00000000000..9aac036e5c6 --- /dev/null +++ b/external-import/email-cases-importer/src/email_client/factory.py @@ -0,0 +1,52 @@ +from email_client.base import BaseEmailClient + + +def create_email_client(config) -> BaseEmailClient: + """Factory: create the appropriate email client based on protocol config.""" + protocol = config.email_cases.protocol + + if protocol == "imap": + from email_client.imap_client import ImapClient + + return ImapClient( + host=config.email_cases.imap_host, + port=config.email_cases.imap_port, + username=config.email_cases.imap_username, + password=config.email_cases.imap_password, + folder=config.email_cases.imap_folder, + use_ssl=config.email_cases.imap_use_ssl, + tls_verify=config.email_cases.tls_verify, + ) + + if protocol == "microsoft_graph": + from email_client.graph_client import GraphClient + + return GraphClient( + tenant_id=config.email_cases.graph_tenant_id, + client_id=config.email_cases.graph_client_id, + client_secret=config.email_cases.graph_client_secret, + user_id=config.email_cases.graph_user_id, + tls_verify=config.email_cases.tls_verify, + ) + + if protocol == "gmail": + from email_client.gmail_client import GmailClient + + return GmailClient( + credentials_file=config.email_cases.gmail_credentials_file, + user_id=config.email_cases.gmail_user_id, + tls_verify=config.email_cases.tls_verify, + ) + + if protocol == "ews": + from email_client.ews_client import EwsClient + + return EwsClient( + server=config.email_cases.ews_server, + username=config.email_cases.ews_username, + password=config.email_cases.ews_password, + auth_type=config.email_cases.ews_auth_type, + tls_verify=config.email_cases.tls_verify, + ) + + raise ValueError(f"Unsupported email protocol: {protocol}") diff --git a/external-import/email-cases-importer/src/email_client/gmail_client.py b/external-import/email-cases-importer/src/email_client/gmail_client.py new file mode 100644 index 00000000000..53b47084918 --- /dev/null +++ b/external-import/email-cases-importer/src/email_client/gmail_client.py @@ -0,0 +1,241 @@ +import base64 +from datetime import datetime, timezone + +import requests +from google.auth.transport.requests import Request +from google.oauth2 import service_account + +from email_client._http import get_with_retry, parse_json +from email_client.base import BaseEmailClient, EmailAttachment, EmailMessage + + +def _b64url(data: str) -> bytes: + """Decode a Gmail base64url payload, tolerating missing '=' padding. + + Gmail message part bodies are base64url-encoded and frequently omit + padding; ``base64.urlsafe_b64decode`` raises ``binascii.Error`` on such + input, so re-pad to a multiple of 4 first. + """ + return base64.urlsafe_b64decode(data + "=" * (-len(data) % 4)) + + +class GmailClient(BaseEmailClient): + """Gmail API email client.""" + + BASE_URL = "https://gmail.googleapis.com/gmail/v1" + SCOPES = ["https://www.googleapis.com/auth/gmail.readonly"] + + def __init__( + self, + credentials_file: str, + user_id: str = "me", + tls_verify: bool = True, + ): + self._credentials_file = credentials_file + self._user_id = user_id + self._tls_verify = tls_verify + self._session: requests.Session | None = None + self._credentials = None + + def connect(self) -> None: + self._credentials = service_account.Credentials.from_service_account_file( + self._credentials_file, scopes=self.SCOPES + ) + if self._user_id != "me": + self._credentials = self._credentials.with_subject(self._user_id) + self._credentials.refresh(Request()) + self._session = requests.Session() + self._session.verify = self._tls_verify + self._session.headers["Authorization"] = f"Bearer {self._credentials.token}" + + def disconnect(self) -> None: + if self._session: + self._session.close() + self._session = None + + def _refresh_if_needed(self) -> None: + if self._credentials and self._credentials.expired: + self._credentials.refresh(Request()) + self._session.headers["Authorization"] = f"Bearer {self._credentials.token}" + + def fetch_emails( + self, + sender: str, + since: datetime | None = None, + max_results: int = 50, + ) -> list[EmailMessage]: + if not self._session: + raise RuntimeError("Not connected to Gmail API") + + self._refresh_if_needed() + + query_parts = [f"from:{sender}"] + if since: + epoch = int(since.timestamp()) + query_parts.append(f"after:{epoch}") + + url = f"{self.BASE_URL}/users/{self._user_id}/messages" + query = " ".join(query_parts) + + # Gmail caps a single list page at 100 results and returns a + # nextPageToken when more match; page through until we have collected + # max_results message IDs (or run out) so max_results > 100 is honoured + # instead of silently truncating to the first page. + message_ids: list[str] = [] + page_token: str | None = None + while len(message_ids) < max_results: + params = { + "q": query, + "maxResults": min(max_results - len(message_ids), 100), + } + if page_token: + params["pageToken"] = page_token + resp = get_with_retry(self._session, url, params=params) + resp.raise_for_status() + data = parse_json(resp) + page = data.get("messages", []) + if not page: + break + message_ids.extend(item["id"] for item in page) + page_token = data.get("nextPageToken") + if not page_token: + break + + messages = [] + for message_id in message_ids[:max_results]: + msg = self._fetch_full_message(message_id) + if msg: + messages.append(msg) + + return messages + + def _fetch_full_message(self, message_id: str) -> EmailMessage | None: + url = ( + f"{self.BASE_URL}/users/{self._user_id}/messages/{message_id}" + f"?format=full" + ) + resp = get_with_retry(self._session, url) + resp.raise_for_status() + data = parse_json(resp) + + headers = {} + for h in data.get("payload", {}).get("headers", []): + headers[h["name"].lower()] = h["value"] + + subject = headers.get("subject", "") + sender_raw = headers.get("from", "") + # Extract email from "Name " format + sender_display = sender_raw + if "<" in sender_raw and ">" in sender_raw: + sender = sender_raw.split("<")[1].rstrip(">") + else: + sender = sender_raw + to_raw = [r.strip() for r in headers.get("to", "").split(",") if r.strip()] + cc_raw = [r.strip() for r in headers.get("cc", "").split(",") if r.strip()] + all_raw = to_raw + cc_raw + recipients = [] + recipients_display = list(all_raw) + for r in all_raw: + if "<" in r and ">" in r: + recipients.append(r.split("<")[1].rstrip(">")) + else: + recipients.append(r) + + date_str = headers.get("date", "") + try: + from email.utils import parsedate_to_datetime + + date = parsedate_to_datetime(date_str) + if date.tzinfo is None: + date = date.replace(tzinfo=timezone.utc) + except Exception: + internal_ts = data.get("internalDate", "0") + date = datetime.fromtimestamp(int(internal_ts) / 1000, tz=timezone.utc) + + in_reply_to = headers.get("in-reply-to", "") + references_raw = headers.get("references", "") + references = references_raw.split() if references_raw else [] + gmail_message_id = headers.get("message-id", message_id) + + thread_id = data.get("threadId", gmail_message_id) + + body_plain, body_html = self._extract_body(data.get("payload", {})) + attachments = self._extract_attachments(data, message_id) + + return EmailMessage( + message_id=gmail_message_id, + subject=subject, + sender=sender, + recipients=recipients, + date=date, + body_plain=body_plain, + body_html=body_html, + thread_id=thread_id, + sender_display=sender_display, + recipients_display=recipients_display, + in_reply_to=in_reply_to, + references=references, + attachments=attachments, + ) + + def _extract_body(self, payload: dict) -> tuple[str, str]: + body_plain = "" + body_html = "" + + mime_type = payload.get("mimeType", "") + body_data = payload.get("body", {}).get("data", "") + + if body_data and mime_type == "text/plain": + body_plain = _b64url(body_data).decode("utf-8", errors="replace") + elif body_data and mime_type == "text/html": + body_html = _b64url(body_data).decode("utf-8", errors="replace") + + for part in payload.get("parts", []): + p_plain, p_html = self._extract_body(part) + if p_plain and not body_plain: + body_plain = p_plain + if p_html and not body_html: + body_html = p_html + + return body_plain, body_html + + def _extract_attachments( + self, data: dict, message_id: str + ) -> list[EmailAttachment]: + attachments = [] + payload = data.get("payload", {}) + self._collect_attachments(payload, message_id, attachments) + return attachments + + def _collect_attachments( + self, part: dict, message_id: str, attachments: list + ) -> None: + filename = part.get("filename", "") + body = part.get("body", {}) + + if filename and body.get("attachmentId"): + content = self._download_attachment(message_id, body["attachmentId"]) + attachments.append( + EmailAttachment( + filename=filename, + content_type=part.get("mimeType", "application/octet-stream"), + content=content, + size=len(content), + ) + ) + + for sub_part in part.get("parts", []): + self._collect_attachments(sub_part, message_id, attachments) + + def _download_attachment(self, message_id: str, attachment_id: str) -> bytes: + url = ( + f"{self.BASE_URL}/users/{self._user_id}/messages/" + f"{message_id}/attachments/{attachment_id}" + ) + resp = get_with_retry(self._session, url) + resp.raise_for_status() + data_b64 = parse_json(resp).get("data", "") + return _b64url(data_b64) + + def get_thread_id(self, message: EmailMessage) -> str: + return message.thread_id diff --git a/external-import/email-cases-importer/src/email_client/graph_client.py b/external-import/email-cases-importer/src/email_client/graph_client.py new file mode 100644 index 00000000000..5c95c16f41d --- /dev/null +++ b/external-import/email-cases-importer/src/email_client/graph_client.py @@ -0,0 +1,199 @@ +import base64 +from datetime import datetime, timezone + +import requests + +from email_client._http import get_with_retry, parse_json +from email_client.base import BaseEmailClient, EmailAttachment, EmailMessage + + +class GraphClient(BaseEmailClient): + """Microsoft Graph API email client for Office 365 / Exchange Online.""" + + BASE_URL = "https://graph.microsoft.com/v1.0" + + def __init__( + self, + tenant_id: str, + client_id: str, + client_secret: str, + user_id: str, + tls_verify: bool = True, + ): + self._tenant_id = tenant_id + self._client_id = client_id + self._client_secret = client_secret + self._user_id = user_id + self._tls_verify = tls_verify + self._session: requests.Session | None = None + self._access_token: str = "" + + def connect(self) -> None: + self._session = requests.Session() + self._session.verify = self._tls_verify + self._authenticate() + + def disconnect(self) -> None: + if self._session: + self._session.close() + self._session = None + + def _authenticate(self) -> None: + token_url = ( + f"https://login.microsoftonline.com/{self._tenant_id}/oauth2/v2.0/token" + ) + data = { + "client_id": self._client_id, + "client_secret": self._client_secret, + "scope": "https://graph.microsoft.com/.default", + "grant_type": "client_credentials", + } + resp = self._session.post(token_url, data=data) + resp.raise_for_status() + self._access_token = parse_json(resp)["access_token"] + self._session.headers["Authorization"] = f"Bearer {self._access_token}" + + def fetch_emails( + self, + sender: str, + since: datetime | None = None, + max_results: int = 50, + ) -> list[EmailMessage]: + if not self._session: + raise RuntimeError("Not connected to Microsoft Graph") + + # Escape single quotes in sender to prevent OData injection + safe_sender = sender.replace("'", "''") + filter_parts = [f"from/emailAddress/address eq '{safe_sender}'"] + if since: + # Normalize to UTC before appending the 'Z' suffix so a non-UTC + # offset doesn't shift the query window. + iso_since = since.astimezone(timezone.utc).strftime("%Y-%m-%dT%H:%M:%SZ") + filter_parts.append(f"receivedDateTime ge {iso_since}") + + params = { + "$filter": " and ".join(filter_parts), + "$select": ( + "id,subject,from,toRecipients,body,receivedDateTime," + "conversationId,internetMessageHeaders,internetMessageId," + "hasAttachments" + ), + "$orderby": "receivedDateTime desc", + "$top": min(max_results, 50), + } + + url = f"{self.BASE_URL}/users/{self._user_id}/mailFolders/inbox/messages" + messages = [] + + while url and len(messages) < max_results: + resp = get_with_retry(self._session, url, params=params) + if resp.status_code == 401: + self._authenticate() + resp = get_with_retry(self._session, url, params=params) + resp.raise_for_status() + data = parse_json(resp) + + for item in data.get("value", []): + msg = self._parse_message(item) + if msg: + messages.append(msg) + + url = data.get("@odata.nextLink") + params = None # nextLink already has params + + return messages[:max_results] + + def _parse_message(self, item: dict) -> EmailMessage | None: + message_id = item.get("internetMessageId", item.get("id", "")) + subject = item.get("subject", "") + from_email = item.get("from", {}).get("emailAddress", {}) + sender = from_email.get("address", "") + sender_name = from_email.get("name", "") + sender_display = f"{sender_name} <{sender}>" if sender_name else sender + + def _parse_recipients(items: list[dict]) -> tuple[list[str], list[str]]: + addrs, displays = [], [] + for r in items: + ea = r.get("emailAddress", {}) + addr = ea.get("address", "") + name = ea.get("name", "") + if addr: + addrs.append(addr) + displays.append(f"{name} <{addr}>" if name else addr) + return addrs, displays + + to_addrs, to_display = _parse_recipients(item.get("toRecipients", [])) + cc_addrs, cc_display = _parse_recipients(item.get("ccRecipients", [])) + recipients = to_addrs + cc_addrs + recipients_display = to_display + cc_display + + received = item.get("receivedDateTime", "") + if received: + date = datetime.fromisoformat(received.replace("Z", "+00:00")) + else: + date = datetime.now(timezone.utc) + + body = item.get("body", {}) + body_content = body.get("content", "") + body_type = body.get("contentType", "text") + body_plain = body_content if body_type == "text" else "" + body_html = body_content if body_type == "html" else "" + + # Extract headers + in_reply_to = "" + references = [] + for header in item.get("internetMessageHeaders", []): + name = header.get("name", "").lower() + value = header.get("value", "") + if name == "in-reply-to": + in_reply_to = value + elif name == "references": + references = value.split() + + thread_id = item.get("conversationId", message_id) + + attachments = [] + if item.get("hasAttachments"): + attachments = self._fetch_attachments(item["id"]) + + return EmailMessage( + message_id=message_id, + subject=subject, + sender=sender, + recipients=recipients, + date=date, + body_plain=body_plain, + body_html=body_html, + thread_id=thread_id, + sender_display=sender_display, + recipients_display=recipients_display, + in_reply_to=in_reply_to, + references=references, + attachments=attachments, + ) + + def _fetch_attachments(self, message_id: str) -> list[EmailAttachment]: + url = ( + f"{self.BASE_URL}/users/{self._user_id}/messages/" + f"{message_id}/attachments" + ) + resp = get_with_retry(self._session, url) + resp.raise_for_status() + + attachments = [] + for att in parse_json(resp).get("value", []): + if att.get("@odata.type") != "#microsoft.graph.fileAttachment": + continue + content_bytes = base64.b64decode(att.get("contentBytes", "")) + attachments.append( + EmailAttachment( + filename=att.get("name", "unknown"), + content_type=att.get("contentType", "application/octet-stream"), + content=content_bytes, + size=len(content_bytes), + ) + ) + return attachments + + def get_thread_id(self, message: EmailMessage) -> str: + return message.thread_id diff --git a/external-import/email-cases-importer/src/email_client/imap_client.py b/external-import/email-cases-importer/src/email_client/imap_client.py new file mode 100644 index 00000000000..d994341eb3d --- /dev/null +++ b/external-import/email-cases-importer/src/email_client/imap_client.py @@ -0,0 +1,216 @@ +import email +import email.header +import email.utils +import imaplib +import ssl +from datetime import datetime, timezone + +from email_client.base import BaseEmailClient, EmailAttachment, EmailMessage + + +class ImapClient(BaseEmailClient): + """IMAP4 email client implementation.""" + + def __init__( + self, + host: str, + port: int = 993, + username: str = "", + password: str = "", + folder: str = "INBOX", + use_ssl: bool = True, + tls_verify: bool = True, + ): + self._host = host + self._port = port + self._username = username + self._password = password + self._folder = folder + self._use_ssl = use_ssl + self._tls_verify = tls_verify + self._connection: imaplib.IMAP4 | imaplib.IMAP4_SSL | None = None + + def connect(self) -> None: + if self._use_ssl: + ctx = ssl.create_default_context() + if not self._tls_verify: + ctx.check_hostname = False + ctx.verify_mode = ssl.CERT_NONE + self._connection = imaplib.IMAP4_SSL( + self._host, self._port, ssl_context=ctx + ) + else: + self._connection = imaplib.IMAP4(self._host, self._port) + self._connection.login(self._username, self._password) + self._connection.select(self._folder) + + def disconnect(self) -> None: + if self._connection: + try: + self._connection.close() + self._connection.logout() + except Exception: + pass + self._connection = None + + def fetch_emails( + self, + sender: str, + since: datetime | None = None, + max_results: int = 50, + ) -> list[EmailMessage]: + if not self._connection: + raise RuntimeError("Not connected to IMAP server") + + criteria_parts = [f'FROM "{sender}"'] + if since: + date_str = since.strftime("%d-%b-%Y") + criteria_parts.append(f"SINCE {date_str}") + + search_criteria = "(" + " ".join(criteria_parts) + ")" + _, data = self._connection.search(None, search_criteria) + + if not data or not data[0]: + return [] + + message_ids = data[0].split() + # Limit results + message_ids = message_ids[-max_results:] + + messages = [] + for mid in message_ids: + _, msg_data = self._connection.fetch(mid, "(RFC822)") + if not msg_data or not msg_data[0]: + continue + raw_email = msg_data[0][1] + msg = self._parse_email(raw_email) + if msg: + messages.append(msg) + + return messages + + def get_thread_id(self, message: EmailMessage) -> str: + # IMAP doesn't have native thread IDs — use Message-ID as base + if message.in_reply_to: + return message.in_reply_to + if message.references: + return message.references[0] + return message.message_id + + def _parse_email(self, raw: bytes) -> EmailMessage | None: + msg = email.message_from_bytes(raw) + + message_id = msg.get("Message-ID", "") + subject = self._decode_header(msg.get("Subject", "")) + sender_raw = msg.get("From", "") + sender_name, sender_addr = email.utils.parseaddr(sender_raw) + sender = sender_addr + sender_display = ( + f"{self._decode_header(sender_name)} <{sender_addr}>" + if sender_name + else sender_addr + ) + to_parsed = [email.utils.parseaddr(r) for r in (msg.get_all("To") or [])] + cc_parsed = [email.utils.parseaddr(r) for r in (msg.get_all("Cc") or [])] + all_parsed = to_parsed + cc_parsed + recipients = [addr for _, addr in all_parsed if addr] + recipients_display = [ + f"{self._decode_header(name)} <{addr}>" if name else addr + for name, addr in all_parsed + if addr + ] + # parsedate_to_datetime returns None (and on some Python versions + # raises) for a missing/malformed Date header — fall back to "now" + # rather than crashing the whole fetch cycle. + try: + date_tuple = email.utils.parsedate_to_datetime(msg.get("Date", "")) + except (TypeError, ValueError): + date_tuple = None + if date_tuple is None: + date_tuple = datetime.now(timezone.utc) + elif date_tuple.tzinfo is None: + date_tuple = date_tuple.replace(tzinfo=timezone.utc) + in_reply_to = msg.get("In-Reply-To", "") + references_raw = msg.get("References", "") + references = references_raw.split() if references_raw else [] + + body_plain = "" + body_html = "" + attachments = [] + + if msg.is_multipart(): + for part in msg.walk(): + content_type = part.get_content_type() + disposition = str(part.get("Content-Disposition", "")) + + if "attachment" in disposition: + att = self._extract_attachment(part) + if att: + attachments.append(att) + elif content_type == "text/plain" and not body_plain: + payload = part.get_payload(decode=True) + if payload: + charset = part.get_content_charset() or "utf-8" + body_plain = payload.decode(charset, errors="replace") + elif content_type == "text/html" and not body_html: + payload = part.get_payload(decode=True) + if payload: + charset = part.get_content_charset() or "utf-8" + body_html = payload.decode(charset, errors="replace") + else: + payload = msg.get_payload(decode=True) + if payload: + charset = msg.get_content_charset() or "utf-8" + if msg.get_content_type() == "text/html": + body_html = payload.decode(charset, errors="replace") + else: + body_plain = payload.decode(charset, errors="replace") + + raw_headers = {} + for key in msg.keys(): + raw_headers[key] = msg[key] + + return EmailMessage( + message_id=message_id, + subject=subject, + sender=sender, + recipients=recipients, + date=date_tuple, + body_plain=body_plain, + body_html=body_html, + thread_id=message_id, + sender_display=sender_display, + recipients_display=recipients_display, + in_reply_to=in_reply_to, + references=references, + attachments=attachments, + raw_headers=raw_headers, + ) + + def _extract_attachment(self, part) -> EmailAttachment | None: + filename = part.get_filename() + if not filename: + return None + filename = self._decode_header(filename) + content = part.get_payload(decode=True) + if not content: + return None + return EmailAttachment( + filename=filename, + content_type=part.get_content_type(), + content=content, + size=len(content), + ) + + @staticmethod + def _decode_header(value: str) -> str: + if not value: + return "" + decoded_parts = email.header.decode_header(value) + result = [] + for part, charset in decoded_parts: + if isinstance(part, bytes): + result.append(part.decode(charset or "utf-8", errors="replace")) + else: + result.append(part) + return " ".join(result) diff --git a/external-import/email-cases-importer/src/main.py b/external-import/email-cases-importer/src/main.py new file mode 100644 index 00000000000..e1925c8b26b --- /dev/null +++ b/external-import/email-cases-importer/src/main.py @@ -0,0 +1,17 @@ +import traceback + +from pycti import OpenCTIConnectorHelper + +from connector import ConnectorSettings, EmailCasesConnector + +if __name__ == "__main__": + try: + settings = ConnectorSettings() + helper = OpenCTIConnectorHelper( + config=settings.to_helper_config(), + ) + connector = EmailCasesConnector(config=settings, helper=helper) + connector.run() + except Exception: + traceback.print_exc() + exit(1) diff --git a/external-import/email-cases-importer/src/requirements.txt b/external-import/email-cases-importer/src/requirements.txt new file mode 100644 index 00000000000..f711435633a --- /dev/null +++ b/external-import/email-cases-importer/src/requirements.txt @@ -0,0 +1,9 @@ +pycti==7.260715.0 +connectors-sdk @ git+https://github.com/OpenCTI-Platform/connectors.git@7.260715.0#subdirectory=connectors-sdk +requests>=2.32.0 +py7zr>=0.22.0 +pikepdf>=9.0.0 +msoffcrypto-tool>=5.4.0 +openpyxl>=3.1.0 +exchangelib>=5.4.0 +google-auth>=2.29.0 diff --git a/external-import/email-cases-importer/test-requirements.txt b/external-import/email-cases-importer/test-requirements.txt new file mode 100644 index 00000000000..44400a3a375 --- /dev/null +++ b/external-import/email-cases-importer/test-requirements.txt @@ -0,0 +1,8 @@ +pytest>=8.0.0 +pytest-mock>=3.12.0 +pytest-cov>=5.0.0 +coverage>=7.4.0 +black>=24.0.0 +isort>=5.13.0 +flake8>=7.0.0 +pylint>=3.2.0 diff --git a/external-import/email-cases-importer/tests/__init__.py b/external-import/email-cases-importer/tests/__init__.py new file mode 100644 index 00000000000..e69de29bb2d diff --git a/external-import/email-cases-importer/tests/conftest.py b/external-import/email-cases-importer/tests/conftest.py new file mode 100644 index 00000000000..ed08e8a8d0e --- /dev/null +++ b/external-import/email-cases-importer/tests/conftest.py @@ -0,0 +1,101 @@ +"""Shared pytest fixtures and path setup for the connector test suite.""" + +import os +import sys +import types + +# Make `src/` importable so tests can do `from connector...`, `from email_client...`, +# `from attachment_handler...` exactly the way the runtime image does. +SRC_DIR = os.path.abspath(os.path.join(os.path.dirname(__file__), "..", "src")) +if SRC_DIR not in sys.path: + sys.path.insert(0, SRC_DIR) + +# --------------------------------------------------------------------------- +# Stub `connectors_sdk` for unit tests. +# +# The connectors-sdk package is installed from a git tag in production, but +# we don't want unit tests to depend on a network install. We replace the +# module with thin pydantic.BaseModel subclasses that match the surface area +# settings.py uses (BaseConfigModel, BaseConnectorSettings, +# BaseExternalImportConnectorConfig). +# --------------------------------------------------------------------------- +if "connectors_sdk" not in sys.modules: + from pydantic import BaseModel, ConfigDict + + fake_sdk = types.ModuleType("connectors_sdk") + + class _SdkBase(BaseModel): + # Allow ignoring extra env-style keys when settings are constructed in tests + model_config = ConfigDict(extra="ignore") + + class BaseConfigModel(_SdkBase): + pass + + class BaseConnectorSettings(_SdkBase): + pass + + class BaseExternalImportConnectorConfig(_SdkBase): + pass + + fake_sdk.BaseConfigModel = BaseConfigModel + fake_sdk.BaseConnectorSettings = BaseConnectorSettings + fake_sdk.BaseExternalImportConnectorConfig = BaseExternalImportConnectorConfig + sys.modules["connectors_sdk"] = fake_sdk + +# --------------------------------------------------------------------------- +# Stub `google.*` for gmail_client unit tests. +# +# gmail_client imports `from google.auth.transport.requests import Request` and +# `from google.oauth2 import service_account` at module load. google-auth is an +# optional protocol dependency that isn't installed in the unit-test env, so we +# register thin module stubs (placeholder Request / service_account.Credentials). +# Tests patch the module-level `Request`/`service_account` names directly. +# --------------------------------------------------------------------------- +if "google.auth" not in sys.modules: + _mod_names = [ + "google", + "google.auth", + "google.auth.transport", + "google.auth.transport.requests", + "google.oauth2", + "google.oauth2.service_account", + ] + for _name in _mod_names: + sys.modules.setdefault(_name, types.ModuleType(_name)) + # Wire submodules as attributes on their parents so `from X import Y` works. + sys.modules["google"].auth = sys.modules["google.auth"] + sys.modules["google.auth"].transport = sys.modules["google.auth.transport"] + sys.modules["google.auth.transport"].requests = sys.modules[ + "google.auth.transport.requests" + ] + sys.modules["google"].oauth2 = sys.modules["google.oauth2"] + sys.modules["google.oauth2"].service_account = sys.modules[ + "google.oauth2.service_account" + ] + + class _Request: # placeholder, patched in tests + pass + + class _Credentials: # placeholder, patched in tests + @staticmethod + def from_service_account_file(*a, **k): + raise NotImplementedError + + sys.modules["google.auth.transport.requests"].Request = _Request + sys.modules["google.oauth2.service_account"].Credentials = _Credentials + +# --------------------------------------------------------------------------- +# Warm-up imports. +# +# `connector/__init__.py` eagerly imports `connector.connector`, which in turn +# imports `attachment_handler.registry`. If a test starts by importing from +# `attachment_handler.*` first, the chain re-enters `attachment_handler.registry` +# while it is still being initialized (because `archive_handler` reaches back +# into `connector.utils`, which triggers the connector package init), and the +# circular import fails. +# +# In production, `main.py` imports `connector` first, so the chain warms up in +# the right order. We replicate that here so tests don't depend on which test +# file is collected first. +# --------------------------------------------------------------------------- +import connector # noqa: E402,F401 pylint: disable=wrong-import-position,unused-import diff --git a/external-import/email-cases-importer/tests/test_attachment_handlers.py b/external-import/email-cases-importer/tests/test_attachment_handlers.py new file mode 100644 index 00000000000..9e8094b2914 --- /dev/null +++ b/external-import/email-cases-importer/tests/test_attachment_handlers.py @@ -0,0 +1,158 @@ +"""Unit tests for attachment_handler.passthrough_handler and registry routing.""" + +import os + +import pytest + +from attachment_handler.base import BaseAttachmentHandler, ExtractedFile +from attachment_handler.passthrough_handler import PassthroughHandler +from attachment_handler.registry import HandlerRegistry + +# --------------------------------------------------------------------------- +# PassthroughHandler +# --------------------------------------------------------------------------- + + +class TestPassthroughHandler: + def setup_method(self): + self.handler = PassthroughHandler() + + def test_supported_extensions(self): + exts = self.handler.supported_extensions() + assert ".csv" in exts + assert ".txt" in exts + assert ".eml" in exts + assert ".json" in exts + + def test_can_handle(self): + assert self.handler.can_handle("notes.txt") is True + assert self.handler.can_handle("DATA.CSV") is True + assert self.handler.can_handle("archive.zip") is False + + @pytest.mark.parametrize( + "ext,expected_mime", + [ + (".csv", "text/csv"), + (".txt", "text/plain"), + (".eml", "message/rfc822"), + (".json", "application/json"), + (".xml", "application/xml"), + (".html", "text/html"), + ], + ) + def test_content_type_mapping(self, tmp_path, ext, expected_mime): + f = tmp_path / f"sample{ext}" + f.write_bytes(b"hello") + results = self.handler.extract(str(f)) + assert len(results) == 1 + assert results[0].content_type == expected_mime + assert results[0].content == b"hello" + assert results[0].was_encrypted is False + + def test_extract_populates_hashes(self, tmp_path): + f = tmp_path / "x.txt" + f.write_bytes(b"hello") + results = self.handler.extract(str(f)) + assert set(results[0].hashes.keys()) == {"MD5", "SHA-1", "SHA-256"} + + +# --------------------------------------------------------------------------- +# HandlerRegistry routing +# --------------------------------------------------------------------------- + + +class TestHandlerRegistryRouting: + def setup_method(self): + self.registry = HandlerRegistry() + + def test_get_handler_by_extension(self): + h = self.registry.get_handler("notes.txt") + assert isinstance(h, PassthroughHandler) + + def test_get_handler_unknown_extension(self): + # .xyz is not registered by any default handler + assert self.registry.get_handler("file.xyz") is None + + def test_get_handler_no_extension(self): + assert self.registry.get_handler("noext") is None + + def test_case_insensitive_matching(self): + assert isinstance(self.registry.get_handler("NOTES.TXT"), PassthroughHandler) + + def test_archive_handler_registered_for_zip(self): + # ArchiveHandler should claim .zip + h = self.registry.get_handler("evidence.zip") + assert h is not None + assert h.__class__.__name__ == "ArchiveHandler" + + +# --------------------------------------------------------------------------- +# HandlerRegistry.process_attachment +# --------------------------------------------------------------------------- + + +class TestProcessAttachment: + def setup_method(self): + self.registry = HandlerRegistry() + + def test_oversized_file_returns_skipped(self): + big = b"x" * (2 * 1024 * 1024) # 2 MiB + results = self.registry.process_attachment("huge.txt", big, max_size_mb=1) + assert len(results) == 1 + assert results[0].metadata.get("skipped") == "exceeds_max_size" + assert results[0].content == b"" + # Hashes are still computed against the original bytes + assert results[0].hashes + + def test_unknown_extension_returns_raw_with_hashes(self): + results = self.registry.process_attachment( + "weird.xyz", b"payload", max_size_mb=10 + ) + assert len(results) == 1 + assert results[0].content == b"payload" + assert results[0].content_type == "application/octet-stream" + assert "SHA-256" in results[0].hashes + + def test_passthrough_file_processed(self): + results = self.registry.process_attachment( + "notes.txt", b"hello world", max_size_mb=10 + ) + assert len(results) == 1 + assert results[0].filename == "notes.txt" + assert results[0].content == b"hello world" + assert results[0].content_type == "text/plain" + + +# --------------------------------------------------------------------------- +# Custom handler registration (extensibility) +# --------------------------------------------------------------------------- + + +class FakeHandler(BaseAttachmentHandler): + def supported_extensions(self): + return [".fake"] + + def extract(self, file_path, passwords=None): + return [ + ExtractedFile( + filename=os.path.basename(file_path), + content=b"FAKE", + content_type="application/x-fake", + ) + ] + + +class TestCustomHandlerRegistration: + def test_register_custom_handler(self): + registry = HandlerRegistry() + registry.register(FakeHandler()) + h = registry.get_handler("thing.fake") + assert isinstance(h, FakeHandler) + + def test_custom_handler_used_by_process(self, tmp_path): + registry = HandlerRegistry() + registry.register(FakeHandler()) + results = registry.process_attachment("a.fake", b"ignored", max_size_mb=10) + assert len(results) == 1 + assert results[0].content == b"FAKE" + assert results[0].content_type == "application/x-fake" diff --git a/external-import/email-cases-importer/tests/test_connector.py b/external-import/email-cases-importer/tests/test_connector.py new file mode 100644 index 00000000000..5b39e78be60 --- /dev/null +++ b/external-import/email-cases-importer/tests/test_connector.py @@ -0,0 +1,636 @@ +"""Orchestration tests for connector.connector.EmailCasesConnector. + +These exercise the import loop, per-email processing, deterministic Case-Incident +id generation, the OpenCTI-resolution helpers (labels/identities/markings/members), +subject/sender rule matching, vocabulary bootstrap, case templates, and the +timeout wrappers — all against a mocked OpenCTI helper and a fake email client. +""" + +from datetime import datetime, timezone +from types import SimpleNamespace +from unittest.mock import MagicMock + +import pycti +import pytest + +import connector.connector as connector_mod +from connector.connector import EmailCasesConnector +from email_client.base import EmailAttachment, EmailMessage + + +# --------------------------------------------------------------------------- # +# Builders +# --------------------------------------------------------------------------- # +def make_config(**overrides): + """Build a config whose `.email_cases` matches the runtime settings surface.""" + ec = SimpleNamespace( + import_interval=300, + sender_address="alerts@example.com", + password_prefix="---BEGIN PASSWORD---", + password_suffix="---END PASSWORD---", + password_strip_whitespace=False, + thread_tracking_strategy="provider_thread_id", + max_attachment_size_mb=25, + max_emails_per_cycle=50, + display_sender_names=True, + email_fetch_timeout=120, + default_severity="medium", + default_priority="P3", + case_prefix="", + protocol="imap", + ) + for key in ( + "import_interval", + "sender_address", + "thread_tracking_strategy", + "case_prefix", + "default_severity", + "default_priority", + "display_sender_names", + "email_fetch_timeout", + "max_emails_per_cycle", + ): + if key in overrides: + setattr(ec, key, overrides[key]) + + ec.get_parsed_subject_filters = lambda: overrides.get("subject_filters", []) + ec.get_parsed_labels = lambda: overrides.get("labels", []) + ec.get_parsed_subject_rules = lambda: overrides.get("subject_rules", []) + ec.get_parsed_sender_rules = lambda: overrides.get("sender_rules", []) + ec.get_parsed_start_date = lambda: overrides.get("start_date", None) + return SimpleNamespace(email_cases=ec) + + +def make_email(**overrides): + base = dict( + message_id="", + subject="Security Alert", + sender="alerts@example.com", + recipients=["soc@company.com"], + date=datetime(2026, 4, 8, 12, 30, 0, tzinfo=timezone.utc), + body_plain="hello body", + body_html="", + thread_id="thread-1", + sender_display="Alerts ", + recipients_display=["SOC "], + ) + base.update(overrides) + return EmailMessage(**base) + + +class FakeClient: + """Context-manager email client returning a fixed list of emails.""" + + def __init__(self, emails=None): + self._emails = emails or [] + self.connected = False + + def __enter__(self): + self.connect() + return self + + def __exit__(self, *a): + self.disconnect() + return False + + def connect(self): + self.connected = True + + def disconnect(self): + self.connected = False + + def fetch_emails(self, sender, since=None, max_results=50): + return self._emails + + def get_thread_id(self, message): + return message.thread_id + + +@pytest.fixture +def helper(): + h = MagicMock() + h.get_state.return_value = None + h.connect_id = "connector-id" + h.api.work.initiate_work.return_value = "work-1" + h.api.case_incident.create.return_value = {"id": "case-internal-1"} + h.api.case_incident.list.return_value = [] + h.api.label.create.return_value = {"id": "label-1"} + h.api.identity.create.return_value = {"id": "identity-1"} + h.api.marking_definition.read.return_value = {"id": "marking-1"} + h.api.query.return_value = {"data": {}} + return h + + +@pytest.fixture +def conn(helper): + return EmailCasesConnector(make_config(), helper) + + +def build(helper, **cfg): + return EmailCasesConnector(make_config(**cfg), helper) + + +# --------------------------------------------------------------------------- # +# _create_case — deterministic id (the PR #6164 standard) +# --------------------------------------------------------------------------- # +class TestCreateCaseDeterministicId: + def test_passes_pycti_generated_stix_id_and_created(self, conn, helper): + email = make_email(subject="Security Alert") + normalized = "Security Alert" + conn._create_case(email, normalized, "

content

") + + kwargs = helper.api.case_incident.create.call_args.kwargs + created_iso = email.date.strftime("%Y-%m-%dT%H:%M:%SZ") + expected_id = pycti.CaseIncident.generate_id( + name=normalized, created=created_iso + ) + assert kwargs["stix_id"] == expected_id + assert kwargs["created"] == created_iso + assert kwargs["name"] == normalized + assert kwargs["createdBy"] == conn.converter.identity_id + + def test_same_email_yields_same_id(self, conn, helper): + email = make_email() + conn._create_case(email, "Security Alert", "c") + first = helper.api.case_incident.create.call_args.kwargs["stix_id"] + helper.api.case_incident.create.reset_mock() + conn._create_case(email, "Security Alert", "c") + second = helper.api.case_incident.create.call_args.kwargs["stix_id"] + assert first == second + + def test_prefix_applied_to_name(self, helper): + c = build(helper, case_prefix="[IR] ") + c._create_case(make_email(), "Security Alert", "c") + assert c.helper.api.case_incident.create.call_args.kwargs["name"] == ( + "[IR] Security Alert" + ) + + def test_optional_kwargs_forwarded(self, conn, helper): + conn._create_case( + make_email(), + "Security Alert", + "c", + label_ids=["l1"], + response_types=["rt"], + severity="high", + priority="P1", + author_id="author-x", + marking_id="m-x", + assignee_ids=["a1"], + participant_ids=["p1"], + ) + kwargs = helper.api.case_incident.create.call_args.kwargs + assert kwargs["objectLabel"] == ["l1"] + assert kwargs["response_types"] == ["rt"] + assert kwargs["severity"] == "high" + assert kwargs["priority"] == "P1" + assert kwargs["createdBy"] == "author-x" + assert kwargs["objectMarking"] == ["m-x"] + assert kwargs["objectAssignee"] == ["a1"] + assert kwargs["objectParticipant"] == ["p1"] + + +# --------------------------------------------------------------------------- # +# _find_existing_case / _append_to_case +# --------------------------------------------------------------------------- # +class TestFindAndAppend: + def test_find_returns_id(self, conn, helper): + helper.api.case_incident.list.return_value = [{"id": "existing-1"}] + assert conn._find_existing_case("Security Alert") == "existing-1" + + def test_find_returns_none_when_empty(self, conn, helper): + helper.api.case_incident.list.return_value = [] + assert conn._find_existing_case("Security Alert") is None + + def test_find_returns_none_on_error(self, conn, helper): + helper.api.case_incident.list.side_effect = RuntimeError("boom") + assert conn._find_existing_case("Security Alert") is None + + def test_append_concatenates_existing_content(self, conn, helper): + helper.api.query.return_value = {"data": {"caseIncident": {"content": "OLD"}}} + conn._append_to_case("case-1", "NEW") + update = helper.api.stix_domain_object.update_field.call_args.kwargs + assert update["id"] == "case-1" + assert update["input"]["value"] == ["OLD\nNEW"] + + def test_append_handles_missing_existing_content(self, conn, helper): + helper.api.query.return_value = {"data": {"caseIncident": None}} + conn._append_to_case("case-1", "NEW") + update = helper.api.stix_domain_object.update_field.call_args.kwargs + assert update["input"]["value"] == ["\nNEW"] + + +# --------------------------------------------------------------------------- # +# _resolve_thread_id +# --------------------------------------------------------------------------- # +class TestResolveThreadId: + def test_provider_thread_id(self, helper): + c = build(helper, thread_tracking_strategy="provider_thread_id") + assert c._resolve_thread_id(make_email(thread_id="T9"), FakeClient()) == "T9" + + def test_provider_falls_back_to_subject(self, helper): + c = build(helper, thread_tracking_strategy="provider_thread_id") + email = make_email(thread_id="") + assert c._resolve_thread_id(email, FakeClient()).startswith("subject:") + + def test_message_headers_in_reply_to(self, helper): + c = build(helper, thread_tracking_strategy="message_headers") + email = make_email(in_reply_to="") + assert c._resolve_thread_id(email, FakeClient()) == "" + + def test_message_headers_references(self, helper): + c = build(helper, thread_tracking_strategy="message_headers") + email = make_email(in_reply_to="", references=["", ""]) + assert c._resolve_thread_id(email, FakeClient()) == "" + + def test_message_headers_message_id(self, helper): + c = build(helper, thread_tracking_strategy="message_headers") + email = make_email(in_reply_to="", references=[], message_id="") + assert c._resolve_thread_id(email, FakeClient()) == "" + + def test_subject_matching(self, helper): + c = build(helper, thread_tracking_strategy="subject_matching") + email = make_email(subject="RE: Security Alert") + assert c._resolve_thread_id(email, FakeClient()) == "subject:Security Alert" + + +# --------------------------------------------------------------------------- # +# Resolution helpers +# --------------------------------------------------------------------------- # +class TestResolvers: + def test_label_ids_create_and_cache(self, conn, helper): + assert conn._resolve_label_ids(["alpha"]) == ["label-1"] + helper.api.label.create.reset_mock() + assert conn._resolve_label_ids(["alpha"]) == ["label-1"] # cache hit + helper.api.label.create.assert_not_called() + + def test_label_ids_skip_on_error(self, conn, helper): + helper.api.label.create.side_effect = RuntimeError("x") + assert conn._resolve_label_ids(["alpha"]) == [] + + def test_identity_id_create_and_cache(self, conn, helper): + assert conn._resolve_identity_id("Vendor") == "identity-1" + helper.api.identity.create.reset_mock() + assert conn._resolve_identity_id("Vendor") == "identity-1" + helper.api.identity.create.assert_not_called() + + def test_identity_id_none_on_error(self, conn, helper): + helper.api.identity.create.side_effect = RuntimeError("x") + assert conn._resolve_identity_id("Vendor") is None + + def test_marking_id_found_and_cache(self, conn, helper): + assert conn._resolve_marking_id("TLP:AMBER") == "marking-1" + helper.api.marking_definition.read.reset_mock() + assert conn._resolve_marking_id("TLP:AMBER") == "marking-1" + helper.api.marking_definition.read.assert_not_called() + + def test_marking_id_not_found(self, conn, helper): + helper.api.marking_definition.read.return_value = None + assert conn._resolve_marking_id("TLP:RED") is None + + def test_marking_id_none_on_error(self, conn, helper): + helper.api.marking_definition.read.side_effect = RuntimeError("x") + assert conn._resolve_marking_id("TLP:RED") is None + + def test_member_ids_resolved_by_email(self, conn, helper): + helper.api.query.return_value = { + "data": { + "users": { + "edges": [{"node": {"id": "u1", "user_email": "SOC@company.com"}}] + } + } + } + assert conn._resolve_member_ids(["soc@company.com"]) == ["u1"] + + def test_member_ids_not_found(self, conn, helper): + helper.api.query.return_value = {"data": {"users": {"edges": []}}} + assert conn._resolve_member_ids(["ghost@x.com"]) == [] + + def test_member_ids_error(self, conn, helper): + helper.api.query.side_effect = RuntimeError("x") + assert conn._resolve_member_ids(["soc@company.com"]) == [] + + +# --------------------------------------------------------------------------- # +# Rule matching +# --------------------------------------------------------------------------- # +class TestRuleMatching: + def test_subject_rules_all_match_types(self, helper): + rules = [ + {"match_type": "exact", "value": "Security Alert", "labels": ["exact"]}, + {"match_type": "contains", "value": "alert", "labels": ["contains"]}, + {"match_type": "starts_with", "value": "Sec", "labels": ["starts"]}, + {"match_type": "regex", "value": r"Alert$", "labels": ["regex"]}, + ] + c = build(helper, subject_rules=rules) + out = c._match_subject_rules("Security Alert") + assert set(out["labels"]) == {"exact", "contains", "starts", "regex"} + + def test_subject_rules_merge_first_wins(self, helper): + rules = [ + { + "match_type": "contains", + "value": "alert", + "severity": "high", + "priority": "P1", + "case_template": "T1", + }, + {"match_type": "contains", "value": "security", "severity": "low"}, + ] + c = build(helper, subject_rules=rules) + out = c._match_subject_rules("Security Alert") + assert out["severity"] == "high" + assert out["priority"] == "P1" + assert out["case_template"] == "T1" + + def test_subject_rules_bad_regex_ignored(self, helper): + rules = [{"match_type": "regex", "value": "(", "labels": ["x"]}] + c = build(helper, subject_rules=rules) + assert c._match_subject_rules("anything")["labels"] == [] + + def test_sender_rules_match_and_merge(self, helper): + rules = [ + { + "sender": "alerts@example.com", + "author": "Vendor", + "marking": "TLP:GREEN", + "assignees": ["a@x"], + "participants": ["p@x"], + }, + ] + c = build(helper, sender_rules=rules) + out = c._match_sender_rules("ALERTS@example.com") + assert out["author"] == "Vendor" + assert out["marking"] == "TLP:GREEN" + assert out["assignees"] == ["a@x"] + + def test_sender_rules_no_match(self, helper): + c = build(helper, sender_rules=[{"sender": "other@x"}]) + out = c._match_sender_rules("alerts@example.com") + assert out["author"] is None + + +# --------------------------------------------------------------------------- # +# Case templates +# --------------------------------------------------------------------------- # +class TestCaseTemplates: + def test_find_template_id(self, conn, helper): + helper.api.query.return_value = { + "data": { + "caseTemplates": {"edges": [{"node": {"id": "tpl-1", "name": "IR"}}]} + } + } + assert conn._find_case_template_id("IR") == "tpl-1" + + def test_apply_template_missing_is_noop(self, conn, helper): + helper.api.query.return_value = {"data": {"caseTemplates": {"edges": []}}} + conn._apply_case_template("case-1", "IR") # should not raise + + def test_apply_template_runs_mutation(self, conn, helper): + helper.api.query.return_value = { + "data": { + "caseTemplates": {"edges": [{"node": {"id": "tpl-1", "name": "IR"}}]} + } + } + conn._apply_case_template("case-1", "IR") + assert helper.api.query.call_count >= 2 # find + mutation + + +# --------------------------------------------------------------------------- # +# Vocabulary bootstrap +# --------------------------------------------------------------------------- # +class TestVocabularies: + def test_creates_missing_values(self, conn, helper): + helper.api.query.return_value = {"data": {"vocabularies": {"edges": []}}} + conn._ensure_vocabularies() + # severity + priority categories each missing one value -> >=2 mutations + assert helper.api.query.call_count >= 2 + + def test_get_vocabulary_values_parses(self, conn, helper): + helper.api.query.return_value = { + "data": { + "vocabularies": { + "edges": [{"node": {"name": "medium"}}, {"node": {"name": "high"}}] + } + } + } + assert conn._get_vocabulary_values("case_severity_ov") == {"medium", "high"} + + def test_get_vocabulary_values_error_returns_empty(self, conn, helper): + helper.api.query.side_effect = RuntimeError("x") + assert conn._get_vocabulary_values("case_severity_ov") == set() + + +# --------------------------------------------------------------------------- # +# _process_email +# --------------------------------------------------------------------------- # +class TestProcessEmail: + def test_new_case_created(self, conn, helper): + thread_map = {} + is_new = conn._process_email(make_email(), thread_map, FakeClient()) + assert is_new is True + assert thread_map["thread-1"] == "case-internal-1" + helper.api.case_incident.create.assert_called_once() + + def test_existing_thread_appends(self, conn, helper): + thread_map = {"thread-1": "case-existing"} + is_new = conn._process_email(make_email(), thread_map, FakeClient()) + assert is_new is False + helper.api.case_incident.create.assert_not_called() + helper.api.stix_domain_object.update_field.assert_called() + + def test_new_thread_finds_existing_case_by_name(self, conn, helper): + helper.api.case_incident.list.return_value = [{"id": "found-1"}] + thread_map = {} + is_new = conn._process_email(make_email(), thread_map, FakeClient()) + assert is_new is True + assert thread_map["thread-1"] == "found-1" + helper.api.case_incident.create.assert_not_called() + + def test_attachment_uploaded(self, conn, helper): + att = EmailAttachment( + filename="report.txt", + content_type="text/plain", + content=b"plain text body", + size=15, + ) + conn._process_email(make_email(attachments=[att]), {}, FakeClient()) + helper.api.stix_domain_object.add_file.assert_called() + names = [ + c.kwargs["file_name"] + for c in helper.api.stix_domain_object.add_file.call_args_list + ] + assert "report.txt" in names + + def test_display_names_disabled(self, helper): + c = build(helper, display_sender_names=False) + email = make_email() + c._process_email(email, {}, FakeClient()) + assert email.sender_display == "" + + +# --------------------------------------------------------------------------- # +# _import_emails +# --------------------------------------------------------------------------- # +class TestImportEmails: + def test_no_matching_emails_marks_work_processed(self, conn, helper, monkeypatch): + monkeypatch.setattr( + connector_mod, "create_email_client", lambda cfg: FakeClient([]) + ) + conn._import_emails() + helper.api.work.to_processed.assert_called_with( + "work-1", "No new matching emails" + ) + helper.set_state.assert_called() + + def test_processes_new_email_and_sets_state(self, conn, helper, monkeypatch): + monkeypatch.setattr( + connector_mod, "create_email_client", lambda cfg: FakeClient([make_email()]) + ) + conn._import_emails() + helper.api.case_incident.create.assert_called_once() + state = helper.set_state.call_args.args[0] + assert "" in state["processed_message_ids"] + assert state["thread_map"]["thread-1"] == "case-internal-1" + + def test_dedup_skips_processed_ids(self, conn, helper, monkeypatch): + helper.get_state.return_value = { + "processed_message_ids": [""], + "thread_map": {}, + } + monkeypatch.setattr( + connector_mod, "create_email_client", lambda cfg: FakeClient([make_email()]) + ) + conn._import_emails() + helper.api.case_incident.create.assert_not_called() + + def test_off_sender_email_is_dropped(self, conn, helper, monkeypatch): + # A fuzzy protocol-level filter can surface a message from a different + # sender; the strict sender check must drop it before subject filtering + # so it is never turned into a case. + on_sender = make_email(message_id="", thread_id="t-on") + off_sender = make_email( + message_id="", + sender="attacker@evil.com", + thread_id="t-off", + ) + monkeypatch.setattr( + connector_mod, + "create_email_client", + lambda cfg: FakeClient([on_sender, off_sender]), + ) + conn._import_emails() + assert helper.api.case_incident.create.call_count == 1 + state = helper.set_state.call_args.args[0] + assert "" in state["processed_message_ids"] + assert "" not in state["processed_message_ids"] + + def test_uses_last_run_as_since(self, conn, helper, monkeypatch): + captured = {} + + class CapturingClient(FakeClient): + def fetch_emails(self, sender, since=None, max_results=50): + captured["since"] = since + return [] + + helper.get_state.return_value = {"last_run": "2026-04-01T00:00:00Z"} + monkeypatch.setattr( + connector_mod, "create_email_client", lambda cfg: CapturingClient() + ) + conn._import_emails() + assert captured["since"].year == 2026 + + def test_failure_marks_work_in_error_and_raises(self, conn, helper, monkeypatch): + def boom(cfg): + raise RuntimeError("fetch exploded") + + monkeypatch.setattr(connector_mod, "create_email_client", boom) + with pytest.raises(RuntimeError, match="fetch exploded"): + conn._import_emails() + helper.api.work.to_processed.assert_called_with( + "work-1", "fetch exploded", in_error=True + ) + + def test_total_processing_failure_errors_work_and_preserves_state( + self, conn, helper, monkeypatch + ): + # A matching email is fetched, but every attempt to process it fails. + monkeypatch.setattr( + connector_mod, "create_email_client", lambda cfg: FakeClient([make_email()]) + ) + monkeypatch.setattr( + conn, "_process_email", MagicMock(side_effect=RuntimeError("process boom")) + ) + with pytest.raises(RuntimeError): + conn._import_emails() + # Work is marked in_error rather than reported as a successful cycle... + assert helper.api.work.to_processed.call_args.kwargs.get("in_error") is True + # ...and the state watermark is NOT advanced, so the email is retried. + helper.set_state.assert_not_called() + + +# --------------------------------------------------------------------------- # +# Connector author identity (default createdBy) +# --------------------------------------------------------------------------- # +class TestConnectorAuthor: + def test_ensure_connector_author_resolves_internal_id(self, conn, helper): + helper.api.identity.create.return_value = {"id": "author-internal-1"} + conn._ensure_connector_author() + # A real OpenCTI internal id is cached, not the converter's STIX id. + assert conn._connector_author_id == "author-internal-1" + assert not conn._connector_author_id.startswith("identity--") + + def test_create_case_uses_connector_author_fallback(self, conn, helper): + # With no sender-rule author, the case is authored by the connector's + # created internal identity (not the never-created STIX id). + conn._connector_author_id = "author-internal-1" + conn._create_case(make_email(), "Security Alert", "c") + kwargs = helper.api.case_incident.create.call_args.kwargs + assert kwargs["createdBy"] == "author-internal-1" + + +# --------------------------------------------------------------------------- # +# Timeout wrappers + run() +# --------------------------------------------------------------------------- # +class TestTimeoutsAndRun: + def test_fetch_with_timeout_returns_emails(self, conn): + emails = conn._fetch_with_timeout(FakeClient([make_email()]), None) + assert len(emails) == 1 + + def test_fetch_with_timeout_propagates_error(self, conn): + class BoomClient(FakeClient): + def fetch_emails(self, *a, **k): + raise ValueError("nope") + + with pytest.raises(ValueError, match="nope"): + conn._fetch_with_timeout(BoomClient(), None) + + def test_test_connection_propagates_error(self, conn, helper, monkeypatch): + class BoomClient(FakeClient): + def connect(self): + raise ConnectionError("refused") + + monkeypatch.setattr( + connector_mod, "create_email_client", lambda cfg: BoomClient() + ) + with pytest.raises(ConnectionError, match="refused"): + conn._test_connection_with_timeout() + + def test_run_exits_on_connection_failure(self, conn, monkeypatch): + monkeypatch.setattr( + conn, + "_test_connection_with_timeout", + MagicMock(side_effect=RuntimeError("down")), + ) + with pytest.raises(SystemExit): + conn.run() + + def test_run_loops_then_stops(self, conn, monkeypatch): + monkeypatch.setattr(conn, "_test_connection_with_timeout", MagicMock()) + monkeypatch.setattr(conn, "_ensure_vocabularies", MagicMock()) + monkeypatch.setattr( + conn, "_import_emails", MagicMock(side_effect=KeyboardInterrupt()) + ) + monkeypatch.setattr(connector_mod.time, "sleep", MagicMock()) + conn.run() # KeyboardInterrupt breaks the loop cleanly + conn._import_emails.assert_called_once() diff --git a/external-import/email-cases-importer/tests/test_converter_to_stix.py b/external-import/email-cases-importer/tests/test_converter_to_stix.py new file mode 100644 index 00000000000..24f17b81bb1 --- /dev/null +++ b/external-import/email-cases-importer/tests/test_converter_to_stix.py @@ -0,0 +1,158 @@ +"""Unit tests for connector.converter_to_stix. + +Note: ConverterToStix needs a `helper` and `config` only for storage on the +instance — it doesn't call them during the methods we test, so a SimpleNamespace +mock is sufficient. +""" + +from datetime import datetime, timezone +from types import SimpleNamespace + +import pycti +import pytest + +from connector.converter_to_stix import ConverterToStix +from email_client.base import EmailMessage + + +@pytest.fixture +def converter(): + helper = SimpleNamespace() + config = SimpleNamespace() + return ConverterToStix(helper=helper, config=config) + + +@pytest.fixture +def sample_email(): + return EmailMessage( + message_id="", + subject="Security Alert", + sender="alerts@example.com", + recipients=["soc@company.com"], + date=datetime(2026, 4, 8, 12, 30, 0, tzinfo=timezone.utc), + body_plain="body", + body_html="

body

", + thread_id="thread-1", + sender_display="Alerts ", + recipients_display=["SOC "], + ) + + +class TestIdentity: + def test_identity_id_is_deterministic(self, converter): + # Stable across instantiations: pycti.Identity.generate_id is deterministic + c2 = ConverterToStix(helper=SimpleNamespace(), config=SimpleNamespace()) + assert converter.identity_id == c2.identity_id + + def test_identity_id_format(self, converter): + assert converter.identity_id.startswith("identity--") + + def test_identity_id_uses_pycti_generator(self, converter): + # Must be produced by the pycti generator (not a hand-rolled uuid5), so the + # platform and other connectors converge on the same STIX id for this entity. + expected = pycti.Identity.generate_id( + name="Email Cases Importer", identity_class="system" + ) + assert converter.identity_id == expected + + def test_connector_identity_object(self, converter): + ident = converter.connector_identity + assert ident.name == "Email Cases Importer" + assert ident.identity_class == "system" + + +class TestFormatCaseDescription: + def test_uses_display_name_when_available(self, converter, sample_email): + out = converter.format_case_description(sample_email) + assert "Alerts " in out + assert "Security Alert" in out + assert "2026-04-08 12:30:00 UTC" in out + + def test_falls_back_to_sender_when_no_display(self, converter, sample_email): + sample_email.sender_display = "" + out = converter.format_case_description(sample_email) + assert "alerts@example.com" in out + + +class TestFormatEmailContentBlock: + def test_basic_block(self, converter, sample_email): + html = converter.format_email_content_block( + sample_email, body_text="hello", attachment_names=[] + ) + assert "

Security Alert

" in html + assert "Original" in html + assert "
hello
" in html + + def test_reply_label(self, converter, sample_email): + html = converter.format_email_content_block( + sample_email, body_text="reply body", attachment_names=[], is_reply=True + ) + assert "Reply" in html + assert "Original" not in html + + def test_passwords_found_block_appears(self, converter, sample_email): + html = converter.format_email_content_block( + sample_email, + body_text="b", + attachment_names=[], + passwords_found=2, + ) + assert "2 password(s) extracted" in html + + def test_attachments_listed(self, converter, sample_email): + html = converter.format_email_content_block( + sample_email, + body_text="b", + attachment_names=["file1.zip", "file2.pdf"], + ) + assert "file1.zip" in html + assert "file2.pdf" in html + assert "Attachments:" in html + + def test_html_escaping_of_subject(self, converter, sample_email): + sample_email.subject = "" + html = converter.format_email_content_block( + sample_email, body_text="b", attachment_names=[] + ) + # Raw " not in html + assert "<script>" in html + + def test_html_escaping_of_body(self, converter, sample_email): + html = converter.format_email_content_block( + sample_email, + body_text="bold & evil", + attachment_names=[], + ) + assert "bold" not in html + assert "<b>bold</b>" in html + assert "& evil" in html + + def test_newlines_in_body_become_br(self, converter, sample_email): + html = converter.format_email_content_block( + sample_email, body_text="line1\nline2", attachment_names=[] + ) + assert "line1
line2" in html + + def test_no_recipients_shows_placeholder(self, converter, sample_email): + sample_email.recipients = [] + sample_email.recipients_display = [] + html = converter.format_email_content_block( + sample_email, body_text="b", attachment_names=[] + ) + # ASCII placeholder (avoids UTF-8 double-encoding of a literal em dash + # on the OpenCTI content write path) + assert "(none)" in html + assert "—" not in html + + def test_content_is_ascii_safe_for_unicode(self, converter, sample_email): + # Unicode in subject/body must be emitted as numeric character + # references so the OpenCTI content field (which double-encodes raw + # non-ASCII on write) cannot corrupt it. + sample_email.subject = "Alert café 北京" + html = converter.format_email_content_block( + sample_email, body_text="body café 北京", attachment_names=[] + ) + assert html.isascii() # pure ASCII on the wire + assert "é" in html # é -> numeric character reference + assert "café" not in html # no raw non-ASCII left in the payload diff --git a/external-import/email-cases-importer/tests/test_email_client_factory.py b/external-import/email-cases-importer/tests/test_email_client_factory.py new file mode 100644 index 00000000000..dd4748cdf9d --- /dev/null +++ b/external-import/email-cases-importer/tests/test_email_client_factory.py @@ -0,0 +1,123 @@ +"""Unit tests for email_client.factory. + +Only the unsupported-protocol path is exercised directly to avoid pulling in +optional protocol libraries (msal, exchangelib, google-auth) that may not be +present in the unit test environment. Routing for the supported protocols is +verified by mocking the imports. +""" + +from types import SimpleNamespace +from unittest.mock import patch + +import pytest + +from email_client.factory import create_email_client + + +def _cfg(**email_cases_overrides): + """Build a fake config object exposing .email_cases like ConnectorSettings does.""" + defaults = { + "protocol": "imap", + "imap_host": "h", + "imap_port": 993, + "imap_username": "u", + "imap_password": "p", + "imap_folder": "INBOX", + "imap_use_ssl": True, + "graph_tenant_id": "", + "graph_client_id": "", + "graph_client_secret": "", + "graph_user_id": "", + "gmail_credentials_file": "", + "gmail_user_id": "me", + "ews_server": "", + "ews_username": "", + "ews_password": "", + "ews_auth_type": "NTLM", + "tls_verify": True, + } + defaults.update(email_cases_overrides) + return SimpleNamespace(email_cases=SimpleNamespace(**defaults)) + + +class TestUnsupportedProtocol: + def test_unsupported_raises(self): + # Bypass Pydantic validation by passing a plain SimpleNamespace + cfg = _cfg(protocol="pop3") + with pytest.raises(ValueError, match="Unsupported email protocol"): + create_email_client(cfg) + + +class TestProtocolRouting: + """Routing tests need the optional protocol libraries installed because the + factory triggers a real `from email_client._client import ...` + inside the function body. Each test uses `pytest.importorskip` to skip + cleanly when the corresponding library isn't available.""" + + def test_imap_routing(self): + # imaplib is stdlib, always available + import email_client.imap_client # noqa: F401 pylint: disable=import-outside-toplevel,unused-import + + cfg = _cfg(protocol="imap") + with patch("email_client.imap_client.ImapClient") as mock_cls: + mock_cls.return_value = "imap-instance" + assert create_email_client(cfg) == "imap-instance" + kwargs = mock_cls.call_args.kwargs + assert kwargs["host"] == "h" + assert kwargs["port"] == 993 + assert kwargs["username"] == "u" + assert kwargs["use_ssl"] is True + + def test_microsoft_graph_routing(self): + pytest.importorskip("msal") + import email_client.graph_client # noqa: F401 pylint: disable=import-outside-toplevel,unused-import + + cfg = _cfg( + protocol="microsoft_graph", + graph_tenant_id="t", + graph_client_id="c", + graph_client_secret="s", + graph_user_id="user@example.com", + ) + with patch("email_client.graph_client.GraphClient") as mock_cls: + mock_cls.return_value = "graph-instance" + assert create_email_client(cfg) == "graph-instance" + kwargs = mock_cls.call_args.kwargs + assert kwargs["tenant_id"] == "t" + assert kwargs["client_id"] == "c" + assert kwargs["client_secret"] == "s" + assert kwargs["user_id"] == "user@example.com" + + def test_gmail_routing(self): + pytest.importorskip("google.auth") + import email_client.gmail_client # noqa: F401 pylint: disable=import-outside-toplevel,unused-import + + cfg = _cfg( + protocol="gmail", + gmail_credentials_file="/run/secrets/gmail.json", + gmail_user_id="me", + ) + with patch("email_client.gmail_client.GmailClient") as mock_cls: + mock_cls.return_value = "gmail-instance" + assert create_email_client(cfg) == "gmail-instance" + kwargs = mock_cls.call_args.kwargs + assert kwargs["credentials_file"] == "/run/secrets/gmail.json" + assert kwargs["user_id"] == "me" + + def test_ews_routing(self): + pytest.importorskip("exchangelib") + import email_client.ews_client # noqa: F401 pylint: disable=import-outside-toplevel,unused-import + + cfg = _cfg( + protocol="ews", + ews_server="https://exchange.local/EWS/Exchange.asmx", + ews_username="DOMAIN\\u", + ews_password="p", + ews_auth_type="NTLM", + ) + with patch("email_client.ews_client.EwsClient") as mock_cls: + mock_cls.return_value = "ews-instance" + assert create_email_client(cfg) == "ews-instance" + kwargs = mock_cls.call_args.kwargs + assert kwargs["server"] == "https://exchange.local/EWS/Exchange.asmx" + assert kwargs["auth_type"] == "NTLM" diff --git a/external-import/email-cases-importer/tests/test_ews_client.py b/external-import/email-cases-importer/tests/test_ews_client.py new file mode 100644 index 00000000000..19386be7ef3 --- /dev/null +++ b/external-import/email-cases-importer/tests/test_ews_client.py @@ -0,0 +1,174 @@ +"""Unit tests for email_client.ews_client.EwsClient parsing + lifecycle. + +Filter-construction is covered in test_ews_filter.py. Here we exercise +_convert_item (item -> EmailMessage), connect()/disconnect() with exchangelib +patched, and get_thread_id. +""" + +from datetime import datetime, timezone +from types import SimpleNamespace +from unittest.mock import MagicMock, patch + +import pytest + +pytest.importorskip("exchangelib") + +from email_client.base import EmailMessage # noqa: E402 +from email_client.ews_client import EwsClient # noqa: E402 + + +def test_oauth2_auth_type_raises_not_implemented(): + # OAuth2 is accepted by config but not implemented — connect() must fail + # fast with a clear error rather than silently using the default flow. + client = EwsClient( + server="https://ews.example.com", + username="user@example.com", + password="secret", + auth_type="OAuth2", + ) + with pytest.raises(NotImplementedError): + client.connect() + + +def _mailbox(email_address, name=""): + return SimpleNamespace(email_address=email_address, name=name) + + +class _Body: + def __init__(self, text, body_type="Text"): + self._text = text + self.body_type = body_type + + def __str__(self): + return self._text + + +def _item(**overrides): + base = dict( + message_id="", + subject="Security Alert", + sender=_mailbox("alerts@x.com", "Alerts"), + to_recipients=[_mailbox("soc@company.com", "SOC")], + cc_recipients=[_mailbox("cc@company.com")], + datetime_received=datetime(2026, 4, 8, 12, 30, tzinfo=timezone.utc), + body=_Body("plain body"), + in_reply_to="", + conversation_id=SimpleNamespace(id="conv-1"), + has_attachments=False, + attachments=[], + ) + base.update(overrides) + return SimpleNamespace(**base) + + +def _client(): + return EwsClient(server="https://x/EWS", username="u@x.com", password="p") + + +class TestConvertItem: + def test_basic_fields(self): + msg = _client()._convert_item(_item()) + assert msg.message_id == "" + assert msg.subject == "Security Alert" + assert msg.sender == "alerts@x.com" + assert msg.sender_display == "Alerts " + assert "soc@company.com" in msg.recipients + assert "cc@company.com" in msg.recipients + assert msg.body_plain == "plain body" + assert msg.thread_id == "conv-1" + assert msg.in_reply_to == "" + + def test_html_body(self): + msg = _client()._convert_item(_item(body=_Body("

x

", "HTML"))) + assert msg.body_html + assert msg.body_plain == "" + + def test_naive_datetime_made_aware(self): + naive = datetime(2026, 4, 8, 12, 30) + msg = _client()._convert_item(_item(datetime_received=naive)) + assert msg.date.tzinfo is not None + + def test_missing_date_defaults_now(self): + msg = _client()._convert_item(_item(datetime_received=None)) + assert msg.date is not None + + def test_thread_id_falls_back_to_message_id(self): + msg = _client()._convert_item(_item(conversation_id=None)) + assert msg.thread_id == "" + + def test_sender_none(self): + msg = _client()._convert_item(_item(sender=None)) + assert msg.sender == "" + + +class TestLifecycle: + def test_connect_with_server(self): + c = _client() + with ( + patch("exchangelib.Credentials"), + patch("exchangelib.Configuration"), + patch("exchangelib.Account") as account_cls, + patch("exchangelib.DELEGATE"), + ): + c.connect() + account_cls.assert_called_once() + assert c._account is account_cls.return_value + + def test_connect_autodiscover_without_server(self): + c = EwsClient(server="", username="u@x.com", password="p") + with ( + patch("exchangelib.Credentials"), + patch("exchangelib.Account") as account_cls, + patch("exchangelib.DELEGATE"), + ): + c.connect() + assert account_cls.call_args.kwargs["autodiscover"] is True + + def test_disconnect_closes_protocol(self): + c = _client() + proto = MagicMock() + c._account = SimpleNamespace(protocol=proto) + c.disconnect() + proto.close.assert_called_once() + assert c._account is None + + def test_disconnect_restores_global_adapter_when_tls_disabled(self): + # tls_verify=False swaps the process-global HTTP_ADAPTER_CLS; disconnect + # must restore the prior value rather than leaving TLS verification + # disabled for every exchangelib user in the process. + from exchangelib.protocol import BaseProtocol + + original = BaseProtocol.HTTP_ADAPTER_CLS + c = EwsClient( + server="https://x/EWS", + username="u@x.com", + password="p", + tls_verify=False, + ) + with ( + patch("exchangelib.Credentials"), + patch("exchangelib.Configuration"), + patch("exchangelib.Account"), + patch("exchangelib.DELEGATE"), + ): + c.connect() + assert BaseProtocol.HTTP_ADAPTER_CLS is not original + c.disconnect() + assert BaseProtocol.HTTP_ADAPTER_CLS is original + + def test_fetch_raises_when_not_connected(self): + with pytest.raises(RuntimeError): + _client().fetch_emails(sender="a@b.com") + + def test_get_thread_id(self): + m = EmailMessage( + message_id="x", + subject="s", + sender="a@b", + recipients=[], + date=None, + body_plain="", + body_html="", + thread_id="conv-z", + ) + assert _client().get_thread_id(m) == "conv-z" diff --git a/external-import/email-cases-importer/tests/test_ews_filter.py b/external-import/email-cases-importer/tests/test_ews_filter.py new file mode 100644 index 00000000000..6d422010ce0 --- /dev/null +++ b/external-import/email-cases-importer/tests/test_ews_filter.py @@ -0,0 +1,87 @@ +"""Regression test: EWS client must use a valid exchangelib filter. + +`Message.sender` is a single-valued Mailbox field — exchangelib does NOT accept +nested lookups like `sender__email_address`, and the older `from_emailaddresses` +path was never valid either. Both raise InvalidField at query time. The +supported form is `sender=Mailbox(email_address=...)`. +""" + +from datetime import datetime, timezone +from types import SimpleNamespace +from unittest.mock import MagicMock + +import pytest + +pytest.importorskip("exchangelib") + +from exchangelib import Mailbox # noqa: E402 + +from email_client.ews_client import EwsClient # noqa: E402 + + +def _make_client_with_fake_inbox(): + """Build an EwsClient bypassing real connect(); wire up a mock inbox that + records the filter kwargs it receives.""" + client = EwsClient( + server="", + username="u@example.com", + password="p", + auth_type="NTLM", + ) + + # Chainable mock: qs = inbox.filter(...).filter(...).order_by(...)[:n] + qs = MagicMock() + qs.filter.return_value = qs + qs.order_by.return_value = qs + qs.__getitem__.return_value = [] # slice [:max_results] -> empty list + + inbox = MagicMock() + inbox.filter.return_value = qs + + client._account = SimpleNamespace(inbox=inbox) + return client, inbox, qs + + +def test_sender_filter_uses_mailbox_object(): + client, inbox, _qs = _make_client_with_fake_inbox() + client.fetch_emails(sender="alerts@vendor.com", max_results=5) + + # Must be filter(sender=Mailbox(email_address="alerts@vendor.com")) + inbox.filter.assert_called_once() + kwargs = inbox.filter.call_args.kwargs + assert ( + "sender" in kwargs + ), f"expected filter kwarg 'sender', got keys: {list(kwargs.keys())}" + passed = kwargs["sender"] + assert isinstance( + passed, Mailbox + ), f"expected Mailbox instance, got {type(passed).__name__}" + assert passed.email_address == "alerts@vendor.com" + + # Regression: neither of the previously-broken field paths may appear + assert "from_emailaddresses" not in kwargs + assert "from_emailaddresses__contains" not in kwargs + assert "sender__email_address" not in kwargs + + +def test_since_filter_passed_as_tz_aware_datetime(): + client, _inbox, qs = _make_client_with_fake_inbox() + since = datetime(2026, 4, 1, 12, 0, 0, tzinfo=timezone.utc) + + client.fetch_emails(sender="a@b.com", since=since, max_results=5) + + # qs.filter should have been called once for the since clause + qs.filter.assert_called_once() + kwargs = qs.filter.call_args.kwargs + assert "datetime_received__gte" in kwargs + passed = kwargs["datetime_received__gte"] + # Must be a tz-aware datetime (UTC) + assert passed.tzinfo is not None + assert passed.utcoffset().total_seconds() == 0 + + +def test_no_since_means_no_second_filter_call(): + client, _inbox, qs = _make_client_with_fake_inbox() + client.fetch_emails(sender="a@b.com", max_results=5) + # Only the sender filter — no since filter applied + qs.filter.assert_not_called() diff --git a/external-import/email-cases-importer/tests/test_gmail_client.py b/external-import/email-cases-importer/tests/test_gmail_client.py new file mode 100644 index 00000000000..f6774bcaf62 --- /dev/null +++ b/external-import/email-cases-importer/tests/test_gmail_client.py @@ -0,0 +1,296 @@ +"""Unit tests for email_client.gmail_client.GmailClient. + +The google-auth dependency is stubbed in conftest; connect() is tested by +patching the module-level `service_account`/`Request`, and the fetch/parse paths +run against a MagicMock session with canned Gmail API payloads. +""" + +import base64 +from unittest.mock import MagicMock, patch + +import pytest + +import email_client.gmail_client as gmail_mod +from email_client.gmail_client import GmailClient + + +def _b64(s: str) -> str: + return base64.urlsafe_b64encode(s.encode()).decode() + + +def test_b64url_tolerates_missing_padding(): + # Gmail base64url payloads frequently omit '=' padding; _b64url must decode + # them without raising binascii.Error. + raw = b"hello world payload with length forcing padding" + unpadded = base64.urlsafe_b64encode(raw).decode().rstrip("=") + assert gmail_mod._b64url(unpadded) == raw + + +def _resp(json_data): + r = MagicMock() + r.json.return_value = json_data + r.raise_for_status.return_value = None + return r + + +def _client(user_id="me"): + return GmailClient(credentials_file="/creds.json", user_id=user_id) + + +def _full_message(message_id="m1", thread_id="t1", with_attachment=False): + payload = { + "mimeType": "multipart/mixed", + "headers": [ + {"name": "Subject", "value": "Security Alert"}, + {"name": "From", "value": "Alerts "}, + {"name": "To", "value": "SOC , plain@company.com"}, + {"name": "Date", "value": "Wed, 08 Apr 2026 12:30:00 +0000"}, + {"name": "Message-ID", "value": ""}, + {"name": "In-Reply-To", "value": ""}, + {"name": "References", "value": " "}, + ], + "parts": [ + {"mimeType": "text/plain", "body": {"data": _b64("plain body")}}, + {"mimeType": "text/html", "body": {"data": _b64("

rich

")}}, + ], + } + if with_attachment: + payload["parts"].append( + { + "filename": "report.txt", + "mimeType": "text/plain", + "body": {"attachmentId": "att-1"}, + } + ) + return { + "id": message_id, + "threadId": thread_id, + "payload": payload, + "internalDate": "1000", + } + + +class TestConnect: + def test_connect_default_user(self): + c = _client(user_id="me") + creds = MagicMock() + creds.token = "TOK" + with ( + patch.object(gmail_mod, "service_account") as sa, + patch.object(gmail_mod, "Request"), + patch("email_client.gmail_client.requests.Session") as sess_cls, + ): + sa.Credentials.from_service_account_file.return_value = creds + sess = sess_cls.return_value + sess.headers = {} + c.connect() + creds.with_subject.assert_not_called() + creds.refresh.assert_called_once() + assert sess.headers["Authorization"] == "Bearer TOK" + + def test_connect_delegated_user(self): + c = _client(user_id="boss@x.com") + creds = MagicMock() + creds.token = "TOK" + creds.with_subject.return_value = creds + with ( + patch.object(gmail_mod, "service_account") as sa, + patch.object(gmail_mod, "Request"), + patch("email_client.gmail_client.requests.Session") as sess_cls, + ): + sa.Credentials.from_service_account_file.return_value = creds + sess_cls.return_value.headers = {} + c.connect() + creds.with_subject.assert_called_once_with("boss@x.com") + + def test_disconnect(self): + c = _client() + sess = MagicMock() + c._session = sess + c.disconnect() + sess.close.assert_called_once() + assert c._session is None + + def test_refresh_if_needed(self): + c = _client() + creds = MagicMock() + creds.expired = True + creds.token = "NEW" + c._credentials = creds + c._session = MagicMock() + c._session.headers = {} + with patch.object(gmail_mod, "Request"): + c._refresh_if_needed() + assert c._session.headers["Authorization"] == "Bearer NEW" + + +class TestFetchEmails: + def test_raises_when_not_connected(self): + with pytest.raises(RuntimeError): + _client().fetch_emails(sender="a@b.com") + + def test_lists_then_fetches_full(self): + c = _client() + sess = MagicMock() + list_resp = _resp({"messages": [{"id": "m1"}]}) + full_resp = _resp(_full_message()) + sess.get.side_effect = [list_resp, full_resp] + c._session = sess + c._credentials = MagicMock(expired=False) + out = c.fetch_emails(sender="alerts@x.com") + assert len(out) == 1 + m = out[0] + assert m.subject == "Security Alert" + assert m.sender == "alerts@x.com" + assert m.body_plain == "plain body" + assert m.body_html == "

rich

" + assert m.in_reply_to == "" + assert m.references == ["", ""] + assert m.thread_id == "t1" + assert "soc@company.com" in m.recipients + assert "plain@company.com" in m.recipients + + def test_since_adds_after_clause(self): + from datetime import datetime, timezone + + c = _client() + sess = MagicMock() + sess.get.return_value = _resp({"messages": []}) + c._session = sess + c._credentials = MagicMock(expired=False) + c.fetch_emails( + sender="a@b.com", since=datetime(2026, 4, 1, tzinfo=timezone.utc) + ) + params = sess.get.call_args.kwargs["params"] + assert "after:" in params["q"] + + def test_empty_list(self): + c = _client() + sess = MagicMock() + sess.get.return_value = _resp({}) + c._session = sess + c._credentials = MagicMock(expired=False) + assert c.fetch_emails(sender="a@b.com") == [] + + def test_paginates_across_pages(self): + # A nextPageToken must trigger a follow-up list call carrying that + # token, so max_results spanning more than one page is honoured instead + # of silently truncating to the first 100 results. + c = _client() + sess = MagicMock() + list1 = _resp({"messages": [{"id": "m1"}], "nextPageToken": "PAGE2"}) + list2 = _resp({"messages": [{"id": "m2"}]}) + full1 = _resp(_full_message("m1", "t1")) + full2 = _resp(_full_message("m2", "t2")) + sess.get.side_effect = [list1, list2, full1, full2] + c._session = sess + c._credentials = MagicMock(expired=False) + out = c.fetch_emails(sender="alerts@x.com") + assert len(out) == 2 + assert sess.get.call_args_list[1].kwargs["params"]["pageToken"] == "PAGE2" + + +class TestAttachmentsAndBody: + def test_attachment_downloaded(self): + c = _client() + sess = MagicMock() + list_resp = _resp({"messages": [{"id": "m1"}]}) + full_resp = _resp(_full_message(with_attachment=True)) + download_resp = _resp({"data": _b64("file-data")}) + sess.get.side_effect = [list_resp, full_resp, download_resp] + c._session = sess + c._credentials = MagicMock(expired=False) + out = c.fetch_emails(sender="a@b.com") + assert len(out[0].attachments) == 1 + assert out[0].attachments[0].filename == "report.txt" + assert out[0].attachments[0].content == b"file-data" + + def test_attachment_download_tolerates_unpadded_b64url(self): + # Gmail attachment payloads are base64url and may omit '=' padding; the + # download path must decode them without raising binascii.Error. + c = _client() + sess = MagicMock() + raw = b"encrypted-evidence-bytes!" + unpadded = base64.urlsafe_b64encode(raw).decode().rstrip("=") + list_resp = _resp({"messages": [{"id": "m1"}]}) + full_resp = _resp(_full_message(with_attachment=True)) + download_resp = _resp({"data": unpadded}) + sess.get.side_effect = [list_resp, full_resp, download_resp] + c._session = sess + c._credentials = MagicMock(expired=False) + out = c.fetch_emails(sender="a@b.com") + assert out[0].attachments[0].content == raw + + def test_extract_body_direct_plain(self): + c = _client() + plain, html = c._extract_body( + {"mimeType": "text/plain", "body": {"data": _b64("hello")}} + ) + assert plain == "hello" + assert html == "" + + def test_get_thread_id(self): + from email_client.base import EmailMessage + + c = _client() + m = EmailMessage( + message_id="x", + subject="s", + sender="a@b", + recipients=[], + date=None, + body_plain="", + body_html="", + thread_id="t-9", + ) + assert c.get_thread_id(m) == "t-9" + + def test_sender_without_angle_brackets(self): + c = _client() + sess = MagicMock() + msg = _full_message() + for h in msg["payload"]["headers"]: + if h["name"] == "From": + h["value"] = "bare@x.com" + sess.get.side_effect = [_resp({"messages": [{"id": "m1"}]}), _resp(msg)] + c._session = sess + c._credentials = MagicMock(expired=False) + out = c.fetch_emails(sender="bare@x.com") + assert out[0].sender == "bare@x.com" + + +class TestRobustness: + def test_non_json_200_raises_typed_error(self): + from email_client._http import EmailClientHTTPError + + c = _client() + sess = MagicMock() + bad = MagicMock() + bad.status_code = 200 + bad.raise_for_status.return_value = None + bad.json.side_effect = ValueError("no json") + bad.headers = {"Content-Type": "text/html"} + bad.text = "WAF" + sess.get.return_value = bad + c._session = sess + c._credentials = MagicMock(expired=False) + with pytest.raises(EmailClientHTTPError): + c.fetch_emails(sender="a@b.com") + + def test_retries_on_429(self, monkeypatch): + import email_client._http as http_mod + + monkeypatch.setattr(http_mod.time, "sleep", lambda s: None) + c = _client() + sess = MagicMock() + throttled = _resp({}) + throttled.status_code = 429 + throttled.headers = {"Retry-After": "0"} + ok = _resp({"messages": []}) + ok.status_code = 200 + sess.get.side_effect = [throttled, ok] + c._session = sess + c._credentials = MagicMock(expired=False) + out = c.fetch_emails(sender="a@b.com") + assert out == [] + assert sess.get.call_count == 2 diff --git a/external-import/email-cases-importer/tests/test_graph_client.py b/external-import/email-cases-importer/tests/test_graph_client.py new file mode 100644 index 00000000000..c02d49d562e --- /dev/null +++ b/external-import/email-cases-importer/tests/test_graph_client.py @@ -0,0 +1,226 @@ +"""Unit tests for email_client.graph_client.GraphClient. + +The requests.Session is replaced with a MagicMock; responses are canned dicts +mirroring the Microsoft Graph /messages and /attachments payloads. +""" + +import base64 +from unittest.mock import MagicMock, patch + +import pytest + +from email_client.graph_client import GraphClient + + +def _client(): + return GraphClient( + tenant_id="t", client_id="c", client_secret="s", user_id="user@x.com" + ) + + +def _resp(json_data, status=200): + r = MagicMock() + r.status_code = status + r.json.return_value = json_data + r.raise_for_status.return_value = None + return r + + +def _message(message_id="AAA", conversation_id="conv-1", with_attachment=False): + return { + "id": message_id, + "internetMessageId": f"<{message_id}@x>", + "subject": "Security Alert", + "from": {"emailAddress": {"name": "Alerts", "address": "alerts@x.com"}}, + "toRecipients": [ + {"emailAddress": {"name": "SOC", "address": "soc@company.com"}} + ], + "ccRecipients": [{"emailAddress": {"name": "", "address": "cc@company.com"}}], + "receivedDateTime": "2026-04-08T12:30:00Z", + "conversationId": conversation_id, + "body": {"contentType": "html", "content": "

hi

"}, + "internetMessageHeaders": [ + {"name": "In-Reply-To", "value": ""}, + {"name": "References", "value": " "}, + ], + "hasAttachments": with_attachment, + } + + +class TestAuth: + def test_connect_authenticates_and_sets_header(self): + c = _client() + with patch("email_client.graph_client.requests.Session") as sess_cls: + sess = sess_cls.return_value + sess.headers = {} + sess.post.return_value = _resp({"access_token": "TOK"}) + c.connect() + assert sess.headers["Authorization"] == "Bearer TOK" + assert c._session is sess + + +class TestFetchEmails: + def test_raises_when_not_connected(self): + with pytest.raises(RuntimeError): + _client().fetch_emails(sender="a@b.com") + + def test_fetches_and_parses_message(self): + c = _client() + sess = MagicMock() + sess.get.return_value = _resp({"value": [_message()]}) + c._session = sess + out = c.fetch_emails(sender="alerts@x.com") + assert len(out) == 1 + m = out[0] + assert m.subject == "Security Alert" + assert m.sender == "alerts@x.com" + assert m.sender_display == "Alerts " + assert "soc@company.com" in m.recipients + assert "cc@company.com" in m.recipients + assert m.body_html == "

hi

" + assert m.in_reply_to == "" + assert m.references == ["", ""] + assert m.thread_id == "conv-1" + + def test_escapes_single_quotes_in_sender(self): + c = _client() + sess = MagicMock() + sess.get.return_value = _resp({"value": []}) + c._session = sess + c.fetch_emails(sender="o'brien@x.com") + params = sess.get.call_args.kwargs["params"] + assert "o''brien@x.com" in params["$filter"] + + def test_since_adds_received_filter(self): + from datetime import datetime, timezone + + c = _client() + sess = MagicMock() + sess.get.return_value = _resp({"value": []}) + c._session = sess + c.fetch_emails( + sender="a@b.com", + since=datetime(2026, 4, 1, tzinfo=timezone.utc), + ) + params = sess.get.call_args.kwargs["params"] + assert "receivedDateTime ge" in params["$filter"] + + def test_pagination_follows_next_link(self): + c = _client() + sess = MagicMock() + page1 = _resp( + { + "value": [_message(message_id="A")], + "@odata.nextLink": "https://graph/next", + } + ) + page2 = _resp({"value": [_message(message_id="B")]}) + sess.get.side_effect = [page1, page2] + c._session = sess + out = c.fetch_emails(sender="a@b.com", max_results=50) + assert {m.message_id for m in out} == {"", ""} + assert sess.get.call_count == 2 + + def test_reauth_on_401(self): + c = _client() + sess = MagicMock() + sess.headers = {} + unauthorized = _resp({}, status=401) + ok = _resp({"value": []}) + sess.get.side_effect = [unauthorized, ok] + sess.post.return_value = _resp({"access_token": "TOK2"}) + c._session = sess + c.fetch_emails(sender="a@b.com") + sess.post.assert_called_once() # re-authenticated + + +class TestAttachments: + def test_fetch_attachments_filters_file_type(self): + c = _client() + sess = MagicMock() + msg = _message(message_id="A", with_attachment=True) + att_payload = { + "value": [ + { + "@odata.type": "#microsoft.graph.fileAttachment", + "name": "r.txt", + "contentType": "text/plain", + "contentBytes": base64.b64encode(b"data").decode(), + }, + {"@odata.type": "#microsoft.graph.itemAttachment", "name": "skip"}, + ] + } + sess.get.side_effect = [_resp({"value": [msg]}), _resp(att_payload)] + c._session = sess + out = c.fetch_emails(sender="a@b.com") + assert len(out[0].attachments) == 1 + assert out[0].attachments[0].filename == "r.txt" + assert out[0].attachments[0].content == b"data" + + +class TestRobustness: + def test_non_json_200_raises_typed_error(self): + from email_client._http import EmailClientHTTPError + + c = _client() + sess = MagicMock() + bad = MagicMock() + bad.status_code = 200 + bad.raise_for_status.return_value = None + bad.json.side_effect = ValueError("no json") + bad.headers = {"Content-Type": "text/html"} + bad.text = "WAF" + sess.get.return_value = bad + c._session = sess + with pytest.raises(EmailClientHTTPError): + c.fetch_emails(sender="a@b.com") + + def test_retries_on_429(self, monkeypatch): + import email_client._http as http_mod + + monkeypatch.setattr(http_mod.time, "sleep", lambda s: None) + c = _client() + sess = MagicMock() + throttled = _resp({}, status=429) + throttled.headers = {"Retry-After": "0"} + sess.get.side_effect = [throttled, _resp({"value": []})] + c._session = sess + out = c.fetch_emails(sender="a@b.com") + assert out == [] + assert sess.get.call_count == 2 + + +class TestMisc: + def test_disconnect_closes_session(self): + c = _client() + sess = MagicMock() + c._session = sess + c.disconnect() + sess.close.assert_called_once() + assert c._session is None + + def test_get_thread_id(self): + from email_client.base import EmailMessage + + c = _client() + m = EmailMessage( + message_id="x", + subject="s", + sender="a@b", + recipients=[], + date=None, + body_plain="", + body_html="", + thread_id="conv-9", + ) + assert c.get_thread_id(m) == "conv-9" + + def test_missing_received_date_defaults_to_now(self): + c = _client() + sess = MagicMock() + msg = _message() + del msg["receivedDateTime"] + sess.get.return_value = _resp({"value": [msg]}) + c._session = sess + out = c.fetch_emails(sender="a@b.com") + assert out[0].date is not None diff --git a/external-import/email-cases-importer/tests/test_handlers_extra.py b/external-import/email-cases-importer/tests/test_handlers_extra.py new file mode 100644 index 00000000000..6b012608703 --- /dev/null +++ b/external-import/email-cases-importer/tests/test_handlers_extra.py @@ -0,0 +1,178 @@ +"""Coverage for archive_handler, document_handler, and registry recursion. + +ZIP archives are built with the stdlib (plain only — stdlib cannot *write* +encrypted zips). py7zr/msoffcrypto/pikepdf are optional and absent in CI, so the +7z/office/pdf decrypt paths are exercised via their graceful-fallback branches. +""" + +import io +import zipfile + +import pytest + +from attachment_handler.archive_handler import ArchiveHandler +from attachment_handler.document_handler import DocumentHandler +from attachment_handler.registry import HandlerRegistry + + +def _zip_bytes(files: dict) -> bytes: + buf = io.BytesIO() + with zipfile.ZipFile(buf, "w") as zf: + for name, data in files.items(): + zf.writestr(name, data) + return buf.getvalue() + + +# --------------------------------------------------------------------------- # +# ArchiveHandler +# --------------------------------------------------------------------------- # +class TestArchiveHandler: + def test_supported_extensions(self): + assert set(ArchiveHandler().supported_extensions()) == {".zip", ".7z", ".rar"} + + def test_extract_plain_zip(self, tmp_path): + p = tmp_path / "a.zip" + p.write_bytes(_zip_bytes({"f1.txt": b"hello", "f2.txt": b"world"})) + out = ArchiveHandler().extract(str(p)) + names = {e.filename for e in out} + assert names == {"f1.txt", "f2.txt"} + assert all(e.was_encrypted is False for e in out) + assert all(e.hashes for e in out) + + def test_extract_zip_skips_directories(self, tmp_path): + buf = io.BytesIO() + with zipfile.ZipFile(buf, "w") as zf: + zf.writestr("dir/", b"") + zf.writestr("dir/file.txt", b"data") + p = tmp_path / "a.zip" + p.write_bytes(buf.getvalue()) + out = ArchiveHandler().extract(str(p)) + assert [e.filename for e in out] == ["file.txt"] + + def test_bad_zip_returns_empty(self, tmp_path): + p = tmp_path / "a.zip" + p.write_bytes(b"not a real zip") + assert ArchiveHandler().extract(str(p)) == [] + + def test_unknown_extension_returns_empty(self, tmp_path): + p = tmp_path / "a.tar" + p.write_bytes(b"x") + assert ArchiveHandler().extract(str(p)) == [] + + def test_7z_without_py7zr_is_graceful(self, tmp_path): + pytest.importorskip # noqa: B018 (documentation: not skipping) + try: + import py7zr # noqa: F401 + + pytest.skip("py7zr installed — graceful-fallback path not exercised") + except ImportError: + pass + p = tmp_path / "a.7z" + p.write_bytes(b"7z\xbc\xaf\x27\x1c") # 7z magic, not a real archive + assert ArchiveHandler().extract(str(p)) == [] + + def test_rar_without_bsdtar_is_graceful(self, tmp_path, monkeypatch): + import shutil + + # _extract_rar imports shutil locally; patch the source attribute. + monkeypatch.setattr(shutil, "which", lambda _name: None) + p = tmp_path / "a.rar" + p.write_bytes(b"Rar!\x1a\x07\x00") + assert ArchiveHandler().extract(str(p)) == [] + + +# --------------------------------------------------------------------------- # +# DocumentHandler +# --------------------------------------------------------------------------- # +class TestDocumentHandler: + def test_supported_extensions(self): + assert set(DocumentHandler().supported_extensions()) == {".xlsx", ".pdf"} + + def test_xlsx_no_password_passthrough(self, tmp_path): + p = tmp_path / "book.xlsx" + p.write_bytes(b"PK\x03\x04 fake xlsx bytes") + out = DocumentHandler().extract(str(p)) + assert len(out) == 1 + assert out[0].content_type.endswith("spreadsheetml.sheet") + assert out[0].was_encrypted is False + assert out[0].hashes + + def test_pdf_no_password_passthrough(self, tmp_path): + p = tmp_path / "doc.pdf" + p.write_bytes(b"%PDF-1.7 fake") + out = DocumentHandler().extract(str(p)) + assert out[0].content_type == "application/pdf" + assert out[0].was_encrypted is False + + def test_pdf_with_password_but_no_pikepdf_keeps_original(self, tmp_path): + # pikepdf absent -> _decrypt_pdf returns None -> content unchanged + try: + import pikepdf # noqa: F401 + + pytest.skip("pikepdf installed — fallback path not exercised") + except ImportError: + pass + p = tmp_path / "doc.pdf" + p.write_bytes(b"%PDF-1.7 fake") + out = DocumentHandler().extract(str(p), passwords=["secret"]) + assert out[0].was_encrypted is False + + def test_xlsx_with_password_but_no_msoffcrypto_keeps_original(self, tmp_path): + try: + import msoffcrypto # noqa: F401 + + pytest.skip("msoffcrypto installed — fallback path not exercised") + except ImportError: + pass + p = tmp_path / "book.xlsx" + p.write_bytes(b"PK\x03\x04 fake") + out = DocumentHandler().extract(str(p), passwords=["secret"]) + assert out[0].was_encrypted is False + + def test_unknown_extension_returns_empty(self, tmp_path): + p = tmp_path / "x.doc" + p.write_bytes(b"x") + assert DocumentHandler().extract(str(p)) == [] + + +# --------------------------------------------------------------------------- # +# Registry recursion +# --------------------------------------------------------------------------- # +class TestRegistryRecursion: + def setup_method(self): + self.registry = HandlerRegistry() + + def test_archive_with_inner_document(self): + zip_bytes = _zip_bytes({"notes.txt": b"plain text note"}) + out = self.registry.process_attachment("bundle.zip", zip_bytes) + assert len(out) == 1 + inner = out[0] + assert inner.filename == "notes.txt" + # passthrough handler runs on the inner .txt + assert inner.inner_files + assert inner.inner_files[0].filename == "notes.txt" + + def test_nested_archive_recurses(self): + inner_zip = _zip_bytes({"deep.txt": b"deep content"}) + outer_zip = _zip_bytes({"inner.zip": inner_zip}) + out = self.registry.process_attachment("outer.zip", outer_zip) + assert len(out) == 1 + assert out[0].filename == "inner.zip" + # nested archive extracted into inner_files + assert out[0].inner_files + assert out[0].inner_files[0].filename == "deep.txt" + + def test_non_archive_preserves_original_filename(self): + out = self.registry.process_attachment("report.pdf", b"%PDF-1.7 x") + assert out[0].filename == "report.pdf" + + def test_oversized_skipped(self): + big = b"x" * (2 * 1024 * 1024) + out = self.registry.process_attachment("big.zip", big, max_size_mb=1) + assert out[0].metadata.get("skipped") == "exceeds_max_size" + assert out[0].content == b"" + + def test_unknown_extension_returns_raw(self): + out = self.registry.process_attachment("data.bin", b"raw") + assert out[0].content == b"raw" + assert out[0].hashes diff --git a/external-import/email-cases-importer/tests/test_http_helpers.py b/external-import/email-cases-importer/tests/test_http_helpers.py new file mode 100644 index 00000000000..0599393e69c --- /dev/null +++ b/external-import/email-cases-importer/tests/test_http_helpers.py @@ -0,0 +1,89 @@ +"""Unit tests for email_client._http (parse_json + get_with_retry).""" + +from unittest.mock import MagicMock + +import pytest + +import email_client._http as http_mod +from email_client._http import EmailClientHTTPError, get_with_retry, parse_json + + +def _resp(status=200, json_data=None, json_exc=None, headers=None, text=""): + r = MagicMock() + r.status_code = status + r.headers = headers or {} + r.text = text + if json_exc is not None: + r.json.side_effect = json_exc + else: + r.json.return_value = json_data + return r + + +class TestParseJson: + def test_returns_parsed_body(self): + assert parse_json(_resp(json_data={"ok": True})) == {"ok": True} + + def test_non_json_body_raises_typed_error(self): + resp = _resp( + status=200, + json_exc=ValueError("no json"), + headers={"Content-Type": "text/html"}, + text="blocked by proxy", + ) + with pytest.raises(EmailClientHTTPError) as exc: + parse_json(resp) + assert "text/html" in str(exc.value) + assert "blocked by proxy" in str(exc.value) + + +class TestGetWithRetry: + def test_returns_immediately_on_200(self): + sess = MagicMock() + sess.get.return_value = _resp(200) + out = get_with_retry(sess, "http://x") + assert out.status_code == 200 + sess.get.assert_called_once() + + def test_retries_on_429_then_succeeds(self, monkeypatch): + slept = [] + monkeypatch.setattr(http_mod.time, "sleep", lambda s: slept.append(s)) + sess = MagicMock() + sess.get.side_effect = [ + _resp(429, headers={"Retry-After": "1"}), + _resp(200, json_data={}), + ] + out = get_with_retry(sess, "http://x") + assert out.status_code == 200 + assert sess.get.call_count == 2 + assert slept == [1] + + def test_gives_up_after_max_retries(self, monkeypatch): + monkeypatch.setattr(http_mod.time, "sleep", lambda s: None) + sess = MagicMock() + sess.get.return_value = _resp(429, headers={"Retry-After": "0"}) + out = get_with_retry(sess, "http://x", max_retries=2) + assert out.status_code == 429 + assert sess.get.call_count == 3 # initial + 2 retries + + def test_bad_retry_after_uses_default(self, monkeypatch): + slept = [] + monkeypatch.setattr(http_mod.time, "sleep", lambda s: slept.append(s)) + sess = MagicMock() + sess.get.side_effect = [ + _resp(429, headers={"Retry-After": "not-a-number"}), + _resp(200), + ] + get_with_retry(sess, "http://x") + assert slept == [http_mod._DEFAULT_RETRY_WAIT] + + def test_retry_after_capped(self, monkeypatch): + slept = [] + monkeypatch.setattr(http_mod.time, "sleep", lambda s: slept.append(s)) + sess = MagicMock() + sess.get.side_effect = [ + _resp(429, headers={"Retry-After": "9999"}), + _resp(200), + ] + get_with_retry(sess, "http://x") + assert slept == [http_mod._MAX_RETRY_WAIT] diff --git a/external-import/email-cases-importer/tests/test_imap_client.py b/external-import/email-cases-importer/tests/test_imap_client.py new file mode 100644 index 00000000000..9f839dc2855 --- /dev/null +++ b/external-import/email-cases-importer/tests/test_imap_client.py @@ -0,0 +1,204 @@ +"""Unit tests for email_client.imap_client.ImapClient. + +connect()/fetch_emails() are tested against a mocked imaplib connection; the +parsing helpers are tested with real RFC822 bytes built via the stdlib email +package. +""" + +from email.message import EmailMessage as PyEmailMessage +from unittest.mock import MagicMock, patch + +from email_client.base import EmailMessage +from email_client.imap_client import ImapClient + + +def _raw( + subject="Security Alert", + from_="Alerts ", + to="SOC ", + cc=None, + date="Wed, 08 Apr 2026 12:30:00 +0000", + message_id="", + in_reply_to=None, + references=None, + plain="hello body", + html=None, + attachment=None, +): + msg = PyEmailMessage() + msg["Subject"] = subject + msg["From"] = from_ + msg["To"] = to + if cc: + msg["Cc"] = cc + msg["Date"] = date + msg["Message-ID"] = message_id + if in_reply_to: + msg["In-Reply-To"] = in_reply_to + if references: + msg["References"] = references + if plain is not None: + msg.set_content(plain) + if html is not None: + msg.add_alternative(html, subtype="html") + if attachment is not None: + name, data, (maintype, subtype) = attachment + msg.add_attachment(data, maintype=maintype, subtype=subtype, filename=name) + return msg.as_bytes() + + +def _client(): + return ImapClient(host="mail.x", username="u", password="p") + + +class TestConnect: + def test_connect_ssl(self): + c = _client() + with patch("email_client.imap_client.imaplib.IMAP4_SSL") as ssl_cls: + conn = ssl_cls.return_value + c.connect() + conn.login.assert_called_once_with("u", "p") + conn.select.assert_called_once_with("INBOX") + + def test_connect_plain(self): + c = ImapClient(host="mail.x", username="u", password="p", use_ssl=False) + with patch("email_client.imap_client.imaplib.IMAP4") as cls: + c.connect() + cls.assert_called_once_with("mail.x", 993) + + def test_connect_no_tls_verify(self): + c = ImapClient(host="mail.x", username="u", password="p", tls_verify=False) + with ( + patch("email_client.imap_client.imaplib.IMAP4_SSL") as ssl_cls, + patch("email_client.imap_client.ssl.create_default_context") as ctx_factory, + ): + ctx = ctx_factory.return_value + c.connect() + assert ctx.check_hostname is False + ssl_cls.assert_called_once() + + def test_disconnect_swallows_errors(self): + c = _client() + conn = MagicMock() + conn.close.side_effect = OSError("already closed") + c._connection = conn + c.disconnect() # must not raise + assert c._connection is None + + +class TestFetchEmails: + def test_raises_when_not_connected(self): + c = _client() + try: + c.fetch_emails(sender="a@b.com") + assert False, "expected RuntimeError" + except RuntimeError: + pass + + def test_search_and_fetch(self): + c = _client() + conn = MagicMock() + conn.search.return_value = ("OK", [b"1 2"]) + conn.fetch.side_effect = [ + ("OK", [(b"1", _raw(message_id=""))]), + ("OK", [(b"2", _raw(message_id=""))]), + ] + c._connection = conn + out = c.fetch_emails(sender="alerts@example.com") + assert [m.message_id for m in out] == ["", ""] + assert "FROM" in conn.search.call_args.args[1] + + def test_since_adds_criteria(self): + from datetime import datetime, timezone + + c = _client() + conn = MagicMock() + conn.search.return_value = ("OK", [b""]) + c._connection = conn + c.fetch_emails( + sender="a@b.com", + since=datetime(2026, 4, 1, tzinfo=timezone.utc), + ) + assert "SINCE" in conn.search.call_args.args[1] + + def test_empty_search_returns_empty(self): + c = _client() + conn = MagicMock() + conn.search.return_value = ("OK", [None]) + c._connection = conn + assert c.fetch_emails(sender="a@b.com") == [] + + +class TestParseEmail: + def test_plain_email_fields(self): + c = _client() + msg = c._parse_email(_raw()) + assert msg.subject == "Security Alert" + assert msg.sender == "alerts@example.com" + assert msg.sender_display == "Alerts " + assert "soc@company.com" in msg.recipients + assert msg.body_plain.strip() == "hello body" + assert msg.date.tzinfo is not None + + def test_cc_recipients(self): + c = _client() + msg = c._parse_email(_raw(cc="Boss ")) + assert "boss@company.com" in msg.recipients + + def test_html_alternative(self): + c = _client() + msg = c._parse_email(_raw(plain="t", html="

rich

")) + assert "rich" in msg.body_html + + def test_references_split(self): + c = _client() + msg = c._parse_email(_raw(references=" ")) + assert msg.references == ["", ""] + + def test_attachment_extracted(self): + c = _client() + raw = _raw(attachment=("report.txt", b"file-bytes", ("text", "plain"))) + msg = c._parse_email(raw) + assert len(msg.attachments) == 1 + assert msg.attachments[0].filename == "report.txt" + assert msg.attachments[0].content == b"file-bytes" + + +class TestThreadIdAndHeaders: + def test_thread_id_prefers_in_reply_to(self): + c = _client() + m = EmailMessage( + message_id="", + subject="s", + sender="a@b", + recipients=[], + date=None, + body_plain="", + body_html="", + thread_id="", + in_reply_to="", + ) + assert c.get_thread_id(m) == "" + + def test_thread_id_references_then_message_id(self): + c = _client() + m = EmailMessage( + message_id="", + subject="s", + sender="a@b", + recipients=[], + date=None, + body_plain="", + body_html="", + thread_id="", + references=[""], + ) + assert c.get_thread_id(m) == "" + m.references = [] + assert c.get_thread_id(m) == "" + + def test_decode_header_handles_empty_and_encoded(self): + assert ImapClient._decode_header("") == "" + # RFC2047 encoded-word + encoded = "=?utf-8?q?Caf=C3=A9?=" + assert ImapClient._decode_header(encoded) == "Café" diff --git a/external-import/email-cases-importer/tests/test_main.py b/external-import/email-cases-importer/tests/test_main.py new file mode 100644 index 00000000000..be54d57ae92 --- /dev/null +++ b/external-import/email-cases-importer/tests/test_main.py @@ -0,0 +1,47 @@ +"""Entrypoint tests for src/main.py. + +main.py is a thin __main__ wrapper. We exec it via runpy with the connector +package + pycti helper patched, covering both the success path and the +exit(1)-on-error path. +""" + +import os +import runpy +from unittest.mock import MagicMock + +import pytest + +MAIN_PATH = os.path.join(os.path.dirname(__file__), "..", "src", "main.py") + + +def test_main_success_runs_connector(monkeypatch): + import pycti + + import connector as connector_pkg + + fake_settings = MagicMock() + fake_conn = MagicMock() + monkeypatch.setattr( + connector_pkg, "ConnectorSettings", MagicMock(return_value=fake_settings) + ) + monkeypatch.setattr( + connector_pkg, "EmailCasesConnector", MagicMock(return_value=fake_conn) + ) + monkeypatch.setattr(pycti, "OpenCTIConnectorHelper", MagicMock()) + + runpy.run_path(MAIN_PATH, run_name="__main__") + + fake_conn.run.assert_called_once() + + +def test_main_exits_on_error(monkeypatch): + import connector as connector_pkg + + monkeypatch.setattr( + connector_pkg, + "ConnectorSettings", + MagicMock(side_effect=RuntimeError("bad config")), + ) + + with pytest.raises(SystemExit): + runpy.run_path(MAIN_PATH, run_name="__main__") diff --git a/external-import/email-cases-importer/tests/test_settings.py b/external-import/email-cases-importer/tests/test_settings.py new file mode 100644 index 00000000000..534c62691e8 --- /dev/null +++ b/external-import/email-cases-importer/tests/test_settings.py @@ -0,0 +1,234 @@ +"""Unit tests for connector.settings — Pydantic field validators and parsers. + +These tests instantiate EmailCasesConfig directly so they don't require the full +ConnectorSettings (which depends on connectors-sdk environment loading). +""" + +import json + +import pytest +from pydantic import ValidationError + +from connector.settings import EmailCasesConfig + + +def _base_kwargs(**overrides): + """Minimal valid kwargs for EmailCasesConfig (only required fields).""" + base = { + "sender_address": "alerts@example.com", + "subject_filters": '[{"type":"contains","value":"Alert"}]', + } + base.update(overrides) + return base + + +# --------------------------------------------------------------------------- +# subject_filters validator +# --------------------------------------------------------------------------- + + +class TestSubjectFiltersValidator: + def test_valid_filters(self): + cfg = EmailCasesConfig(**_base_kwargs()) + assert cfg.get_parsed_subject_filters() == [ + {"type": "contains", "value": "Alert"} + ] + + def test_invalid_json_raises(self): + with pytest.raises(ValidationError): + EmailCasesConfig(**_base_kwargs(subject_filters="not-json")) + + def test_must_be_array(self): + with pytest.raises(ValidationError): + EmailCasesConfig(**_base_kwargs(subject_filters='{"type":"exact"}')) + + def test_filter_must_have_type_and_value(self): + with pytest.raises(ValidationError): + EmailCasesConfig(**_base_kwargs(subject_filters='[{"type":"exact"}]')) + + def test_invalid_filter_type(self): + with pytest.raises(ValidationError): + EmailCasesConfig( + **_base_kwargs(subject_filters='[{"type":"glob","value":"*"}]') + ) + + @pytest.mark.parametrize("ftype", ["exact", "contains", "regex"]) + def test_all_supported_filter_types(self, ftype): + cfg = EmailCasesConfig( + **_base_kwargs(subject_filters=json.dumps([{"type": ftype, "value": "x"}])) + ) + assert cfg.get_parsed_subject_filters()[0]["type"] == ftype + + +# --------------------------------------------------------------------------- +# subject_rules validator +# --------------------------------------------------------------------------- + + +class TestSubjectRulesValidator: + def test_default_empty_list(self): + cfg = EmailCasesConfig(**_base_kwargs()) + assert cfg.get_parsed_subject_rules() == [] + + def test_valid_rule(self): + rule = { + "match_type": "contains", + "value": "Threat", + "labels": ["Threat Alert"], + } + cfg = EmailCasesConfig(**_base_kwargs(subject_rules=json.dumps([rule]))) + assert cfg.get_parsed_subject_rules() == [rule] + + def test_invalid_match_type(self): + with pytest.raises(ValidationError): + EmailCasesConfig( + **_base_kwargs(subject_rules='[{"match_type":"glob","value":"*"}]') + ) + + def test_missing_required_keys(self): + with pytest.raises(ValidationError): + EmailCasesConfig(**_base_kwargs(subject_rules='[{"match_type":"exact"}]')) + + def test_must_be_array(self): + with pytest.raises(ValidationError): + EmailCasesConfig(**_base_kwargs(subject_rules="{}")) + + def test_invalid_json(self): + with pytest.raises(ValidationError): + EmailCasesConfig(**_base_kwargs(subject_rules="not-json")) + + @pytest.mark.parametrize( + "match_type", ["exact", "contains", "starts_with", "regex"] + ) + def test_all_supported_match_types(self, match_type): + cfg = EmailCasesConfig( + **_base_kwargs( + subject_rules=json.dumps([{"match_type": match_type, "value": "x"}]) + ) + ) + assert cfg.get_parsed_subject_rules()[0]["match_type"] == match_type + + +# --------------------------------------------------------------------------- +# sender_rules validator +# --------------------------------------------------------------------------- + + +class TestSenderRulesValidator: + def test_default_empty_list(self): + cfg = EmailCasesConfig(**_base_kwargs()) + assert cfg.get_parsed_sender_rules() == [] + + def test_valid_rule(self): + rule = {"sender": "x@y.com", "author": "Acme"} + cfg = EmailCasesConfig(**_base_kwargs(sender_rules=json.dumps([rule]))) + assert cfg.get_parsed_sender_rules() == [rule] + + def test_must_be_array(self): + with pytest.raises(ValidationError): + EmailCasesConfig(**_base_kwargs(sender_rules='{"sender":"x"}')) + + def test_missing_sender_field(self): + with pytest.raises(ValidationError): + EmailCasesConfig(**_base_kwargs(sender_rules='[{"author":"Acme"}]')) + + def test_invalid_json(self): + with pytest.raises(ValidationError): + EmailCasesConfig(**_base_kwargs(sender_rules="not-json")) + + +# --------------------------------------------------------------------------- +# labels parsing +# --------------------------------------------------------------------------- + + +class TestParsedLabels: + def test_empty_default(self): + cfg = EmailCasesConfig(**_base_kwargs()) + assert cfg.get_parsed_labels() == [] + + def test_comma_separated(self): + cfg = EmailCasesConfig(**_base_kwargs(labels="A,B,C")) + assert cfg.get_parsed_labels() == ["A", "B", "C"] + + def test_strips_whitespace(self): + cfg = EmailCasesConfig(**_base_kwargs(labels=" A , B , C ")) + assert cfg.get_parsed_labels() == ["A", "B", "C"] + + def test_drops_empty_entries(self): + cfg = EmailCasesConfig(**_base_kwargs(labels="A,,B,")) + assert cfg.get_parsed_labels() == ["A", "B"] + + +# --------------------------------------------------------------------------- +# Defaults / typing +# --------------------------------------------------------------------------- + + +class TestStartDateValidator: + def test_empty_default(self): + cfg = EmailCasesConfig(**_base_kwargs()) + assert cfg.start_date == "" + assert cfg.get_parsed_start_date() is None + + def test_accepts_date_only(self): + cfg = EmailCasesConfig(**_base_kwargs(start_date="2026-04-01")) + parsed = cfg.get_parsed_start_date() + assert parsed is not None + assert parsed.year == 2026 and parsed.month == 4 and parsed.day == 1 + # must be tz-aware (UTC) + assert parsed.tzinfo is not None + assert parsed.utcoffset().total_seconds() == 0 + + def test_accepts_iso_with_z(self): + cfg = EmailCasesConfig(**_base_kwargs(start_date="2026-04-01T08:30:00Z")) + parsed = cfg.get_parsed_start_date() + assert parsed is not None + assert parsed.hour == 8 and parsed.minute == 30 + assert parsed.tzinfo is not None + + def test_accepts_iso_with_offset(self): + cfg = EmailCasesConfig(**_base_kwargs(start_date="2026-04-01T08:30:00+02:00")) + parsed = cfg.get_parsed_start_date() + assert parsed is not None + assert parsed.tzinfo is not None + + def test_rejects_garbage(self): + with pytest.raises(ValidationError): + EmailCasesConfig(**_base_kwargs(start_date="not-a-date")) + + def test_rejects_wrong_format(self): + with pytest.raises(ValidationError): + EmailCasesConfig(**_base_kwargs(start_date="04/01/2026")) + + +class TestDefaults: + def test_protocol_default_is_imap(self): + cfg = EmailCasesConfig(**_base_kwargs()) + assert cfg.protocol == "imap" + + def test_invalid_protocol_rejected(self): + with pytest.raises(ValidationError): + EmailCasesConfig(**_base_kwargs(protocol="pop3")) + + def test_invalid_thread_strategy_rejected(self): + with pytest.raises(ValidationError): + EmailCasesConfig(**_base_kwargs(thread_tracking_strategy="hashing")) + + def test_invalid_ews_auth_rejected(self): + with pytest.raises(ValidationError): + EmailCasesConfig(**_base_kwargs(ews_auth_type="Kerberos")) + + def test_password_marker_defaults(self): + cfg = EmailCasesConfig(**_base_kwargs()) + assert cfg.password_prefix == "---BEGIN PASSWORD---" + assert cfg.password_suffix == "---END PASSWORD---" + assert cfg.password_strip_whitespace is False + + def test_import_settings_defaults(self): + cfg = EmailCasesConfig(**_base_kwargs()) + assert cfg.import_interval == 300 + assert cfg.max_emails_per_cycle == 50 + assert cfg.tls_verify is True + assert cfg.max_attachment_size_mb == 25 + assert cfg.attachment_store_in_opencti is True diff --git a/external-import/email-cases-importer/tests/test_utils.py b/external-import/email-cases-importer/tests/test_utils.py new file mode 100644 index 00000000000..a9f2e6f567c --- /dev/null +++ b/external-import/email-cases-importer/tests/test_utils.py @@ -0,0 +1,243 @@ +"""Unit tests for connector.utils.""" + +import hashlib + +import pytest + +from connector.utils import ( + collapse_blank_lines, + compute_file_hashes, + extract_passwords, + matches_subject_filter, + normalize_subject, + sanitize_html, +) + +# --------------------------------------------------------------------------- +# extract_passwords +# --------------------------------------------------------------------------- + + +class TestExtractPasswords: + def test_basic_extraction(self): + body = "Hello\n---BEGIN PASSWORD---hunter2---END PASSWORD---\nbye" + assert extract_passwords( + body, "---BEGIN PASSWORD---", "---END PASSWORD---" + ) == ["hunter2"] + + def test_multiple_passwords(self): + body = "[BEGIN]p1[END] some text [BEGIN]p2[END]" + assert extract_passwords(body, "[BEGIN]", "[END]") == ["p1", "p2"] + + def test_returns_empty_when_no_match(self): + assert extract_passwords("nothing here", "[BEGIN]", "[END]") == [] + + def test_returns_empty_when_prefix_or_suffix_blank(self): + assert extract_passwords("body", "", "[END]") == [] + assert extract_passwords("body", "[BEGIN]", "") == [] + + def test_strip_whitespace_removes_internal_whitespace(self): + body = "[BEGIN]hun ter\n2[END]" + assert extract_passwords(body, "[BEGIN]", "[END]", strip_whitespace=True) == [ + "hunter2" + ] + + def test_strip_whitespace_false_keeps_internal(self): + body = "[BEGIN]hun ter[END]" + assert extract_passwords(body, "[BEGIN]", "[END]", strip_whitespace=False) == [ + "hun ter" + ] + + def test_html_entities_decoded_in_body(self): + # & in body should be unescaped to & before matching + body = "[BEGIN]a&b[END]" + assert extract_passwords(body, "[BEGIN]", "[END]") == ["a&b"] + + def test_skips_empty_match(self): + body = "[BEGIN] [END]" + assert extract_passwords(body, "[BEGIN]", "[END]") == [] + + def test_dotall_matches_across_newlines(self): + body = "[BEGIN]line1\nline2[END]" + # default strip_whitespace=False keeps the newline-bearing match + assert extract_passwords(body, "[BEGIN]", "[END]") == ["line1\nline2"] + + def test_regex_special_chars_in_markers_are_escaped(self): + body = "*PWD*hunter2*END*" + assert extract_passwords(body, "*PWD*", "*END*") == ["hunter2"] + + +# --------------------------------------------------------------------------- +# normalize_subject +# --------------------------------------------------------------------------- + + +class TestNormalizeSubject: + def test_empty_string(self): + assert normalize_subject("") == "" + + def test_no_prefix(self): + assert normalize_subject("Security Alert") == "Security Alert" + + def test_strips_re_prefix(self): + assert normalize_subject("RE: Security Alert") == "Security Alert" + + def test_strips_fwd_prefix(self): + assert normalize_subject("FWD: Security Alert") == "Security Alert" + + def test_strips_multiple_prefixes(self): + assert normalize_subject("RE: FW: RE: Alert") == "Alert" + + def test_case_insensitive(self): + assert normalize_subject("re: Alert") == "Alert" + assert normalize_subject("Re: Alert") == "Alert" + + @pytest.mark.parametrize( + "raw,expected", + [ + ("AW: Test", "Test"), # German RE + ("WG: Test", "Test"), # German FW + ("TR: Test", "Test"), # French FW + ("RV: Test", "Test"), # Spanish FW + ("ENC: Test", "Test"), # Portuguese FW + ("VS: Test", "Test"), # Norwegian/Swedish FW + ], + ) + def test_localized_prefixes(self, raw, expected): + assert normalize_subject(raw) == expected + + def test_strips_whitespace(self): + assert normalize_subject(" Alert ") == "Alert" + + +# --------------------------------------------------------------------------- +# sanitize_html +# --------------------------------------------------------------------------- + + +class TestCollapseBlankLines: + def test_empty_returns_empty(self): + assert collapse_blank_lines("") == "" + assert collapse_blank_lines(None) is None # type: ignore[arg-type] + + def test_no_blank_lines_unchanged(self): + assert collapse_blank_lines("a\nb\nc") == "a\nb\nc" + + def test_single_blank_line_preserved(self): + # One blank line between paragraphs is a paragraph break — keep it + assert collapse_blank_lines("a\n\nb") == "a\n\nb" + + def test_many_blank_lines_collapsed(self): + # Classic Outlook-explosion: 10 newlines → 2 (one blank line) + assert collapse_blank_lines("a\n\n\n\n\n\n\n\n\n\nb") == "a\n\nb" + + def test_whitespace_only_lines_treated_as_blank(self): + # Lines containing only spaces/tabs should still collapse + assert collapse_blank_lines("a\n \n\t\n \nb") == "a\n\nb" + + def test_crlf_normalized(self): + assert collapse_blank_lines("a\r\n\r\n\r\n\r\nb") == "a\n\nb" + + def test_leading_trailing_blanks_trimmed(self): + assert collapse_blank_lines("\n\n\nhello\n\n\n") == "hello" + + +class TestSanitizeHtml: + def test_empty_returns_empty(self): + assert sanitize_html("") == "" + + def test_strips_tags(self): + assert sanitize_html("hello") == "hello" + + def test_removes_style_block(self): + html_in = "visible" + assert "color:red" not in sanitize_html(html_in) + assert "visible" in sanitize_html(html_in) + + def test_removes_script_block(self): + html_in = "safe" + assert "alert" not in sanitize_html(html_in) + assert "safe" in sanitize_html(html_in) + + def test_br_becomes_newline(self): + assert sanitize_html("a
b") == "a\nb" + + def test_p_becomes_double_newline(self): + assert "para1" in sanitize_html("

para1

para2

") + + def test_decodes_html_entities(self): + assert sanitize_html("a & b") == "a & b" + + +# --------------------------------------------------------------------------- +# compute_file_hashes +# --------------------------------------------------------------------------- + + +class TestComputeFileHashes: + def test_known_hashes(self): + data = b"hello" + out = compute_file_hashes(data) + assert out["MD5"] == hashlib.md5(data).hexdigest() # noqa: S324 + assert out["SHA-1"] == hashlib.sha1(data).hexdigest() # noqa: S324 + assert out["SHA-256"] == hashlib.sha256(data).hexdigest() + + def test_empty_bytes(self): + out = compute_file_hashes(b"") + # all three hashes should still be returned + assert set(out.keys()) == {"MD5", "SHA-1", "SHA-256"} + assert all(isinstance(v, str) and v for v in out.values()) + + +# --------------------------------------------------------------------------- +# matches_subject_filter +# --------------------------------------------------------------------------- + + +class TestMatchesSubjectFilter: + def test_empty_filters_matches_all(self): + # Empty list means "accept any subject" — intuitive "no filter" semantics. + assert matches_subject_filter("anything", []) is True + assert matches_subject_filter("", []) is True + assert matches_subject_filter("INC-1234 fired", []) is True + + def test_exact_match(self): + assert ( + matches_subject_filter("Alert", [{"type": "exact", "value": "Alert"}]) + is True + ) + + def test_exact_no_match(self): + assert ( + matches_subject_filter("alert", [{"type": "exact", "value": "Alert"}]) + is False + ) + + def test_contains_match(self): + assert ( + matches_subject_filter( + "Big Alert Today", [{"type": "contains", "value": "Alert"}] + ) + is True + ) + + def test_regex_match(self): + assert ( + matches_subject_filter( + "INC-1234 fired", [{"type": "regex", "value": r"INC-\d+"}] + ) + is True + ) + + def test_invalid_regex_is_skipped(self): + # Bad regex should not raise — function continues to the next filter + filters = [ + {"type": "regex", "value": "["}, # invalid + {"type": "exact", "value": "ok"}, + ] + assert matches_subject_filter("ok", filters) is True + + def test_unknown_filter_type_returns_false(self): + assert ( + matches_subject_filter("subj", [{"type": "wat", "value": "subj"}]) is False + )