4112 - OpenCTI VS Connectors

.pre-commit-config.yaml

@@ -129,12 +129,52 @@ repos:
     rev: v1.5.1
     hooks:
       - id: mypy
+        # Excludes every VS connector below (all four now built -- see
+        # OpenCTI-connector.md §8's build order). Each has its own
+        # config/db/mapping/connector module under a bare `src` package (§10i:
+        # built structurally identical on purpose) -- checking them in one mypy
+        # invocation (this hook's repo-wide sweep, or the connectors against each
+        # other) fails with "Duplicate module named 'src...'", since mypy keys its
+        # module graph by dotted name, not file path, and every connector resolves
+        # to the same names (`src.db`, `src.mapping`, ...). Checked independently
+        # below instead -- same per-subtree scoping this file already uses for the
+        # backend-only flake8 hook (T20/print-statements) further up.
+        exclude: ^open-cti/connectors/(vs-org-bootstrap|vs-ticket-ingestion|vs-vulnscan-enrichment|vs-port-inventory)/
         additional_dependencies:
           - types-python-dateutil
           - types-pytz
           - types-requests
           - types-retry
+          - types-PyYAML
+          - types-psycopg2
+          - types-python-dateutil
           - redis==5.2.0
+      - id: mypy
+        name: "mypy (connector: vs-org-bootstrap)"
+        files: ^open-cti/connectors/vs-org-bootstrap/
+        additional_dependencies:
+          - types-PyYAML
+          - types-psycopg2
+      - id: mypy
+        name: "mypy (connector: vs-ticket-ingestion)"
+        files: ^open-cti/connectors/vs-ticket-ingestion/
+        additional_dependencies:
+          - types-PyYAML
+          - types-psycopg2
+          - types-python-dateutil
+      - id: mypy
+        name: "mypy (connector: vs-vulnscan-enrichment)"
+        files: ^open-cti/connectors/vs-vulnscan-enrichment/
+        additional_dependencies:
+          - types-PyYAML
+          - types-psycopg2
+      - id: mypy
+        name: "mypy (connector: vs-port-inventory)"
+        files: ^open-cti/connectors/vs-port-inventory/
+        additional_dependencies:
+          - types-PyYAML
+          - types-psycopg2
+          - types-python-dateutil
   - repo: https://github.com/asottile/pyupgrade
     rev: v3.10.1
     hooks:

backend/pe/peScanController.py

@@ -250,7 +250,11 @@ def fetch_orgs_from_db(*, report_on: bool = False, demo: bool = False) -> List[s
     else:
         raise ValueError("fetch_orgs_from_db requires report_on=True or demo=True")
 
-    with psycopg2.connect(**pe_db_connection_params()) as conn:
+    # mypy can't resolve **dict unpacking against connect()'s overloads even though this is
+    # valid, standard psycopg2 usage at runtime -- a known typeshed/mypy limitation with
+    # unpacking a plain Mapping into an @overload'd function (no overload's signature is a
+    # verifiable match for an arbitrary Dict[str, str] spread, even the **kwargs: Any catch-all).
+    with psycopg2.connect(**pe_db_connection_params()) as conn:  # type: ignore[call-overload]
         with conn.cursor() as cur:
             cur.execute(query)
             names = [row[0] for row in cur.fetchall()]

backend/src/xfd_django/xfd_api/tasks/utils/query_redshift.py

@@ -277,6 +277,7 @@ def fetch_in_chunks_keyset_frozen_bulk(
         where_sql = sql.SQL(" AND ").join(where_clauses)
 
         # Handle schema-qualified table names
+        table_ident: sql.Composable
         if "." in table:
             schema, table_name = table.split(".", 1)
             table_ident = sql.SQL(".").join(

infrastructure/open_cti.tf

@@ -275,6 +275,8 @@ locals {
       wiz_backend_env_ssm_param       = var.ssm_wiz_backend_env
       wiz_api_client_id_ssm_param     = var.ssm_wiz_api_client_id
       wiz_api_client_secret_ssm_param = var.ssm_wiz_api_client_secret
+      vs_org_bootstrap_db_host        = aws_db_instance.db.address
+      vs_org_bootstrap_aws_region     = var.aws_region
     }),
     "",
     "cat > /opt/open-cti/install-deps.sh <<'INSTALL_DEPS_EOF'",

infrastructure/open_cti_user_data_env.sh.tpl

@@ -27,6 +27,8 @@ CENSYS_ORG_ID="${censys_org_id}"
 QUALYS_API_USERNAME="${qualys_api_username}"
 XTM_ONE_HOST="${xtm_one_host}"
 XTM_ONE_ADMIN_EMAIL="${xtm_one_admin_email}"
+VS_ORG_BOOTSTRAP_DB_HOST="${vs_org_bootstrap_db_host}"
+VS_ORG_BOOTSTRAP_AWS_REGION="${vs_org_bootstrap_aws_region}"
 WIZ_BACKEND_ENV_SSM_PARAM="${wiz_backend_env_ssm_param}"
 WIZ_API_CLIENT_ID_SSM_PARAM="${wiz_api_client_id_ssm_param}"
 WIZ_API_CLIENT_SECRET_SSM_PARAM="${wiz_api_client_secret_ssm_param}"

infrastructure/vars.tf

@@ -688,6 +688,10 @@ variable "open_cti_secret_keys" {
     "CONNECTOR_QUALYS_CVE_ENRICHMENT_API_KEY",
     "CONNECTOR_SHODAN_API_KEY",
     "CONNECTOR_VULNCHECK_API_KEY",
+    "CONNECTOR_VS_ORG_BOOTSTRAP_API_KEY",
+    "CONNECTOR_VS_TICKET_INGESTION_API_KEY",
+    "CONNECTOR_VS_VULNSCAN_ENRICHMENT_API_KEY",
+    "CONNECTOR_VS_PORT_INVENTORY_API_KEY",
     "MINIO_ROOT_PASSWORD",
     "NVD_API_KEY",
     "OPENCTI_ADMIN_PASSWORD",

mypy.ini

@@ -0,0 +1,17 @@
+[mypy]
+
+# pycti and stix2 (open-cti/connectors/vs-org-bootstrap/) have no stub packages available on
+# PyPI/typeshed at all -- unlike types-PyYAML/types-psycopg2 (added to .pre-commit-config.yaml's
+# mypy hook instead, since those genuinely exist), there's no "pip install types-X" that fixes
+# this for these two. Scoped narrowly rather than a repo-wide ignore_missing_imports.
+[mypy-pycti.*]
+ignore_missing_imports = True
+
+[mypy-stix2.*]
+ignore_missing_imports = True
+
+# boto3 has real stubs available (boto3-stubs), but they're a heavy, per-service package and
+# boto3 is already imported repo-wide in backend/ without them -- matching that existing
+# precedent here rather than introducing a different standard for just this one new directory.
+[mypy-boto3.*]
+ignore_missing_imports = True