diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index d90dee982..13948de84 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -129,12 +129,52 @@ repos: rev: v1.5.1 hooks: - id: mypy + # Excludes every VS connector below (all four now built -- see + # OpenCTI-connector.md §8's build order). Each has its own + # config/db/mapping/connector module under a bare `src` package (§10i: + # built structurally identical on purpose) -- checking them in one mypy + # invocation (this hook's repo-wide sweep, or the connectors against each + # other) fails with "Duplicate module named 'src...'", since mypy keys its + # module graph by dotted name, not file path, and every connector resolves + # to the same names (`src.db`, `src.mapping`, ...). Checked independently + # below instead -- same per-subtree scoping this file already uses for the + # backend-only flake8 hook (T20/print-statements) further up. + exclude: ^open-cti/connectors/(vs-org-bootstrap|vs-ticket-ingestion|vs-vulnscan-enrichment|vs-port-inventory)/ additional_dependencies: - types-python-dateutil - types-pytz - types-requests - types-retry + - types-PyYAML + - types-psycopg2 + - types-python-dateutil - redis==5.2.0 + - id: mypy + name: "mypy (connector: vs-org-bootstrap)" + files: ^open-cti/connectors/vs-org-bootstrap/ + additional_dependencies: + - types-PyYAML + - types-psycopg2 + - id: mypy + name: "mypy (connector: vs-ticket-ingestion)" + files: ^open-cti/connectors/vs-ticket-ingestion/ + additional_dependencies: + - types-PyYAML + - types-psycopg2 + - types-python-dateutil + - id: mypy + name: "mypy (connector: vs-vulnscan-enrichment)" + files: ^open-cti/connectors/vs-vulnscan-enrichment/ + additional_dependencies: + - types-PyYAML + - types-psycopg2 + - id: mypy + name: "mypy (connector: vs-port-inventory)" + files: ^open-cti/connectors/vs-port-inventory/ + additional_dependencies: + - types-PyYAML + - types-psycopg2 + - types-python-dateutil - repo: https://github.com/asottile/pyupgrade rev: v3.10.1 hooks: diff --git a/backend/pe/peScanController.py b/backend/pe/peScanController.py index 3a5cecbdd..e7d80370a 100644 --- a/backend/pe/peScanController.py +++ b/backend/pe/peScanController.py @@ -250,7 +250,11 @@ def fetch_orgs_from_db(*, report_on: bool = False, demo: bool = False) -> List[s else: raise ValueError("fetch_orgs_from_db requires report_on=True or demo=True") - with psycopg2.connect(**pe_db_connection_params()) as conn: + # mypy can't resolve **dict unpacking against connect()'s overloads even though this is + # valid, standard psycopg2 usage at runtime -- a known typeshed/mypy limitation with + # unpacking a plain Mapping into an @overload'd function (no overload's signature is a + # verifiable match for an arbitrary Dict[str, str] spread, even the **kwargs: Any catch-all). + with psycopg2.connect(**pe_db_connection_params()) as conn: # type: ignore[call-overload] with conn.cursor() as cur: cur.execute(query) names = [row[0] for row in cur.fetchall()] diff --git a/backend/src/xfd_django/xfd_api/tasks/utils/query_redshift.py b/backend/src/xfd_django/xfd_api/tasks/utils/query_redshift.py index 6fe70ffcb..68f6724d0 100644 --- a/backend/src/xfd_django/xfd_api/tasks/utils/query_redshift.py +++ b/backend/src/xfd_django/xfd_api/tasks/utils/query_redshift.py @@ -277,6 +277,7 @@ def fetch_in_chunks_keyset_frozen_bulk( where_sql = sql.SQL(" AND ").join(where_clauses) # Handle schema-qualified table names + table_ident: sql.Composable if "." in table: schema, table_name = table.split(".", 1) table_ident = sql.SQL(".").join( diff --git a/infrastructure/open_cti.tf b/infrastructure/open_cti.tf index 023fcf80d..ea0025ef1 100644 --- a/infrastructure/open_cti.tf +++ b/infrastructure/open_cti.tf @@ -275,6 +275,8 @@ locals { wiz_backend_env_ssm_param = var.ssm_wiz_backend_env wiz_api_client_id_ssm_param = var.ssm_wiz_api_client_id wiz_api_client_secret_ssm_param = var.ssm_wiz_api_client_secret + vs_org_bootstrap_db_host = aws_db_instance.db.address + vs_org_bootstrap_aws_region = var.aws_region }), "", "cat > /opt/open-cti/install-deps.sh <<'INSTALL_DEPS_EOF'", diff --git a/infrastructure/open_cti_user_data_env.sh.tpl b/infrastructure/open_cti_user_data_env.sh.tpl index 08be3b170..36a8130db 100644 --- a/infrastructure/open_cti_user_data_env.sh.tpl +++ b/infrastructure/open_cti_user_data_env.sh.tpl @@ -27,6 +27,8 @@ CENSYS_ORG_ID="${censys_org_id}" QUALYS_API_USERNAME="${qualys_api_username}" XTM_ONE_HOST="${xtm_one_host}" XTM_ONE_ADMIN_EMAIL="${xtm_one_admin_email}" +VS_ORG_BOOTSTRAP_DB_HOST="${vs_org_bootstrap_db_host}" +VS_ORG_BOOTSTRAP_AWS_REGION="${vs_org_bootstrap_aws_region}" WIZ_BACKEND_ENV_SSM_PARAM="${wiz_backend_env_ssm_param}" WIZ_API_CLIENT_ID_SSM_PARAM="${wiz_api_client_id_ssm_param}" WIZ_API_CLIENT_SECRET_SSM_PARAM="${wiz_api_client_secret_ssm_param}" diff --git a/infrastructure/stage.tfvars b/infrastructure/stage.tfvars index 9cc34d354..5e25c44a9 100644 --- a/infrastructure/stage.tfvars +++ b/infrastructure/stage.tfvars @@ -1,146 +1,152 @@ -adot_python_layer_arn_govcloud = "arn:aws-us-gov:lambda:us-gov-west-1:263492004256:layer:adot-python-with-collector:1" -aws_region = "us-gov-east-1" -aws_other_region = "us-gov-west-1" -aws_partition = "aws-us-gov" -is_dmz = false -project = "Crossfeed" -stage = "staging" -frontend_domain = "staging.crossfeed.cyber.dhs.gov" -frontend_lambda_function = "crossfeed-security-headers-staging" -frontend_bucket = "staging.crossfeed.cyber.dhs.gov" -api_domain = "api.staging.crossfeed.cyber.dhs.gov" -db_name = "crossfeed-stage-db" -db_port = 5432 -xpanse_org_sync_bucket_name = "crossfeed-xpanse-org-sync-staging" -db_table_name = "cfstagingdb" -db_instance_class = "db.m7i.4xlarge" -django_env_bucket_name = "cyhy-staging-django-env" -log_metric_namespace = "LogMetrics" -log_metric_root_user = "crossfeed-staging-RootUserAccess" -log_metric_api_error_rate = "crossfeed-staging-APIErrorRate" -log_metric_unauthorized_api_call = "crossfeed-staging-UnauthorizedApiCall" -log_metric_login_without_mfa = "crossfeed-staging-ConsoleSignInWithoutMFA" -log_metric_iam_policy = "crossfeed-staging-IAMPolicyChange" -log_metric_cloudtrail = "crossfeed-staging-CloudTrailConfigurationChanges" -log_metric_login_failure = "crossfeed-staging-ConsoleLoginFailure" -log_metric_cmk_delete_disable = "crossfeed-staging-DisablingOrScheduledDeletionOfCMK" -log_metric_s3_bucket_policy = "crossfeed-staging-S3BucketPolicyChanges" -log_metric_aws_config = "crossfeed-staging-AWSConfigConfigurationChange" -log_metric_security_group = "crossfeed-staging-SecurityGroupChange" -log_metric_nacl = "crossfeed-staging-NACLChange" -log_metric_network_gateway = "crossfeed-staging-NetworkGatewayChange" -log_metric_route_table = "crossfeed-staging-RouteTableChange" -log_metric_vpc = "crossfeed-staging-VPCChange" -log_metric_ec2_shutdown = "crossfeed-staging-EC2Shutdown" -log_metric_db_shutdown = "crossfeed-staging-DBShutdown" -log_metric_db_deletion = "crossfeed-staging-DBDeletion" -matomo_force_index_url = "staging.crossfeed.cyber.dhs.gov/matomo" -sns_topic_alarms = "crossfeed-staging-cis-alarms" -ssm_crossfeed_vpc_name = "/crossfeed/staging/VPC_NAME" -ssm_django_env_kms_arn = "/crossfeed/staging/DJANGO_ENV_KMS_ARN" -ssm_lambda_subnet = "/crossfeed/staging/SUBNET_ID" -ssm_lambda_sg = "/crossfeed/staging/SG_ID" -ssm_worker_subnet = "/crossfeed/staging/WORKER_SUBNET_ID" -ssm_worker_sg = "/crossfeed/staging/WORKER_SG_ID" -ssm_worker_arn = "/crossfeed/staging/WORKER_CLUSTER_ARN" -ssm_db_name = "/crossfeed/staging/DATABASE_NAME" -ssm_db_host = "/crossfeed/staging/DATABASE_HOST" -ssm_db_username = "/crossfeed/staging/DATABASE_USER" -ssm_db_password = "/crossfeed/staging/DATABASE_PASSWORD" -ssm_pe_db_name = "/crossfeed/staging/PE_DB_NAME" -ssm_pe_db_username = "/crossfeed/staging/PE_DB_USERNAME" -ssm_pe_db_password = "/crossfeed/staging/PE_DB_PASSWORD" -ssm_matomo_db_password = "/crossfeed/staging/MATOMO_DATABASE_PASSWORD" -ssm_worker_signature_public_key = "/crossfeed/staging/WORKER_SIGNATURE_PUBLIC_KEY" -ssm_worker_signature_private_key = "/crossfeed/staging/WORKER_SIGNATURE_PRIVATE_KEY" -ssm_censys_api_id = "/crossfeed/staging/CENSYS_API_ID" -ssm_censys_api_secret = "/crossfeed/staging/CENSYS_API_SECRET" -ssm_shodan_api_key = "/crossfeed/staging/SHODAN_API_KEY" -ssm_shodan_ip_chunk_size = "/crossfeed/staging/SHODAN_IP_CHUNK_SIZE" -ssm_shodan_query_days_back = "/crossfeed/staging/SHODAN_QUERY_DAYS_BACK" -ssm_pe_shodan_api_keys = "/crossfeed/staging/PE_SHODAN_API_KEYS" -ssm_wiz_registry_secret_arn = "/crossfeed/staging/WIZ_REGISTRY_CREDENTIAL_SECRET_ARN" -ssm_wiz_service_account_secret_arn = "/crossfeed/staging/WIZ_SERVICE_ACCOUNT_SECRET_ARN" -ssm_wiz_http_proxy_cert_secret_arn = "/crossfeed/staging/WIZ_HTTP_PROXY_CERT_SECRET_ARN" -ssm_sixgill_client_id = "/crossfeed/staging/SIXGILL_CLIENT_ID" -ssm_sixgill_client_secret = "/crossfeed/staging/SIXGILL_CLIENT_SECRET" -ssm_lg_api_key = "/crossfeed/staging/LG_API_KEY" -ssm_lg_workspace_name = "/crossfeed/staging/LG_WORKSPACE_NAME" -ssm_ses_email_identity_arn = "/crossfeed/staging/SES_EMAIL_IDENTITY_ARN" -ssm_worker_kms_keys = "/crossfeed/staging/WORKER_KMS_KEYS" -db_group_name = "crossfeed-staging-db-group" -worker_ecs_repository_name = "crossfeed-staging-worker" -worker_ecs_cluster_name = "crossfeed-staging-worker" -worker_ecs_task_definition_family = "crossfeed-staging-worker" -worker_ecs_log_group_name = "crossfeed-staging-worker" -worker_ecs_role_name = "crossfeed-staging-worker" -logging_bucket_name = "cisa-crossfeed-staging-logging" -export_bucket_name = "cisa-crossfeed-staging-exports" -reports_bucket_name = "cisa-crossfeed-staging-reports" -pe_db_backups_bucket_name = "cisa-crossfeed-staging-pe-db-backups" -user_pool_name = "crossfeed-staging" -user_pool_domain = "crossfeed-staging" -ssm_user_pool_id = "/crossfeed/staging/USER_POOL_ID" -ssm_user_pool_client_id = "/crossfeed/staging/USER_POOL_CLIENT_ID" -ses_support_email_sender = "noreply@staging.crossfeed.cyber.dhs.gov" -ses_support_email_replyto = "vulnerability@cisa.dhs.gov" -matomo_availability_zone = "us-gov-east-1b" -matomo_ecs_cluster_name = "crossfeed-matomo-staging" -matomo_ecs_task_definition_family = "crossfeed-matomo-staging" -matomo_ecs_log_group_name = "crossfeed-matomo-staging" -matomo_db_name = "crossfeed-matomo-staging" -matomo_db_instance_class = "db.m5.xlarge" -matomo_ecs_role_name = "crossfeed-matomo-staging" -es_instance_type = "t3.small.elasticsearch" -es_instance_count = 3 -es_instance_volume_size = 100 -create_db_accessor_instance = false -create_pe_instance = false -db_accessor_instance_class = "t3.2xlarge" -create_elk_instance = false -elk_instance_class = "t3.2xlarge" -ami_id = "ami-0a1445a13e666a557" -cloudtrail_name = "crossfeed-staging-all-events" -cloudtrail_bucket_name = "cisa-crossfeed-staging-cloudtrail" -cloudtrail_role_name = "crossfeed-staging-cloudtrail" -cloudtrail_log_group_name = "crossfeed-staging-cloudtrail" -cloudwatch_bucket_name = "cisa-crossfeed-staging-cloudwatch" -cloudwatch_log_group_name = "crossfeed-staging-cloudwatch-bucket" -es_instance_master_count = 3 -ssm_vpc_id = "/LZ/VPC_ID" -ssm_vpc_cidr_block = "/LZ/VPC_CIDR_BLOCK" -ssm_sctask_cidr_block = "/LZ/SCTASK_CIDR_BLOCK" -ssm_route_table_endpoints_id = "/LZ/ROUTE_TABLE_ENDPOINTS_ID" -ssm_route_table_private_A_id = "/LZ/ROUTE_TABLE_PRIVATE_A_ID" -ssm_route_table_private_B_id = "/LZ/ROUTE_TABLE_PRIVATE_B_ID" -ssm_route_table_private_C_id = "/LZ/ROUTE_TABLE_PRIVATE_C_ID" -ssm_subnet_backend_id = "/LZ/SUBNET_ENDPOINT_A_ID" -ssm_subnet_worker_id = "/LZ/SUBNET_ENDPOINT_B_ID" -ssm_subnet_matomo_id = "/LZ/SUBNET_ENDPOINT_C_ID" -ssm_subnet_db_1_id = "/LZ/SUBNET_PRIVATE_A_ID" -ssm_subnet_db_2_id = "/LZ/SUBNET_PRIVATE_B_ID" -ssm_subnet_es_id = "/LZ/SUBNET_PRIVATE_C_ID" -ssm_mdl_name = "/crossfeed/staging/MDL_NAME" -ssm_mdl_username = "/crossfeed/staging/MDL_USERNAME" -ssm_mdl_password = "/crossfeed/staging/MDL_PASSWORD" -ssm_redshift_host = "/crossfeed/staging/REDSHIFT_HOST" -ssm_redshift_database = "/crossfeed/staging/REDSHIFT_DATABASE" -ssm_redshift_user = "/crossfeed/staging/REDSHIFT_USER" -ssm_redshift_password = "/crossfeed/staging/REDSHIFT_PASSWORD" -ssm_pe_api_key = "/crossfeed/staging/PE_API_KEY" -ssm_pe_api_url = "/crossfeed/staging/PE_API_URL" -ssm_mailer_arn = "/crossfeed/staging/MAILER_ARN" -ssm_cf_api_key = "/crossfeed/staging/CF_API_KEY" -ssm_checksum_salt = "/crossfeed/staging/CHECKSUM_SALT" -create_elasticache_cluster = true -ssm_dmz_api_key = "/crossfeed/staging/DMZ_API_KEY" -ssm_vs_pull_date_range = "/crossfeed/staging/VS_PULL_DATE_RANGE" -ssm_latest_port_scan_cutoff = "/crossfeed/staging/LATEST_PORT_SCAN_CUTOFF" -ssm_dmz_sync_endpoint = "/crossfeed/staging/MDL_SYNC_ENDPOINT" -create_email_sender_instance = false -email_sender_instance_type = "t3.small" -create_open_cti_instance = true -open_cti_instance_type = "m5.4xlarge" # matches stage-cd's real, already-running instance -- confirm availability in this gov-cloud region/AZ before relying on it +adot_python_layer_arn_govcloud = "arn:aws-us-gov:lambda:us-gov-west-1:263492004256:layer:adot-python-with-collector:1" +aws_region = "us-gov-east-1" +aws_other_region = "us-gov-west-1" +aws_partition = "aws-us-gov" +is_dmz = false +project = "Crossfeed" +stage = "staging" +frontend_domain = "staging.crossfeed.cyber.dhs.gov" +frontend_lambda_function = "crossfeed-security-headers-staging" +frontend_bucket = "staging.crossfeed.cyber.dhs.gov" +api_domain = "api.staging.crossfeed.cyber.dhs.gov" +db_name = "crossfeed-stage-db" +db_port = 5432 +xpanse_org_sync_bucket_name = "crossfeed-xpanse-org-sync-staging" +db_table_name = "cfstagingdb" +db_instance_class = "db.m7i.4xlarge" +django_env_bucket_name = "cyhy-staging-django-env" +log_metric_namespace = "LogMetrics" +log_metric_root_user = "crossfeed-staging-RootUserAccess" +log_metric_api_error_rate = "crossfeed-staging-APIErrorRate" +log_metric_unauthorized_api_call = "crossfeed-staging-UnauthorizedApiCall" +log_metric_login_without_mfa = "crossfeed-staging-ConsoleSignInWithoutMFA" +log_metric_iam_policy = "crossfeed-staging-IAMPolicyChange" +log_metric_cloudtrail = "crossfeed-staging-CloudTrailConfigurationChanges" +log_metric_login_failure = "crossfeed-staging-ConsoleLoginFailure" +log_metric_cmk_delete_disable = "crossfeed-staging-DisablingOrScheduledDeletionOfCMK" +log_metric_s3_bucket_policy = "crossfeed-staging-S3BucketPolicyChanges" +log_metric_aws_config = "crossfeed-staging-AWSConfigConfigurationChange" +log_metric_security_group = "crossfeed-staging-SecurityGroupChange" +log_metric_nacl = "crossfeed-staging-NACLChange" +log_metric_network_gateway = "crossfeed-staging-NetworkGatewayChange" +log_metric_route_table = "crossfeed-staging-RouteTableChange" +log_metric_vpc = "crossfeed-staging-VPCChange" +log_metric_ec2_shutdown = "crossfeed-staging-EC2Shutdown" +log_metric_db_shutdown = "crossfeed-staging-DBShutdown" +log_metric_db_deletion = "crossfeed-staging-DBDeletion" +matomo_force_index_url = "staging.crossfeed.cyber.dhs.gov/matomo" +sns_topic_alarms = "crossfeed-staging-cis-alarms" +ssm_crossfeed_vpc_name = "/crossfeed/staging/VPC_NAME" +ssm_django_env_kms_arn = "/crossfeed/staging/DJANGO_ENV_KMS_ARN" +ssm_lambda_subnet = "/crossfeed/staging/SUBNET_ID" +ssm_lambda_sg = "/crossfeed/staging/SG_ID" +ssm_worker_subnet = "/crossfeed/staging/WORKER_SUBNET_ID" +ssm_worker_sg = "/crossfeed/staging/WORKER_SG_ID" +ssm_worker_arn = "/crossfeed/staging/WORKER_CLUSTER_ARN" +ssm_db_name = "/crossfeed/staging/DATABASE_NAME" +ssm_db_host = "/crossfeed/staging/DATABASE_HOST" +ssm_db_username = "/crossfeed/staging/DATABASE_USER" +ssm_db_password = "/crossfeed/staging/DATABASE_PASSWORD" +ssm_pe_db_name = "/crossfeed/staging/PE_DB_NAME" +ssm_pe_db_username = "/crossfeed/staging/PE_DB_USERNAME" +ssm_pe_db_password = "/crossfeed/staging/PE_DB_PASSWORD" +ssm_matomo_db_password = "/crossfeed/staging/MATOMO_DATABASE_PASSWORD" +ssm_worker_signature_public_key = "/crossfeed/staging/WORKER_SIGNATURE_PUBLIC_KEY" +ssm_worker_signature_private_key = "/crossfeed/staging/WORKER_SIGNATURE_PRIVATE_KEY" +ssm_censys_api_id = "/crossfeed/staging/CENSYS_API_ID" +ssm_censys_api_secret = "/crossfeed/staging/CENSYS_API_SECRET" +ssm_shodan_api_key = "/crossfeed/staging/SHODAN_API_KEY" +ssm_shodan_ip_chunk_size = "/crossfeed/staging/SHODAN_IP_CHUNK_SIZE" +ssm_shodan_query_days_back = "/crossfeed/staging/SHODAN_QUERY_DAYS_BACK" +ssm_pe_shodan_api_keys = "/crossfeed/staging/PE_SHODAN_API_KEYS" +ssm_wiz_registry_secret_arn = "/crossfeed/staging/WIZ_REGISTRY_CREDENTIAL_SECRET_ARN" +ssm_wiz_service_account_secret_arn = "/crossfeed/staging/WIZ_SERVICE_ACCOUNT_SECRET_ARN" +ssm_wiz_http_proxy_cert_secret_arn = "/crossfeed/staging/WIZ_HTTP_PROXY_CERT_SECRET_ARN" +ssm_sixgill_client_id = "/crossfeed/staging/SIXGILL_CLIENT_ID" +ssm_sixgill_client_secret = "/crossfeed/staging/SIXGILL_CLIENT_SECRET" +ssm_lg_api_key = "/crossfeed/staging/LG_API_KEY" +ssm_lg_workspace_name = "/crossfeed/staging/LG_WORKSPACE_NAME" +ssm_ses_email_identity_arn = "/crossfeed/staging/SES_EMAIL_IDENTITY_ARN" +ssm_worker_kms_keys = "/crossfeed/staging/WORKER_KMS_KEYS" +db_group_name = "crossfeed-staging-db-group" +worker_ecs_repository_name = "crossfeed-staging-worker" +worker_ecs_cluster_name = "crossfeed-staging-worker" +worker_ecs_task_definition_family = "crossfeed-staging-worker" +worker_ecs_log_group_name = "crossfeed-staging-worker" +worker_ecs_role_name = "crossfeed-staging-worker" +logging_bucket_name = "cisa-crossfeed-staging-logging" +export_bucket_name = "cisa-crossfeed-staging-exports" +reports_bucket_name = "cisa-crossfeed-staging-reports" +pe_db_backups_bucket_name = "cisa-crossfeed-staging-pe-db-backups" +user_pool_name = "crossfeed-staging" +user_pool_domain = "crossfeed-staging" +ssm_user_pool_id = "/crossfeed/staging/USER_POOL_ID" +ssm_user_pool_client_id = "/crossfeed/staging/USER_POOL_CLIENT_ID" +ses_support_email_sender = "noreply@staging.crossfeed.cyber.dhs.gov" +ses_support_email_replyto = "vulnerability@cisa.dhs.gov" +matomo_availability_zone = "us-gov-east-1b" +matomo_ecs_cluster_name = "crossfeed-matomo-staging" +matomo_ecs_task_definition_family = "crossfeed-matomo-staging" +matomo_ecs_log_group_name = "crossfeed-matomo-staging" +matomo_db_name = "crossfeed-matomo-staging" +matomo_db_instance_class = "db.m5.xlarge" +matomo_ecs_role_name = "crossfeed-matomo-staging" +es_instance_type = "t3.small.elasticsearch" +es_instance_count = 3 +es_instance_volume_size = 100 +create_db_accessor_instance = false +create_pe_instance = false +db_accessor_instance_class = "t3.2xlarge" +create_elk_instance = false +elk_instance_class = "t3.2xlarge" +ami_id = "ami-0a1445a13e666a557" +cloudtrail_name = "crossfeed-staging-all-events" +cloudtrail_bucket_name = "cisa-crossfeed-staging-cloudtrail" +cloudtrail_role_name = "crossfeed-staging-cloudtrail" +cloudtrail_log_group_name = "crossfeed-staging-cloudtrail" +cloudwatch_bucket_name = "cisa-crossfeed-staging-cloudwatch" +cloudwatch_log_group_name = "crossfeed-staging-cloudwatch-bucket" +es_instance_master_count = 3 +ssm_vpc_id = "/LZ/VPC_ID" +ssm_vpc_cidr_block = "/LZ/VPC_CIDR_BLOCK" +ssm_sctask_cidr_block = "/LZ/SCTASK_CIDR_BLOCK" +ssm_route_table_endpoints_id = "/LZ/ROUTE_TABLE_ENDPOINTS_ID" +ssm_route_table_private_A_id = "/LZ/ROUTE_TABLE_PRIVATE_A_ID" +ssm_route_table_private_B_id = "/LZ/ROUTE_TABLE_PRIVATE_B_ID" +ssm_route_table_private_C_id = "/LZ/ROUTE_TABLE_PRIVATE_C_ID" +ssm_subnet_backend_id = "/LZ/SUBNET_ENDPOINT_A_ID" +ssm_subnet_worker_id = "/LZ/SUBNET_ENDPOINT_B_ID" +ssm_subnet_matomo_id = "/LZ/SUBNET_ENDPOINT_C_ID" +ssm_subnet_db_1_id = "/LZ/SUBNET_PRIVATE_A_ID" +ssm_subnet_db_2_id = "/LZ/SUBNET_PRIVATE_B_ID" +ssm_subnet_es_id = "/LZ/SUBNET_PRIVATE_C_ID" +ssm_mdl_name = "/crossfeed/staging/MDL_NAME" +ssm_mdl_username = "/crossfeed/staging/MDL_USERNAME" +ssm_mdl_password = "/crossfeed/staging/MDL_PASSWORD" +ssm_redshift_host = "/crossfeed/staging/REDSHIFT_HOST" +ssm_redshift_database = "/crossfeed/staging/REDSHIFT_DATABASE" +ssm_redshift_user = "/crossfeed/staging/REDSHIFT_USER" +ssm_redshift_password = "/crossfeed/staging/REDSHIFT_PASSWORD" +ssm_pe_api_key = "/crossfeed/staging/PE_API_KEY" +ssm_pe_api_url = "/crossfeed/staging/PE_API_URL" +ssm_mailer_arn = "/crossfeed/staging/MAILER_ARN" +ssm_cf_api_key = "/crossfeed/staging/CF_API_KEY" +ssm_checksum_salt = "/crossfeed/staging/CHECKSUM_SALT" +create_elasticache_cluster = true +ssm_dmz_api_key = "/crossfeed/staging/DMZ_API_KEY" +ssm_vs_pull_date_range = "/crossfeed/staging/VS_PULL_DATE_RANGE" +ssm_latest_port_scan_cutoff = "/crossfeed/staging/LATEST_PORT_SCAN_CUTOFF" +ssm_dmz_sync_endpoint = "/crossfeed/staging/MDL_SYNC_ENDPOINT" +create_email_sender_instance = false +email_sender_instance_type = "t3.small" +create_open_cti_instance = true +open_cti_instance_type = "m5.4xlarge" # matches stage-cd's real, already-running instance -- confirm availability in this gov-cloud region/AZ before relying on it +# Was unset here (falling back to var.open_cti_host's intentionally-invalid "" default) until +# now -- the live instance's actual OPENCTI_HOST=localhost was coming from open-cti/env.static's +# own hardcoded STABLE IDENTIFIERS line, not from Terraform. VPN access for +# demo-staging-cyber-hygiene.cisa.dhs.gov requires this to be the real external hostname so +# APP__BASE_URL generates correct links -- see open_cti_host's variable description. +open_cti_host = "demo-staging-cyber-hygiene.cisa.dhs.gov" crossfeed_playwright = "crossfeed-playwright-staging" automated_test_reports_bucket_name = "cisa-crossfeed-staging-automated-test-reports" ssm_qualys_username = "/crossfeed/staging/QUALYS_USERNAME" diff --git a/infrastructure/vars.tf b/infrastructure/vars.tf index c14f7a8bd..e7245d210 100644 --- a/infrastructure/vars.tf +++ b/infrastructure/vars.tf @@ -688,6 +688,10 @@ variable "open_cti_secret_keys" { "CONNECTOR_QUALYS_CVE_ENRICHMENT_API_KEY", "CONNECTOR_SHODAN_API_KEY", "CONNECTOR_VULNCHECK_API_KEY", + "CONNECTOR_VS_ORG_BOOTSTRAP_API_KEY", + "CONNECTOR_VS_TICKET_INGESTION_API_KEY", + "CONNECTOR_VS_VULNSCAN_ENRICHMENT_API_KEY", + "CONNECTOR_VS_PORT_INVENTORY_API_KEY", "MINIO_ROOT_PASSWORD", "NVD_API_KEY", "OPENCTI_ADMIN_PASSWORD", diff --git a/mypy.ini b/mypy.ini new file mode 100644 index 000000000..080d0baac --- /dev/null +++ b/mypy.ini @@ -0,0 +1,17 @@ +[mypy] + +# pycti and stix2 (open-cti/connectors/vs-org-bootstrap/) have no stub packages available on +# PyPI/typeshed at all -- unlike types-PyYAML/types-psycopg2 (added to .pre-commit-config.yaml's +# mypy hook instead, since those genuinely exist), there's no "pip install types-X" that fixes +# this for these two. Scoped narrowly rather than a repo-wide ignore_missing_imports. +[mypy-pycti.*] +ignore_missing_imports = True + +[mypy-stix2.*] +ignore_missing_imports = True + +# boto3 has real stubs available (boto3-stubs), but they're a heavy, per-service package and +# boto3 is already imported repo-wide in backend/ without them -- matching that existing +# precedent here rather than introducing a different standard for just this one new directory. +[mypy-boto3.*] +ignore_missing_imports = True diff --git a/open-cti/OpenCTI-connector.md b/open-cti/OpenCTI-connector.md new file mode 100644 index 000000000..8929be1b1 --- /dev/null +++ b/open-cti/OpenCTI-connector.md @@ -0,0 +1,812 @@ +# OpenCTI VS (VulnScanning) Connector — Planning Doc + +Living doc for planning a custom OpenCTI connector that ingests VulnScanning ("VS") data. Update +this as understanding solidifies and as implementation decisions get made — it's meant to track +reality and open questions, not be a one-time snapshot. See also [`open-cti/STATUS.md`](STATUS.md) +for the separate (unrelated) EC2 bootstrap-automation / gov-cloud migration thread. + +## 1. What "VS data" actually is + +"VS" = CISA's VulnScanning service (Nessus/nmap-based scanning of stakeholder-owned IP space). +The data does **not** come from VS directly — it's synced from **Redshift** into this repo's +Postgres RDS instance, and *that* Postgres copy (`mini_data_lake`) is what the connector will read +from. Chain: + +```text +VS scanners --> Redshift (vmtableau schema) --> [two separate sync tasks] --> RDS Postgres (mini_data_lake DB) --> [new] OpenCTI connectors --> OpenCTI +``` + +**It's two separate sync tasks, not one** — this matters for the connectors' own scheduling/ +dependency assumptions: + +- **`VulnScanningSync`** + ([`vulnScanningSync.py`](../backend/src/xfd_django/xfd_api/tasks/vulnScanningSync.py)) — pulls + `vmtableau.vuln_scans`, `.port_scans`, `.tickets` from Redshift into `VulnScan`/`PortScan`/ + `LatestPortScan`/`Ticket`. This is the task Connectors A and C ultimately depend on. Notably, it + does **not** sync organizations itself — it only *reads* already-populated `Organization` rows + via a plain Postgres lookup + (`fetch_org_id_dict_fast()` in + [`vs_requests.py:38`](../backend/src/xfd_django/xfd_api/tasks/utils/vs_requests.py#L38)), on the + assumption that org sync has already run. +- **`vs_org_sync`** ([`vs_org_sync.py`](../backend/src/xfd_django/xfd_api/tasks/vs_org_sync.py)) — + the task that *actually* pulls `vmtableau.requests` from Redshift + (`fetch_orgs_from_redshift()` in + [`vs_requests.py:29`](../backend/src/xfd_django/xfd_api/tasks/utils/vs_requests.py#L29)) and + populates/updates `Organization`, `Sector`, `Cidr`/`CidrOrgs`, and `Location`. Same run also + retires unseen CIDRs (`flag_cidr_changes()`), re-links IPs to CIDRs + (`bulk_assign_ips_to_cidrs()`), rebuilds `HostSummary`, refreshes `Cidr.live_ips`, and — + separately from all the Postgres-side work — **pushes org/CIDR/sector data out to a "DMZ" HTTP + sync endpoint** (`send_organizations_to_dmz()` in + [`vs_send_orgs_to_dmz.py`](../backend/src/xfd_django/xfd_api/tasks/utils/vs_send_orgs_to_dmz.py)). + That last step is one-way, outbound, HTTP-based (not a DB replication) — it confirms + `mini_data_lake` in this (LZ) environment is the canonical source for org data, and the DMZ side + is a downstream consumer, consistent with where OpenCTI's own DB connectivity already points + (§2). + +Practical implication for the connectors: **Identity (organization) data in OpenCTI is only as +fresh as `vs_org_sync`'s last run, which is an independent schedule from `VulnScanningSync`'s.** A +new/renamed org, or a CIDR change, won't be visible to any of the other three connectors until +`vs_org_sync` has run — worth confirming that cadence alongside `VulnScanningSync`'s (§6, both +still unresolved). + +Redshift source tables referenced across both sync tasks (schema `vmtableau`): `requests` (orgs), +`vuln_scans`, `port_scans`, `hosts`, `tickets`. The connectors should **not** talk to Redshift at +all — they only need the `mini_data_lake` Postgres side. + +## 2. Where the data lives (connectivity) + +- Same RDS instance as the main Crossfeed app DB (`aws_db_instance.db` in + [`infrastructure/database.tf`](../infrastructure/database.tf)), Postgres 17.9, but a **separate + database** on that instance: `mini_data_lake` (env var `MDL_NAME`), distinct from the app's + `crossfeed-stage-db`. See `DATABASES["mini_data_lake"]` in + [`backend/src/xfd_django/xfd_django/settings.py:145`](../backend/src/xfd_django/xfd_django/settings.py#L145). +- **No read replica exists** — the connector will query the same primary the app and the + VulnScanningSync bulk-writer use. Worth flagging as a load concern (see §5). +- **IAM DB auth is already scaffolded for OpenCTI specifically**, in + [`infrastructure/open_cti.tf`](../infrastructure/open_cti.tf): + - `aws_iam_role_policy.open_cti_rds_iam_auth` grants OpenCTI's EC2 instance role + `rds-db:connect` scoped to one Postgres role, `var.open_cti_db_username` (default + `open_cti`), via short-lived IAM auth tokens instead of a static password. + - `aws_security_group.open_cti_db_access` opens `var.db_port` (5432) from OpenCTI's EC2 SG to + the DB instance. + - **Only wired up in the LZ (`!is_dmz`, i.e. gov-cloud `stage`/`prod`) branch** — stage-cd + (commercial DMZ) currently has *no* DB connectivity of this kind. Per + [`open-cti/STATUS.md`](STATUS.md) the whole instance is mid-migration from stage-cd into the + LZ; the VS connector likely only becomes practical to run once that migration lands (or the + same IAM-auth/SG pattern gets backported to stage-cd). + - Still needed, out-of-band (not Terraform-managed): the actual `open_cti` Postgres role and a + `GRANT rds_iam TO open_cti` in the DB itself, plus read grants on whatever `mini_data_lake` + tables the connector needs. Nothing here yet suggests that grant has been done — confirm + before assuming connectivity works. +- Auth model for the connector to plan around: **IAM database authentication**, not a static + password — generate a token per connection (`aws rds generate-db-auth-token`), over TLS + (required for IAM auth regardless of `rds.force_ssl`). Tokens are short-lived (~15 min), so the + connector needs to refresh them per-connection/on-reconnect, not cache one long-term. + +## 3. Django ORM model = de facto schema reference + +The `mini_data_lake` tables are defined as Django models in +[`backend/src/xfd_django/xfd_mini_dl/models.py`](../backend/src/xfd_django/xfd_mini_dl/models.py) +(~7000 lines, `managed = False` in most `Meta` classes — Django doesn't own migrations for this +DB, it's describing an externally-managed schema). This is the fastest way to get accurate column +names/types/nullability without a live DB connection. Core VS-relevant tables: + +| Model | `db_table` | PK | Purpose | +|---|---|---|---| +| `VulnScan` | `vuln_scan` | `id` (string) | Raw per-finding vuln scan record (1 row per vuln instance per host/port/time) | +| `VulnScanSummary` | `vuln_scan_summary` | (`summary_date`, `organization`) | Daily rollup per org: severity counts, KEV counts, top CVEs, top risky hosts | +| `Ticket` | `ticket` | `id` (string) | The **stateful** vuln record — open/closed lifecycle, false-positive flag, KEV flag, risky-service flag. Distinct from `VulnScan` (see §4) | +| `TicketEvent` | (n/a, not yet inspected) | — | Presumably ticket state-change history | +| `Host` | `host` | `id` (string) | Per-IP scan-lifecycle metadata (stage, priority, last scan timestamps) | +| `HostSummary` | `host_summary` | (`organization`, `summary_date`) | Daily rollup of host scan status counts per org | +| `PortScan` | `port_scan` | `id` (string) | Raw per-port-scan record | +| `LatestPortScan` | `latest_port_scan` | UUID | Materialized "current open port" per (org, ip, port, protocol) — dedup of `PortScan` | +| `PortScanSummary` / `PortScanServiceSummary` | `port_scan_summary` / `port_scan_service_summary` | (org, date[, service]) | Daily rollups | +| `Ip` | `ip` | UUID | Canonical IP record per org; `ip_hash` used elsewhere for privacy-preserving joins | +| `IpsSubs` | `ips_subs` | UUID | IP ↔ subdomain link table | +| `Cidr` / `CidrOrgs` | `cidr` / `cidr_orgs` | UUID | Org-owned network blocks, used to compute `assets_owned_count` | +| `Cve` | `cve` | UUID | CVE master record — CVSS v2/v3/v4, weaknesses (CWE), references, CNA/ADP JSON blobs | +| `CveSsvc` | (1:1 with `Cve`) | — | SSVC triplet (Exploitation/Automatable/Technical Impact) + ADP provenance | +| `CisaKevCatalog` | `cisa_kev_catalog` | UUID | Full CISA KEV catalog (not VS-specific, but joined against for `is_kev`) | +| `Organization` | `organization` | UUID | Stakeholder org; `acronym` is the join key back to Redshift's `owner`/`_id` fields | +| `RiskyServiceGroup` / `NMIServiceGroup` | — | — | Lookup tables mapping service names to risk categories, applied during sync | + +Full field-level detail for the highest-value tables (`VulnScan`, `Ticket`, `Organization`) is in +models.py at: +[`VulnScan` (L1737)](../backend/src/xfd_django/xfd_mini_dl/models.py#L1737), +[`VulnScanSummary` (L2045)](../backend/src/xfd_django/xfd_mini_dl/models.py#L2045), +[`Ticket` (L2846)](../backend/src/xfd_django/xfd_mini_dl/models.py#L2846), +[`Host`/`HostSummary` (L2466)](../backend/src/xfd_django/xfd_mini_dl/models.py#L2466), +[`PortScan`/`LatestPortScan` (L3116)](../backend/src/xfd_django/xfd_mini_dl/models.py#L3116), +[`Organization` (L617)](../backend/src/xfd_django/xfd_mini_dl/models.py#L617), +[`Cve`/`CveSsvc` (L255)](../backend/src/xfd_django/xfd_mini_dl/models.py#L255). + +## 4. `VulnScan` vs `Ticket` — the key modeling distinction + +This matters a lot for connector design (what becomes an OpenCTI Vulnerability/Observation vs. +what becomes a Case/Incident-style stateful object): + +- **`VulnScan`** — near-raw scanner output. One row per detection event; carries plugin + metadata, CVSS vectors (v2 + v3), solution text, exploit-availability flags, `latest` boolean. + No open/closed lifecycle of its own. +- **`Ticket`** — CyHy's derived, stateful notion of "this vulnerability, on this host, is + currently a problem." Has `is_open`, `opened_timestamp`/`closed_timestamp`, `false_positive`, + `is_kev`/`is_kev_ransomware`, `is_risky`, `vuln_source` (`"nessus"` vs `"nmap"` — i.e. vuln scan + vs port scan derived tickets). **`VulnScanSummary` is built almost entirely from `Ticket`, not + `VulnScan`** — see `create_vuln_scan_summary()` in + [`vs_vuln_scans.py:322`](../backend/src/xfd_django/xfd_api/tasks/utils/vs_vuln_scans.py#L322), + which filters `Ticket` on `is_open=True, false_positive in (False, None), vuln_source="nessus"` + as its base "included" set. +- **Decision (2026-08-25): split into two connectors along exactly this line** rather than picking + one table — see §7 for the full architecture. `Ticket` drives ingestion, `VulnScan` drives + enrichment. + +## 5. Sync cadence / freshness / write pattern (informs polling design) + +- `VulnScanningSync` runs on a fixed window, not full-table: default `VS_PULL_DATE_RANGE=2` days, + computed via `freeze_window()` in + [`datetime_utils.py:69`](../backend/src/xfd_django/xfd_api/tasks/utils/datetime_utils.py#L69) + (midnight UTC to midnight UTC, so there's a rolling window, not "since last run" tracking). + Could not find the CloudWatch/EventBridge schedule expression for how often this task itself + fires — only found a CloudWatch dashboard widget referencing it + ([`cloudwatch_dashboards.tf:392`](../infrastructure/cloudwatch_dashboards.tf#L392)). **Open + item: find the actual schedule** (likely daily, given the 2-day window) before assuming how + fresh `mini_data_lake` is at any given moment. +- Writes are bulk (`bulk_create(..., ignore_conflicts=True)`), chunked (10k row chunks for vuln + scans, keyset-paginated from Redshift), and organization-scoped per run + (`org_list`/`org_id_dict` passed into the task event). This means the connector could see + large bursts of new/updated rows within short windows right after a sync run, not a steady + trickle — relevant for polling interval / batch-size choices. +- No explicit "updated_at" watermark column on `VulnScan` itself (`vuln_detection_timestamp` is + the closest thing — mirrors Redshift's `time` column). `Ticket.updated_timestamp` and + `PortScan.time_scanned` / `LatestPortScan.time_scanned` are the more reliable incremental-sync + anchors. +- **Verified: no hard deletes.** Grepped the task codebase for `.delete()` calls against `Ticket`, + `VulnScan`, `PortScan`/`LatestPortScan` — none exist (the only model-deletion patterns found + anywhere in `xfd_api/tasks/` are unrelated: `ScanResult` pruning, `CisaKevCatalog`'s own + truncate-and-reload, `Blocklist`/`User` cleanup). This matters because it rules out the one + failure mode a pure timestamp-watermark poll genuinely can't handle — a row vanishing outright + between polls with no trace. Connector A and C's incremental strategies (§7a, §7c) only have to + account for state *transitions* (open→closed, current→stale), not disappearance, and that's now + a checked assumption rather than an implicit one. + +## 6. Constraints & open questions to resolve before designing the connector + +#### Connectivity / access + +- [ ] Confirm the `open_cti` Postgres role + `GRANT rds_iam TO open_cti` actually exist in + `mini_data_lake` today (Terraform only creates the *IAM* side of this grant, not the DB-side + role — see §2). +- [ ] Confirm which environment (stage-cd DMZ vs. LZ stage/prod) the connector is meant to target + first — DB connectivity as currently built only exists in the LZ branch. +- [ ] Decide connector auth refresh strategy for IAM tokens (~15 min TTL) — connection-pooling + approach matters here (a long-lived pool needs per-checkout token refresh, not a single token + reused for the pool's lifetime). + +#### Data scope / modeling + +- [x] `Ticket` vs `VulnScan` as the primary sync source (§4) — resolved by splitting into two + connectors, see §7. +- [ ] Multi-tenancy: does the connector pull VS data for *all* organizations, or a filtered subset + (e.g. only orgs with `vs_stakeholder=True` and `retired=False` on `Organization`)? All the sync + task's own org-scoping happens by list of org acronyms passed in per Lambda invocation — the + connector will need its own equivalent org-selection logic against `Organization`. +- [ ] KEV overlay: `CisaKevCatalog` is a separate table joined in only at summary-build time + (`Ticket.is_kev`) — decide whether the connector re-derives/join this itself or trusts the + already-flagged `Ticket.is_kev` / `is_kev_ransomware` columns. +- [ ] CVE enrichment: `Cve`/`CveSsvc` carry CVSS v2/v3/v4 + SSVC + CNA/ADP raw JSON — decide how + much of this rides along vs. being left to OpenCTI's own CVE/NVD connectors to fill in + (potential duplicate-enrichment risk if both write CVE objects). +- [ ] Several fields in the model have literal `"????"`/`"??????"` help text (e.g. + `VulnScan.cisa_known_exploited`, `.asset_inventory`, `.cisco_bug_id`, `.cisco_sa`, + `WasFindings.potential`) — meaning is genuinely unclear even to whoever wrote the model + docstrings; don't build connector logic that depends on these without checking real data first. + +#### Load / operational + +- [ ] No read replica exists (§2) — querying `mini_data_lake` from the connector hits the same + primary the app and the bulk sync writer use. Needs a load-conscious query pattern (indexed + incremental pulls, not table scans) — see indexes already defined on `Ticket` + (`tickets_is_open_idx`, `ticket_last_seen_idx`) and `PortScan`/`LatestPortScan` for what's cheap + to filter on today. +- [ ] Actual `VulnScanningSync` **and** `vs_org_sync` schedule/cadence (§1, §5) — both unresolved. + `vs_org_sync` populates `Organization`/`Cidr`/`Sector` independently of `VulnScanningSync`, so + Identity freshness in OpenCTI depends on finding *that* cadence too, not just the ticket/scan + one. +- [ ] Confirm the acronym-based External Reference contract in §7d actually resolves reliably — + i.e. no pre-existing OpenCTI Identities for these orgs already exist under a *different* key + (e.g. from manual analyst entry) that Connector D's upserts would end up duplicating instead of + matching. +- [ ] Check the actual overlap between CISA's ~20 VS sector acronyms/country-state `Location` + values and whatever `connector-opencti` (the built-in "Datasets" connector, already running — + see §7d) seeded by default. Decide the name-matching rule Connector D uses to reuse existing + `Identity`(sector)/`Location` objects instead of creating a parallel taxonomy. +- [ ] Data volumes: not yet measured. Need row counts / growth rate on `vuln_scan`, `ticket`, + `port_scan` before deciding batch sizes. +- [ ] Connector C specifically: confirm whether a full-table poll of `LatestPortScan` per run + (needed to catch `current` flipping `False` without a timestamp change — see §7c) stays cheap. + `LatestPortScan` is bounded to one row per `(org, ip, port, protocol)` rather than growing + per-scan-event like `PortScan`, but the actual row count across all in-scope orgs isn't known yet. + +## 7. Connector architecture: four connectors + +Splitting `Ticket` (vulnerability ingestion), `VulnScan` (enrichment), `LatestPortScan` +(port/service inventory ingestion), and now `Organization`/`Sector`/`Cidr`/`Location` (org/CIDR +bootstrap) into four separate connectors instead of one monolithic one. This isn't just a +data-source split — it maps cleanly onto OpenCTI's own connector-type taxonomy, and mirrors what's +**already running in this exact deployment**: see +[`open-cti/docker-compose.yml`](docker-compose.yml) — `connector-cisa-known-exploited-vulnerabilities` +(L392) and `connector-vulncheck` (L378) are `EXTERNAL_IMPORT` connectors that poll a source on a +schedule and create/update STIX objects; `connector-qualys-cve-enrichment` (L408) and +`connector-censys-enrichment` (L362) are `INTERNAL_ENRICHMENT` connectors that decorate an +already-existing entity on demand. Our four connectors are the same two shapes (three +`EXTERNAL_IMPORT`, one `INTERNAL_ENRICHMENT`), pointed at `mini_data_lake` instead of a +third-party API. + +**At a glance:** + +| | **A — Ticket Ingestion** | **B — VulnScan Enrichment** | **C — Port/Service Inventory** | **D — Org/CIDR Bootstrap** | +|---|---|---|---|---| +| **Type** | `EXTERNAL_IMPORT` | `INTERNAL_ENRICHMENT` | `EXTERNAL_IMPORT` | `EXTERNAL_IMPORT` | +| **Reads** | `Ticket` | `VulnScan` (per-entity lookup) | `LatestPortScan` | `Organization`, `Sector`, `Cidr`/`CidrOrgs`, `Location` | +| **Trigger** | Scheduled, incremental on `COALESCE(closed_timestamp, updated_timestamp)` | Manual "Enrich" / auto on A's entities | Scheduled, full-table diff per run (§7c) | Scheduled, slow cadence, incremental on `Organization.updated_at` | +| **Creates** | `IPv4-Addr`/`IPv6-Addr`; `Vulnerability` | `Note` only | Reuses A's IP; `Network-Traffic`; `Software` | `Identity` (organization + sector); `Location`; `IPv4-Addr`/`IPv6-Addr` (CIDR-notation) | +| **Depends on** | Connector D (org Identity must exist) | Connector A (entity to enrich must exist) | Connector D (org Identity must exist) | Nothing — this is the root of the dependency chain | + +### 7a. Connector A — VS Ticket Ingestion (`EXTERNAL_IMPORT`) + +**Source:** `Ticket` table, scoped by org and by `is_open` / recency. + +**Trigger/cadence:** Polls on a schedule (`CONNECTOR_DURATION_PERIOD`, same knob +`connector-cisa-known-exploited-vulnerabilities` uses, currently `P1D` there). Should not poll +tighter than `VulnScanningSync`'s own cadence (§5, still unresolved) — no point checking for +updates more often than the source data actually changes. + +**Incremental strategy:** Use OpenCTI's built-in connector state store (`helper.get_state()` / +`set_state()`, keyed by `CONNECTOR_ID` — no separate watermark table needed on our side) to track +the last-seen high-water mark, and query `Ticket` on `COALESCE(closed_timestamp, +updated_timestamp)` — already indexed as `ticket_last_seen_idx` +([`models.py:3044`](../backend/src/xfd_django/xfd_mini_dl/models.py#L3044)). This also means a +ticket flipping *open → closed*, or *not-false-positive → false-positive*, shows up naturally as +an "updated" row on the next poll — the connector doesn't need separate logic to detect closures, +just to react to them (see idempotency below). + +**STIX mapping (per ticket):** + +- `IPv4-Addr` / `IPv6-Addr` observable for `ip_string`. +- `Vulnerability` SDO keyed by `cve_string` when present — **but minimal: name only, no CVSS/ + description of our own.** [`docker-compose.yml`](docker-compose.yml)'s `connector-cve` (L338) is + already running continuously (`CVE_INTERVAL=2` hours, `MAINTAIN_DATA=true`, NVD-sourced back to + `CVE_HISTORY_START_YEAR=2019`) — by the time Connector A sees a `cve_string`, a `Vulnerability` + for that CVE almost certainly already exists (or will shortly), created and kept current by + `connector-cve`. A bare-minimum upsert naturally resolves to the same object rather than forking + a second, VS-flavored copy of the same CVE's data — same "additive, not overwriting" principle + §7b already applies to Connector B, just realized here as "don't populate it at all," since + there's a dedicated connector whose whole job is keeping this object's CVSS/description current + from the authoritative source. For CVE-less tickets (e.g. `nmap`-sourced risky-service tickets, + `vuln_source="nmap"`), still create a `Vulnerability` named from `vuln_name`/`service_name` rather + than skipping the ticket — STIX `Vulnerability` doesn't require a CVE ID, and there's no + `connector-cve` equivalent for these to defer to. +- `Identity` (organization, class `organization`) resolved by `acronym` via the External Reference + Connector D creates (§7d) — **look up, don't blindly create**. Sourced from a *different* task + than the rest of this connector's data — see §1 on `vs_org_sync` — so treat org/CIDR freshness as + its own dependency, owned by Connector D, not something `VulnScanningSync` keeps current as a + side effect. +- Relationship (SRO) connecting the IP observable to the `Vulnerability`, carrying the ticket's + lifecycle as native SRO fields rather than inventing a new container object: `start_time` = + `opened_timestamp`, `stop_time` = `closed_timestamp` (omitted while open). Severity + (`cvss_severity` → low/medium/high/critical), `is_kev`, `is_kev_ransomware`, `is_risky`, + `vuln_source` become labels; an `External Reference` with `source_name="VS"`, + `external_id=Ticket.id` for our own bookkeeping. **Correction from an earlier version of this + doc:** the External Reference does *not* itself make the relationship idempotent — see §10 for + what actually does and what to build instead. +- **False positives:** don't ingest tickets already `false_positive=True`. If a previously-ingested + ticket flips to `false_positive=True` on a later poll, revoke (or delete) its relationship rather + than leaving a stale "still vulnerable" edge in the graph. + +**Case/Incident decision — recommendation: don't use one.** Model tickets as the +observable/Vulnerability/relationship graph above, with no `Case-Incident`/`Case-RFI` container in +the loop. Reasoning: + +- None of the connectors already running in this stack use Cases — Cases in OpenCTI are meant for + analyst-tracked investigations with a bounded scope, and CyHy tickets can number in the + thousands and stay open for months, which would flood the Case list rather than help an analyst. +- SRO `start_time`/`stop_time` already model the open/close lifecycle natively — a Case adds a + second, redundant place to track the same state and a second thing to keep in sync. +- If org-level analyst-facing rollups are wanted later (a periodic "here's this org's vuln posture" + digest), that's better sourced from `VulnScanSummary` — which already *is* that rollup, built + daily by the existing sync task (§3/§4) — as a **separate, later** connector or report generator, + not folded into raw per-ticket ingestion. Flagging this as a phase-2 idea, not in scope now. +- Open to revisiting if you have a specific analyst workflow in mind that needs a Case — flag it + and we can design around it, but there's no default reason to add one here. + +### 7b. Connector B — VulnScan Enrichment (`INTERNAL_ENRICHMENT`) + +**Source:** `VulnScan` table, looked up per-entity rather than polled wholesale. + +**Trigger:** `CONNECTOR_SCOPE` on the `Vulnerability` and/or `IPv4-Addr`/`IPv6-Addr` types Connector +A creates (mirrors `connector-censys-enrichment`'s +`CONNECTOR_SCOPE=IPv4-Addr,IPv6-Addr,X509-Certificate,Domain-Name`). Every existing enrichment +connector in this deployment runs `CONNECTOR_AUTO=false` (manual "Enrich" trigger in the OpenCTI +UI, not automatic on every entity write) — recommend matching that convention to start, since +auto-triggering enrichment on every one of Connector A's writes means one `VulnScan` lookup per +ticket per poll, hitting the same primary DB Connector A just queried. Can revisit once real data +volume (§6) is known. + +**Behavior:** Given a triggered entity (an IP, or a Vulnerability/CVE), query `VulnScan` by +`ip_string` (+ `cve_string` when scoped to a specific Vulnerability) and surface scanner-level +detail: CVSS v2/v3 vectors, `plugin_name`/`plugin_id`/`plugin_family`, `solution`, `synopsis`, +`description`, `exploit_available`/`exploitability_ease`, `risk_factor`, and `see_also`/`xref` as +external references. + +**Output shape — additive, not overwriting:** write enrichment as a `Note` attached to the +triggered entity (same pattern Qualys/Censys enrichment connectors use), rather than mutating the +`Vulnerability` SDO's own CVSS fields directly. This resolves the "duplicate enrichment" concern +in §6 — `connector-vulncheck` and `connector-qualys-cve-enrichment` already write to `Vulnerability` +objects for CVE data; Connector B shouldn't clobber that by overwriting the same fields with VS's +Nessus-flavored numbers. If a field is genuinely worth promoting onto the SDO itself (e.g. +`exploit_available` if no other connector supplies it), that's a targeted, individually-justified +addition, not a blanket overwrite policy. + +**Dependency note:** Connector B only has something to enrich once Connector A (or some other +process) has created the `Vulnerability`/observable in the first place — it's not a standalone +ingestion path. + +### 7c. Connector C — VS Port/Service Inventory (`EXTERNAL_IMPORT`) + +**Source:** `LatestPortScan` — deliberately **not** raw `PortScan` (an unbounded, append-only scan +history log). `LatestPortScan` is already deduped to one row per `(organization, ip, port, +protocol)` by the sync task's own upsert +([`insert_port_scans_sql()`](../backend/src/xfd_django/xfd_api/tasks/utils/vs_port_scans.py#L230)), +so it's the "current state of the world" table — exactly the shape an inventory-style connector +wants, and much cheaper to poll than the raw log. + +**Why this is a separate connector from 7a, not folded into it:** `Ticket` already captures a +*curated subset* of port-scan findings — rows where `vuln_source="nmap"` and `is_risky=True` +(service name matched `"Potentially Risky Service Detected:"`, see +[`vs_tickets.py:316`](../backend/src/xfd_django/xfd_api/tasks/utils/vs_tickets.py#L316)) already +flow into OpenCTI via Connector A as `Vulnerability` relationships. Connector C's job is the +**complementary full inventory** — every open port/service, risky or not — which is a materially +different STIX shape (network/asset exposure, not vulnerability findings) and, as below, a +different incremental-sync strategy. Keeping them separate keeps each connector's polling/diffing +logic single-purpose, matching this deployment's existing one-connector-per-concern convention; +they can still share a small internal helper module (e.g. "resolve org Identity," "get-or-create +IP observable") without merging into one process. + +**Incremental strategy — cannot be a simple timestamp watermark, unlike Connector A.** This is the +important gotcha: a port going stale is driven by +[`mark_stale_latest_port_scans()`](../backend/src/xfd_django/xfd_api/tasks/utils/vs_port_scans.py#L771), +which flips `current` to `False` after 14 days (`LATEST_PORT_SCAN_CUTOFF`) **without touching +`time_scanned`**. A pure `WHERE time_scanned > last_watermark` poll would never see that +transition, and the connector would keep reporting a closed port as open indefinitely. Recommended +approach instead: since `LatestPortScan` is bounded (one row per unique key, not per scan event), +**poll the full current-scope table each run** — all `LatestPortScan` rows for in-scope +organizations — and diff against the connector's own last-known `(org, ip, port, protocol) → +current` map (kept in OpenCTI connector state) to compute newly-opened / still-open / newly-closed +per run, rather than relying on `time_scanned` as a change indicator. Needs the row-count question +in §6 answered to confirm "full poll per run" stays cheap as org coverage grows. + +**Note on first-seen:** `LatestPortScan` only retains the *latest* `time_scanned` for a key (it's +an upsert target), not a first-observed timestamp. The connector needs to preserve `start_time` on +the OpenCTI-side relationship itself once set (only write it on first creation of that key, never +overwrite on subsequent updates) — the source table can't supply "first seen" after the fact. + +**STIX mapping (per `LatestPortScan` row):** + +- Reuses the same `IPv4-Addr`/`IPv6-Addr` observable Connector A creates — STIX SCOs for these + types get deterministic, value-derived IDs, so OpenCTI naturally merges them into the same + object with no coordination code needed between the two connectors. +- `Network-Traffic` SCO per `(ip, port, protocol)`: `src_ref` = the IP observable, `src_port` = + `port`, `protocols` = `[protocol]` (kept to valid IANA-style tcp/udp values — `service_name` is + often not a valid STIX protocol string, so it goes on a label/description instead, not crammed + into `protocols`). +- `Software` SCO when `service_product`/`service_version`/`service_cpe` are present — `service_cpe` + maps directly onto STIX `Software.cpe`, `service_product`/`service_version` onto `name`/`version`. + Related to the `Network-Traffic` object via a relationship (STIX 2.1 allows SROs between SCOs). +- Labels reused verbatim from the same fields `Ticket` also carries (`risky_service_group`, + `nmi_service_group`, `source`, `state`) — keeps filtering/labeling consistent between what + Connector A and Connector C both produce. +- Relationship lifecycle: `start_time` set once (see above), `stop_time` set once `current` flips + to `False` on a poll (using that poll's run time, since the source itself doesn't timestamp the + transition either). Same idempotency caveat as Connector A — see §10 for the actual mechanism to + rely on. +- Same organization-`Identity` resolution/reuse pattern as Connector A — resolved by `acronym` via + Connector D's External Reference (§7d). + +**Out of scope for now:** `PortScanSummary`/`PortScanServiceSummary` (daily org rollups) — same +"phase 2, seed a rollup/Case from the existing summary tables" bucket as `VulnScanSummary` in §7a, +not part of raw inventory ingestion. + +### 7d. Connector D — VS Organization & CIDR Bootstrap (`EXTERNAL_IMPORT`) + +**Why this exists:** Connectors A and C both resolve an org `Identity` by "look up, don't create," +which was only ever an assumption that one exists. Nothing owned actually creating/maintaining it +from VS's own org data. This connector is that owner — and the root of the dependency chain: A and +C both need it to have run at least once for a given org before their own polls produce anything +useful. + +**Source:** `Organization`, `Sector`, `Cidr`/`CidrOrgs`, `Location` — all populated by `vs_org_sync` +from Redshift's `vmtableau.requests` (§1). Like the other connectors, this one only reads +`mini_data_lake`; it never talks to Redshift. + +**Cadence — deliberately slower than A/B/C:** org/sector/CIDR data is reference data (stakeholder +onboarding, network ownership) that churns far less often than scan findings. Unlike `Ticket`/ +`LatestPortScan`, `Organization` actually has a plain `updated_at` auto-now column +([`models.py`](../backend/src/xfd_django/xfd_mini_dl/models.py#L630-634)) — no watermark gotchas +here, a simple `updated_at > last_watermark` poll works. `Sector` has no timestamp at all, but it's +a small, low-cardinality table (CISA's sector taxonomy, ~20 rows) — cheap to poll in full every run +rather than tracking incrementally. `CidrOrgs.last_seen`/`current` mirrors the same open/closed +pattern `Ticket.is_open` and `LatestPortScan.current` use, for detecting a CIDR no longer claimed +by an org. + +**STIX mapping:** + +- `Organization` → `Identity` (class `organization`): `name` = `Organization.name`. Use + `acronym` as the durable join key — carried as an `External Reference` + (`source_name="VS"`, `external_id=Organization.acronym`) on the Identity, the same + idempotency pattern Connector A uses for tickets — so Connectors A/C can resolve "the Identity + for org X" by that stable reference instead of fuzzy-matching on `name` (which can drift in + formatting; `acronym` is the actual join key this whole data model already keys on, e.g. + `fetch_org_id_dict_fast()`). Labels for `stakeholder`/`retired`/`vs_stakeholder`-equivalent + flags. `Organization.parent` → a `part-of` relationship from child Identity to parent Identity. +- `Sector` → `Identity` (class `sector`): `name`, `acronym` as its own External Reference. Each + member org gets a `part-of` relationship from its Organization Identity to the Sector Identity + (from the `Sector.organizations` M2M). **Collision risk found:** + [`docker-compose.yml`](docker-compose.yml)'s `connector-opencti` (L430, `CONNECTOR_SCOPE= + marking-definition,identity,location`) is OpenCTI's own built-in "Datasets" connector, and it + already seeds a default set of `identity` (including generic sectors) and `location` objects on + install. There's a real chance CISA's ~20 VS sector acronyms substantially overlap with whatever + generic sector taxonomy that connector already seeded. **Look up by name before creating** — a + duplicate parallel sector taxonomy is worse than reusing what's already there — and confirm the + actual overlap during ground-truthing (§8) rather than assuming either way. +- `Location` → OpenCTI `Location` entity (country/administrative-area, per STIX 2.1's location + extensions OpenCTI supports), built from `country`/`country_abrv`/`state`/`state_abrv`/`county`. + Related to the Organization Identity via `located-at`. Low cardinality (one per `gnis_id`) — + resolve/create as a side effect of processing each org row rather than polling `Location` + independently. Same collision risk as `Sector` above — `connector-opencti` likely already seeded + country-level `Location` entities; look up (e.g. by country/state name) before creating. +- `Cidr` → `IPv4-Addr`/`IPv6-Addr` observable using the CIDR-notation value directly (STIX 2.1 + explicitly allows a CIDR block as an address SCO's `value`, e.g. `"198.51.100.0/24"` — no custom + observable type needed). Related to the Organization Identity (`related-to`/an "owns"-flavored + relationship), with `start_time` = `CidrOrgs.first_seen` and `stop_time` set once + `CidrOrgs.current` flips to `False` (retired) — same lifecycle-via-SRO pattern as Connectors A + and C, kept consistent across all three ingestion connectors rather than inventing a fourth way + to represent "still active vs. no longer." Same idempotency caveat as A/C — see §10. + +**Resilience note:** given A/C's own polls could in principle run before D has ever seen a +brand-new org, consider a defensive fallback in A/C — if an Identity lookup-by-acronym misses, +create a minimal placeholder Identity (acronym + name only) rather than dropping the +ticket/port-scan row, and let D's next run enrich it fully via the same upsert-by-External-Reference +path. Avoids a hard ordering dependency turning into dropped data if schedules ever drift. + +## 8. Recommended build order: D → A → B → C + +Dependency graph is simple — D has no dependencies, A and C both depend on D, B depends on A: + +```text + ┌── A (Ticket Ingestion) ──── B (VulnScan Enrichment) +D ──────┤ +(Org/CIDR) └── C (Port/Service Inventory) +``` + +That graph alone only forces D first and B after A — it leaves A-vs-C and B-vs-C open. Recommended +sequence, and why it's **D → A → B → C** rather than, say, D → A → C → B: + +1. **D — Org/CIDR Bootstrap.** Nothing else can produce a correctly-linked object until stakeholder + `Identity`/`Location`/`Sector` data exists. Also the lowest-risk connector to build first: + simple `updated_at` watermark (§7d), no timestamp gotchas like A/C have. Its output (the + acronym-keyed External Reference convention) is a contract every other connector consumes, so + getting it right early matters more than getting to A or C fast. +2. **A — Ticket Ingestion.** The actual point of this whole effort — open, actionable + vulnerabilities in OpenCTI. Straightforward incremental strategy (timestamp watermark, no + gotchas — see the "verified: no hard deletes" note in §5) makes it the natural second build. +3. **B — VulnScan Enrichment.** Small, additive, and low-risk (writes `Note`s, touches nothing + else) — and building it right after A closes the loop on a **complete, useful, shippable + vertical slice** (org data → open vulns → enriched detail) before taking on C's structurally + different, harder problem. Validating the Identity/Vulnerability graph actually enriches + cleanly here also de-risks C, which reuses the same Identity-resolution and IP-observable + patterns. +4. **C — Port/Service Inventory.** Saved for last on purpose: it's the one connector whose + incremental strategy isn't a simple timestamp watermark (§7c's full-poll-per-run diff to catch + `current` flipping without a timestamp change) and whose row-count/performance question (§6) + is still open. Better to tackle that novel piece once the team has already shipped two + watermark-style connectors (D, A) and has real production experience with this DB and OpenCTI's + connector-state store to lean on. + +**Legitimate alternative:** if full asset-inventory coverage matters more up front than enrichment +depth, C and B can swap — both only depend on A/D, not on each other, so D → A → C → B is equally +valid dependency-wise. The order above is a value/risk call, not a hard constraint; revisit if +priorities differ. + +## 9. Development & testing workflow + +The concern driving this section: nothing in the platform stops a connector from flooding the +queue. `rabbitmq.conf` sets `max_message_size = 536870912` (512MB) and `consumer_timeout = +86400000` (24 hours for the `worker` service to ack one message) — RabbitMQ will happily accept +and sit on a huge bundle. The `worker` service (`docker-compose.yml`, L194) runs 3 replicas total, +shared across *every* connector in the stack, not just ours — a runaway pull during dev doesn't +just slow our own iteration, it eats capacity from whatever else is running. Nothing here is a +platform guardrail; it has to be connector-side discipline, built in from the start rather than +bolted on. + +### 9a. Reuse this codebase's own convention: `IS_LOCAL` fixture mode + +This isn't a new idea to introduce — it's already the house style. `IS_LOCAL` is checked in +*nearly every* sync task in this repo (`vulnScanningSync.py`, `vs_org_sync.py`, `vs_requests.py`, +`vs_port_scans.py`, `vs_vuln_scans.py`, and a dozen unrelated tasks besides). Concretely, in +[`query_redshift.py:136-169`](../backend/src/xfd_django/xfd_api/tasks/utils/query_redshift.py#L136), +`fetch_from_redshift()`/`fetch_from_redshift_with_params()` short-circuit under `IS_LOCAL` to +`load_test_data()`, which loads a small canned JSON sample per dataset instead of ever touching +Redshift. + +Our four connectors should do the exact same thing against `mini_data_lake`: an `IS_LOCAL`- (or +`CONNECTOR_LOCAL_FIXTURES`-) gated mode that loads a handful of representative `Ticket`/ +`LatestPortScan`/`Organization`/`Cidr` rows from small local JSON fixture files instead of +querying Postgres at all. Zero DB connection, zero IAM token, zero network — the fastest possible +loop for iterating on the row→STIX mapping logic, which is where most connector bugs will +actually live. + +Getting the fixtures: capture them as part of the ground-truthing session already planned (§10 +step 1) — one supervised, capped pull (`LIMIT 10`, one or two known low-volume test orgs) from +each source table, saved as JSON. This is real government vuln-scanning data even at 10 rows — +review before committing anything derived from it to the repo, same as any other sample-data +handling on this program. + +### 9b. Three separate test loops, from fastest/cheapest to slowest/most-real + +Keeping these distinct (rather than always testing end-to-end) is what actually protects the +queue, because only the last one touches RabbitMQ at all: + +1. **Pure unit tests — no I/O.** Structure each connector's row→STIX mapping as pure functions + (a Postgres row dict in, `stix2` objects out). Test against the §9a fixtures. No DB, no + OpenCTI, no queue. This is where the bulk of iteration should happen — instant feedback, + run on every save. +2. **Dry-run against real data — no queue writes.** A `CONNECTOR_DRY_RUN` mode that runs the real + poll-and-map pipeline against a scoped/capped `mini_data_lake` query (see §9c), but writes the + resulting STIX bundle to a local JSON file (or just logs it) instead of calling the OpenCTI + helper's bundle-submission step — the actual point where a message would hit RabbitMQ. Validates + real column types/data quirks against production data shapes without ever touching the queue. +3. **Real submission — capped and scoped, never "everything."** Only after 1 and 2 look right, + actually run the connector against a real OpenCTI instance — but always through the scoping + levers in §9c, never a bare unscoped run during development. + +### 9c. Scoping levers to build in from day one, not bolt on later + +- **Org allowlist** (`CONNECTOR_VS_TEST_ORG_ACRONYMS` or similar), comma-separated — mirrors the + `org_list` parameter `VulnScanningSync` itself already accepts per invocation. Dev/staging config + pins this to one or two known low-volume orgs; empty/unset = full production scope. Every + connector (A/C/D) should take this from day one, not as an afterthought. +- **Hard row cap** (`CONNECTOR_MAX_ROWS_PER_RUN`) as a second, independent safety net — so a bug in + org-scoping logic can't turn into an accidental full-table pull. +- **Lookback override** for Connector A's incremental poll — force a short window (e.g. "last + hour") during dev instead of picking up whatever backlog has accumulated since the connector + state was last reset. Matters especially the first time a fresh connector state points at a real + environment. +- **Connector C needs this most.** Its whole strategy (§7c) is "poll everything in scope, diff + against last-known state" — without a tight org allowlist, *every single manual dev run* + re-pulls and re-diffs the entire `LatestPortScan` table for every org. This is the sharpest + version of the risk being guarded against here; C should refuse to run with an empty org + allowlist outside of `CONNECTOR_AUTO`-style production config, not just default to "all orgs." + +### 9d. Don't point early dev at the shared instance's queue + +`open-cti/docker-compose.yml` already runs a complete, self-contained stack (OpenCTI platform + +Redis + Elasticsearch + MinIO + RabbitMQ + workers), with both the platform (`${OPENCTI_PORT}`) +and RabbitMQ's management UI (port 15672, `rabbitmq:4.3-management` image) published to the host. +That means a disposable local stack — `docker compose up` against this same compose file, pointed +at throwaway local containers instead of the real EC2 instance — is already one command away, and +`docker compose down -v` is a clean, total reset (wipes RabbitMQ/OpenCTI/Elasticsearch state +entirely) between iterations. No local-dev override file for `open-cti/` exists yet (unlike the +main app's `docker-compose.override.local.yml`) — worth adding one so this isn't reinvented each +time. + +Recommend defaulting to that local stack for the bulk of connector development (loops 2 and 3 in +§9b), and reserving the real EC2 instance for the things a local stack genuinely can't +replicate — namely testing actual IAM DB-auth connectivity to `mini_data_lake` (§2) once that's +wired up. When the shared instance is unavoidable, keep the §9c scoping caps engaged for every run +and watch the RabbitMQ management UI (already enabled, no setup needed) for queue depth so a +runaway push gets caught immediately rather than silently backing up the 3-replica `worker` pool +other connectors also depend on. + +## 10. Connector-side implementation discipline + +"Nothing protects the queue except connector-side discipline" (§9) is really a statement about +correctness generally, not just queue safety — most of what follows is the same handful of +principles applied to idempotency, DB load, and error handling, not a new topic each time. + +### 10a. Idempotency — corrected, and more specific than "attach an External Reference" + +An earlier version of this doc claimed an `External Reference` makes a relationship idempotent +across re-syncs. Checked that against +[OpenCTI's actual deduplication rule](https://docs.opencti.io/latest/usage/deduplication/) and +it's not quite right, which matters a lot here since all three ingestion connectors' lifecycle +modeling (open→closed, current→stale, active→retired) lives entirely on relationships: + +- **What OpenCTI actually keys relationship dedup on:** `relationship_type` + `source_ref` + + `target_ref` + `start_time` (±30 days) + `stop_time` (±30 days) — a fuzzy windowed match. The + External Reference we attach is bookkeeping metadata; it plays no role in whether a resubmission + merges into an existing relationship or creates a new one. +- **The fuzzy window is fine for the common case** (any poll cadence in the daily-ish range this + plan assumes throughout), but **the null→set `stop_time` transition specifically — a ticket open + for months, then finally closing — isn't clearly documented at that boundary.** Not something to + assume either way; verify it empirically against a real OpenCTI instance (the dry-run/local-stack + loop in §9b/§9d is exactly for this) before Connector A ships, since that transition is the single + most common lifecycle event all three ingestion connectors need to get right. +- **The robust pattern, independent of how the fuzzy window actually behaves:** each connector + persists its own `(external key → OpenCTI object ID)` map in connector state — `Ticket.id →` + relationship ID for A, the `(org, ip, port, protocol)` key `→` relationship ID for C, + `Organization.acronym`/`Cidr.network` → `Identity`/observable ID for D — and explicitly updates + the known object (via the STIX ID pycti returns on creation, or a GraphQL lookup by the External + Reference already attached) rather than resubmitting blind and hoping the platform's fuzzy match + finds it. Costs little to build in from the start and turns an assumption into a guarantee. +- SCOs (`IPv4-Addr` etc.) are the one type this doc's dedup claims (§7c) can stay confident about + as written — deterministic per the STIX 2.1 spec itself, not an OpenCTI-specific behavior. SDOs + (`Vulnerability`, `Identity`) use OpenCTI's own content-based "ID contributing properties" per + type — generally reliable, but confirm per-type rather than assuming it extends to every field + we might set, the same way relationships turned out to need a closer look. + +### 10b. Bundle construction — use what pycti already provides, confirm rather than assume its defaults + +pycti's client ships bundle-splitting (`OpenCTIStix2Splitter`, used internally by +`send_stix2_bundle()`) — capping bundle size isn't something to hand-roll. What to actually do: +confirm its default thresholds/behavior against the pycti version this stack pins (couldn't pin +that down from public docs alone) before assuming it does the right thing unconfigured. Separately, +**keep DB page/chunk size and STIX-bundle size as two independent knobs** — how many rows a +connector pulls from `mini_data_lake` in one query (tuned against DB load, §6) and how many STIX +objects go in one message to RabbitMQ (tuned against queue/worker load, §9) are different +constraints; don't let one implicitly set the other. + +### 10c. Attribution and marking — a real open decision, not a default to pick alone + +Two things every one of the four connectors needs to set *consistently*, not ad hoc per-connector: + +- **`createdBy` (author Identity):** all VS-sourced content should trace back to one consistent + system Identity (e.g. "CISA VulnScanning") across all four connectors, not left unset or + inconsistent, so it's filterable/attributable as a single source in OpenCTI. +- **TLP/marking definition — flagging this as a genuine open question, not something to default + silently.** This is government vulnerability-scan data describing specific stakeholder + organizations' live, unpatched exposures. `TLP:CLEAR` is very unlikely to be the right default + for that; something more restricted (`TLP:AMBER` or an internal/custom marking) is the safer + starting assumption, but the actual policy call belongs to whoever owns data-handling rules for + this program, not this planning doc. **Needs an explicit answer before any real data flows**, + not just before "launch" — even the dry-run/local-stack dev loop in §9 should use whatever + marking is eventually decided, so dev habits match production from the start. +- Confidence score: pick one fixed value per connector (existing connectors in + [`docker-compose.yml`](docker-compose.yml) set this per-connector, e.g. `100` for + `connector-opencti`, `75` for `connector-mitre`) — decide once, not per-run or per-row. + +### 10d. Database query discipline + +- **Explicit column projection, never `SELECT *`** — every query should name only the columns a + connector actually maps to STIX. Cheaper on the no-read-replica primary (§2/§6), and more + resilient to schema drift on columns we don't care about. +- **Shape `WHERE` clauses to hit existing indexes**, not whatever's convenient to write — + `ticket_last_seen_idx`, `tickets_is_open_idx` for Connector A, `LatestPortScan`'s compound + `(organization, ip, port, protocol)`/`(ip, current, time_scanned)` indexes for Connector C, + `Organization.updated_at` for D. A query shape that doesn't match one of these forces a + sequential scan on a primary that already has no failover capacity to spare. +- **Resolve Identity/org lookups once per run, not once per row.** Fetch the acronym→Identity + mapping (or whatever Connector D's lookup produces) into memory at the start of a run and reuse + it — avoid N+1 GraphQL calls against OpenCTI itself, a separate rate/latency concern from the DB + side entirely. +- **Reuse this codebase's own connection-pooling pattern** (`SimpleConnectionPool` in + [`query_redshift.py`](../backend/src/xfd_django/xfd_api/tasks/utils/query_redshift.py)) rather + than opening a fresh connection per poll — but remember IAM auth tokens are ~15 minutes (§2), so + pooled connections need per-checkout token refresh, not a connect-once-at-startup assumption. + +### 10e. Error handling — mirror the isolation pattern already used in this codebase + +`create_port_scan_summaries_bulk()` and `create_vuln_scan_summary()` (§3/§4) both isolate failures +per-chunk/per-org — log and continue, don't abort the whole run over one bad row. Every connector +should do the same at row-mapping granularity: one `Ticket`/`LatestPortScan` row with unexpected +data shouldn't sink an entire poll's bundle. This is an existing house convention to follow, not a +new pattern to invent. + +### 10f. State-checkpoint ordering and processing guarantees + +Only advance the connector-state watermark **after** the corresponding bundle has been confirmed +accepted by OpenCTI (via `send_stix2_bundle()`'s work-tracking, not just after building it locally) +— advancing first risks losing rows if the connector crashes mid-run. Prefer **at-least-once** +processing (safe to resend a poll's results if a run is interrupted) over engineering for +exactly-once, which is fragile and, given §10a's idempotent-write pattern, unnecessary — a resent +bundle should be a safe no-op, not a risk to guard against separately. + +### 10g. Scheduling discipline across all four connectors + +Stagger `CONNECTOR_DURATION_PERIOD` start offsets for D/A/C rather than letting them all fire at +the same wall-clock moment — otherwise every scheduled run piles three connectors' queries onto +the same no-read-replica primary (§2) and three ingestion runs' bundles onto the same 3-replica +`worker` pool (§9) simultaneously, for no benefit over spacing them out. + +### 10h. Observability + +Check what OpenCTI's own connector status page (backed by `helper.api.work`) already surfaces +(last run, duration, errors) before building custom metrics on top of it. Where that doesn't +reach — DB-side query duration, actual bundle sizes sent, row-to-STIX-object conversion counts — +mirror the `@cloudwatch_metric()` decorator pattern already used pervasively across the sync tasks +(§3) rather than inventing a separate observability convention for just these four connectors. + +## 11. Implementation status + +**Connector D scaffolded** at [`connectors/vs-org-bootstrap/`](connectors/vs-org-bootstrap/) (see +its own README.md) — config loading with the §10c/§9c fail-closed checks, `mini_data_lake` access +with an `IS_LOCAL` fixture fallback, the full row→STIX mapping, and the state-based idempotency +pattern from §10a. Both test loops from §9b are real and passing, not just described: 14 pure +unit tests (Loop 1) and 3 full-pipeline dry-run tests against fixtures with the OpenCTI helper +stubbed out (Loop 2, no network/queue at all) — `python3 -m pytest tests/ -v` in that directory. +Wired into [`docker-compose.yml`](docker-compose.yml) as `connector-vs-org-bootstrap` (build-based, +since this has no published image). + +Building this against a real, installed `pycti==7.260824.0` (rather than reasoning about the SDK +from memory) surfaced two concrete corrections to earlier sections of this doc: + +- **§10a/§10b upgraded from "confirm" to actually confirmed**, by reading the installed source: + `send_stix2_bundle()` always splits via `OpenCTIStix2Splitter` unless `no_split=True` is passed + (§10b's bundle-splitting question), and its docstring states outright that "OpenCTI always + upserts data by standard id/hash regardless of \[the `update`\] flag" — the `update` kwarg only + matters for a rare ambiguous-multi-match case, not for whether upserting happens at all. This + directly validates §10a's recommended pattern: pin a relationship's id once (via + `pycti.StixCoreRelationship.generate_id`) and reuse it verbatim on every later write, letting the + changed field (e.g. `stop_time`) be the update OpenCTI applies to that same id — proven working + end-to-end in `tests/test_connector_dry_run.py`, including the specific case (a CIDR retiring) + that motivated §10a in the first place. +- **New gotcha, not previously documented: `DateField`-sourced values need timestamp + normalization before use in STIX.** `CidrOrgs.first_seen`/`last_seen` are `DateField` (date-only) + in `mini_data_lake`, and come back as bare dates — which `stix2`'s `TimestampProperty` rejects + outright (`must be a datetime object, date object, or timestamp string in a recognizable + format`, but a bare `"2025-01-01"` string does *not* count as recognizable to it, despite being + ISO 8601). Found by actually running the dry-run test against fixture data, not by inspection — + `mapping.normalize_timestamp()` handles it now. **Worth checking for the same issue on any other + `DateField` column Connectors A/C/B end up touching** (e.g. `VulnScanSummary.summary_date`, + `HostSummary.summary_date`) rather than assuming this was a one-off. + +**Deliberately not done yet, and why:** + +- **Not wired to a real OpenCTI or `mini_data_lake` instance.** Verified as far as is possible + without one: `python3 -m src.main` with `IS_LOCAL=true` correctly loads fixtures, passes + fail-closed config validation, initializes `OpenCTIConnectorHelper`, and fails at the GraphQL + health check against an intentionally-unreachable OpenCTI URL — i.e. it fails at exactly the + first point that genuinely requires infrastructure this session doesn't have, not anywhere in + the connector's own code. Real validation needs §9d's local stack (or real credentials). +- **DB connectivity and the OpenCTI API token are not plumbed through yet.** The + `docker-compose.yml` service references `VS_ORG_BOOTSTRAP_DB_HOST`, `AWS_REGION`, and + `CONNECTOR_VS_ORG_BOOTSTRAP_API_KEY`, none of which exist yet in + [`env.static`](env.static)/[`bootstrap.sh`](bootstrap.sh) or Terraform's + `open_cti_secret_keys` (`infrastructure/vars.tf`). Deliberately not touched in this pass — + `bootstrap.sh` has unrelated in-flight edits on the current branch, and this is real + infra/secrets work worth its own deliberate change, not a drive-by edit alongside connector code. +- **The §7d/§6 sector/location "look up before create" logic is a stubbed `TODO`** in + `connector.py`, not implemented — it creates `Identity`(class=sector)/`Location` objects + unconditionally for now. Needs the actual overlap-with-`connector-opencti`'s-default-taxonomy + question answered first (§10, next steps below) before this is safe to run against a real + instance with real data. + +## 12. Next steps + +1. **Get the TLP/marking policy decision from whoever owns data-handling rules for this program** + (§10c) — the single thing actually blocking Connector D from running anywhere real; everything + else in §11's "not done yet" list can proceed in parallel with this. +2. Stand up the disposable local OpenCTI stack described in §9d (a `docker-compose.override.local.yml` + sibling under `open-cti/`) — the next concrete step for Connector D specifically: it's built and + test-passing, but has never talked to a real OpenCTI. Use it to empirically verify the + null→set `stop_time` relationship-merge behavior (§10a) with real platform round-trips, not + just the stubbed-helper dry run already passing. +3. Plumb `VS_ORG_BOOTSTRAP_DB_HOST`/`AWS_REGION`/`CONNECTOR_VS_ORG_BOOTSTRAP_API_KEY` through + `env.static`/`bootstrap.sh`/Terraform's `open_cti_secret_keys` (§11) — real infra work, + deliberately deferred rather than bundled into the connector code change. +4. Get a live psql session against `mini_data_lake` (once `open_cti` role/grant is confirmed) and + pull real row counts, distinct `owner`/org counts, and samples of `Ticket`/`VulnScan`/ + `LatestPortScan` rows to ground-truth this doc against actual data — `LatestPortScan` row count + is specifically needed to validate Connector C's full-poll-per-run strategy (§7c). While in + there: check the actual `Identity`/`Location` taxonomy already in OpenCTI against CISA's VS + sector list (resolves §11's sector/location `TODO` in `connector.py`), confirm the + `sector_organizations` join-table name assumption (`db.py`'s one unverified query), and replace + Connector D's synthetic test fixtures with real (scrubbed, reviewed) samples from one or two + low-volume test orgs. +5. Resolve the remaining open questions in §6 (org filtering, KEV re-derivation, data volumes, + actual `VulnScanningSync`/`vs_org_sync` schedules). +6. Confirm the Case/Incident recommendation in §7a, or flag the analyst workflow that would + change it. +7. Build Connectors A, C, and B in that order (§8), following the same structure Connector D just + established (`config.py`/`db.py`/`mapping.py`/`connector.py` split, both test loops from §9b + passing before ever running unscoped) and the discipline in §10. diff --git a/open-cti/bootstrap.sh b/open-cti/bootstrap.sh old mode 100644 new mode 100755 index 004ad465f..2244a3d6b --- a/open-cti/bootstrap.sh +++ b/open-cti/bootstrap.sh @@ -35,7 +35,7 @@ cat "$ENV_STATIC" >> "$ENV_TMP" # 2) Per-environment config from Terraform vars. XTM One's two values are optional (inactive on # stage-cd); the rest fail closed on empty. empty_deploy_keys=() -for key in OPENCTI_HOST OPENCTI_ADMIN_EMAIL SMTP_HOSTNAME CENSYS_ORG_ID QUALYS_API_USERNAME; do +for key in OPENCTI_HOST OPENCTI_ADMIN_EMAIL SMTP_HOSTNAME CENSYS_ORG_ID QUALYS_API_USERNAME VS_ORG_BOOTSTRAP_DB_HOST VS_ORG_BOOTSTRAP_AWS_REGION; do [[ -n "${!key:-}" ]] || empty_deploy_keys+=("$key") done if (( ${#empty_deploy_keys[@]} > 0 )); then @@ -49,6 +49,8 @@ fi echo "QUALYS_API_USERNAME=\"${QUALYS_API_USERNAME}\"" echo "XTM_ONE_HOST=\"${XTM_ONE_HOST:-}\"" echo "XTM_ONE_ADMIN_EMAIL=\"${XTM_ONE_ADMIN_EMAIL:-}\"" + echo "VS_ORG_BOOTSTRAP_DB_HOST=\"${VS_ORG_BOOTSTRAP_DB_HOST}\"" + echo "VS_ORG_BOOTSTRAP_AWS_REGION=\"${VS_ORG_BOOTSTRAP_AWS_REGION}\"" } >> "$ENV_TMP" # 3) Secrets. Names fetched in one call (safe -- no whitespace/newlines in names), each value fetched diff --git a/open-cti/connectors/vs-org-bootstrap/Dockerfile b/open-cti/connectors/vs-org-bootstrap/Dockerfile new file mode 100644 index 000000000..e4fe7abb7 --- /dev/null +++ b/open-cti/connectors/vs-org-bootstrap/Dockerfile @@ -0,0 +1,42 @@ +FROM python:3.12-slim + +WORKDIR /opt/connector + +# Zscaler TLS-inspects outbound HTTPS on this network, re-signing it with its own root CA -- the +# EC2 host trusts it, a fresh build container doesn't, hence pip failing here with "self-signed +# certificate in certificate chain" even though the host itself is fine. Appended to the image's +# own CA bundle via Debian's update-ca-certificates (not a wholesale replacement of it), so normal +# public-CA verification still works for anything that isn't actually Zscaler-intercepted. +# +# Pulled from a named build context (docker-compose.yml's additional_contexts, pointed at +# /opt/open-cti/tls -- runtime state, never this repo checkout) rather than COPYing it from +# anywhere inside this directory -- the cert never touches git, not even untracked in a +# gitignored folder. See README.md "TLS interception" for the full explanation and setup step. +# +# Deliberately exported for the rest of the build AND the running container, not just this one +# RUN step -- boto3 (AWS calls) and pycti's GraphQL client (requests, calling OPENCTI_URL) hit +# the exact same interception at runtime, not only pip at build time. Four separate env vars +# because pip/requests/botocore each resolve their own trust bundle independently and don't share +# one convention: +# PIP_CERT -- pip itself +# REQUESTS_CA_BUNDLE -- the `requests` library (pycti's HTTP client) +# AWS_CA_BUNDLE -- botocore/boto3 (does not fall back to REQUESTS_CA_BUNDLE) +# SSL_CERT_FILE -- the stdlib ssl module's default context, as a catch-all +COPY --from=zscaler_cert zscaler.pem /usr/local/share/ca-certificates/zscaler.crt +RUN apt-get update \ + && apt-get install -y --no-install-recommends ca-certificates \ + && update-ca-certificates \ + && rm -rf /var/lib/apt/lists/* +ENV PIP_CERT=/etc/ssl/certs/ca-certificates.crt \ + REQUESTS_CA_BUNDLE=/etc/ssl/certs/ca-certificates.crt \ + AWS_CA_BUNDLE=/etc/ssl/certs/ca-certificates.crt \ + SSL_CERT_FILE=/etc/ssl/certs/ca-certificates.crt + +COPY requirements.txt . +RUN pip install --no-cache-dir -r requirements.txt + +COPY src/ src/ +COPY tests/ tests/ + +# Relative imports in src/main.py require running as a module, not a script -- see README.md. +ENTRYPOINT ["python3", "-m", "src.main"] diff --git a/open-cti/connectors/vs-org-bootstrap/README.md b/open-cti/connectors/vs-org-bootstrap/README.md new file mode 100644 index 000000000..d1f10d347 --- /dev/null +++ b/open-cti/connectors/vs-org-bootstrap/README.md @@ -0,0 +1,174 @@ +# VS Organization & CIDR Bootstrap Connector + +Connector D in the four-connector VS architecture — see +[`../../OpenCTI-connector.md`](../../OpenCTI-connector.md) §7d for the full design, §8 for why +this one is built first, and §10 for the implementation discipline this code follows. + +Root of the dependency chain: Connectors A (Ticket Ingestion) and C (Port/Service Inventory) both +resolve their organization `Identity` against what this connector creates. + +## What it does + +Reads `Organization`, `Sector`, `Cidr`/`CidrOrgs`, `Location` from `mini_data_lake` and upserts +them into OpenCTI as `Identity` (organization + sector), `Location`, and `IPv4-Addr`/`IPv6-Addr` +(CIDR-notation) objects, plus the relationships between them (`part-of`, `located-at`, +`related-to`). + +## Structure + +```text +src/ + config.py # env/yaml config loading, fails closed on tlp_marking + org scope (see below) + db.py # mini_data_lake access + IS_LOCAL fixture fallback + mapping.py # pure row -> STIX mapping functions (no I/O) -- Loop 1 test target + connector.py # orchestration: fetch -> map -> bundle -> send, state/idempotency handling + main.py # entry point +tests/ + fixtures/ # small synthetic sample rows, shaped like real mini_data_lake queries + test_mapping.py # Loop 1: pure unit tests, no DB/OpenCTI/queue at all + test_connector_dry_run.py # Loop 2: full pipeline against fixtures, OpenCTI helper stubbed out +``` + +This mirrors the three test loops in `OpenCTI-connector.md` §9b. Run the fast ones constantly: + +```bash +pip install -r requirements.txt +python3 -m pytest tests/ -v +``` + +## Two things this connector refuses to start without + +Both are deliberate fail-closed checks in `src/config.py`, not oversights: + +- **`VS_ORG_BOOTSTRAP_TLP_MARKING`** — the TLP/marking policy decision (`OpenCTI-connector.md` + §10c) hasn't been made yet. Rather than default to something permissive like `TLP:CLEAR` for + stakeholder vulnerability data, this connector won't start until a value is set. +- **`VS_ORG_BOOTSTRAP_ORG_ACRONYM_ALLOWLIST`** (or explicit `VS_ORG_BOOTSTRAP_ALLOW_UNSCOPED_RUN=true`) + — an empty allowlist means "scoped to nothing," not "all orgs" (§9c), so an unscoped run during + dev never happens by accident. + +## Running it + +**Fixture mode (fastest, no DB/OpenCTI at all)** — set `IS_LOCAL=true`; `db.py` loads +`tests/fixtures/*.json` instead of querying Postgres. Still needs a real (or locally stacked) +OpenCTI to actually send to, since only the DB side is stubbed here — see `test_connector_dry_run.py` +for the fully-stubbed version that needs neither. + +**Against real `mini_data_lake` + a local OpenCTI stack** — see `OpenCTI-connector.md` §9d for +standing up the disposable local stack this should be developed against before ever pointing at +the shared EC2 instance. Copy `config.yml.sample` to `config.yml`, fill in `db_host` and a real +`org_acronym_allowlist` scoped to one or two low-volume test orgs, and run: + +```bash +python3 -m src.main +``` + +(Must run as a module — `src/main.py` uses relative imports. The Dockerfile's `ENTRYPOINT` already +does this correctly; running it directly outside Docker, `cd` into this directory first.) + +## TLS interception (Zscaler) — required manual, per-instance setup, not git-managed + +If this connector's build fails with `pip` complaining about a "self-signed certificate in +certificate chain," it means the network this instance sits behind (Zscaler, confirmed on the +CAWS-staging box) TLS-inspects outbound HTTPS and re-signs it with an org root CA. The EC2 host +already trusts that CA; a fresh Docker build container doesn't inherit it, and neither does the +running container at runtime (`boto3`'s AWS calls and `pycti`'s GraphQL client to `OPENCTI_URL` +would hit the identical failure the moment the container starts, not just `pip install` at build +time) — which is why the Dockerfile installs it into the image's own CA bundle (Debian's +`update-ca-certificates`, appending rather than replacing, so normal public-CA verification still +works for anything that isn't actually Zscaler-intercepted) and sets `PIP_CERT`/ +`REQUESTS_CA_BUNDLE`/`AWS_CA_BUNDLE`/`SSL_CERT_FILE` to point at the combined result for both. + +**The cert itself never touches this repo, not even untracked.** `docker-compose.yml` declares a +named build context (`additional_contexts: zscaler_cert=${ZSCALER_CERT_DIR:-/opt/open-cti/tls}`) +and the Dockerfile pulls from it directly (`COPY --from=zscaler_cert`) — the file lives in +`/opt/open-cti` (this stack's runtime state, same place `.env`/`env.deploy` already live, and +distinct from `/opt/open-cti-repo`, the replaceable git checkout) and is never read from or +written into any path git tracks. Requires BuildKit; there's a known regression for +`additional_contexts` in Compose 2.24.0 specifically (docker/compose#11351) — confirm +`docker compose build` actually resolves it on your Compose version before trusting this blindly. + +This is real infrastructure setup specific to a *network*, not to this codebase — same category +as the `open_cti` DB role/grant or the OpenCTI API token: manual, out-of-band, done once per +instance rather than automated. A genuinely new instance needs this step too; it's not a +one-time fix for all environments: + +```bash +mkdir -p /opt/open-cti/tls +cp /home/ec2-user/tmp/zscaler.pem /opt/open-cti/tls/zscaler.pem +# Combined bundle for every OTHER (non-custom, image-only) service that also needs to trust +# Zscaler at runtime -- see "Non-custom connectors" below. Portable across whatever this host's +# own OS CA bundle path actually is (varies by distro; Amazon Linux's `ec2-user` default user +# above suggests this box isn't Debian/Ubuntu, so don't assume /etc/ssl/certs/ca-certificates.crt +# specifically). +HOST_CA_BUNDLE=$(for p in /etc/pki/tls/certs/ca-bundle.crt /etc/ssl/certs/ca-certificates.crt /etc/ssl/cert.pem; do [ -f "$p" ] && echo "$p" && break; done) +cat "$HOST_CA_BUNDLE" /opt/open-cti/tls/zscaler.pem > /opt/open-cti/tls/combined-ca-bundle.pem +docker compose --env-file /opt/open-cti/.env up -d --build +``` + +### Non-custom connectors (the official `opencti/connector-*`/`opencti/platform`/`opencti/worker` images) + +Everything above is about *this repo's four custom connectors*, which trust Zscaler by building +it into their own image. The rest of `docker-compose.yml` (the stock OpenCTI platform, worker, +and the whole catalog of official Filigran connectors — `connector-cve`, `connector-shodan`, +`connector-mitre`, etc.) makes outbound HTTPS calls too (NVD, GitHub, Shodan, Censys, VulnCheck, +CISA's KEV feed, Qualys...) and sits behind the exact same network-wide TLS interception — with no +`Dockerfile`/build step of its own to install a cert into, since these are unmodified published +images. + +Verified directly against OpenCTI's own public source (not assumed) that these split into two +runtimes needing two different mechanisms: + +- **`opencti` (the platform itself) is Node.js** (`node:22-alpine`, confirmed via its + `opencti-platform/Dockerfile`). Node trusts extra CAs via `NODE_EXTRA_CA_CERTS` — genuinely + *additive*, unlike the Python vars below, so it only needs the bare `zscaler.pem`, not the + combined bundle. +- **`worker` and every official `opencti/connector-*` image are Python + `pip`** (confirmed via + `opencti-worker/Dockerfile` and the `cve` connector's own `Dockerfile` — same `python:3.12-alpine` + base, same `requests`/pycti stack this repo's own four connectors use). Python's + `REQUESTS_CA_BUNDLE`/`SSL_CERT_FILE` are **not** additive — pointing them at a bare + `zscaler.pem` would trust Zscaler and *nothing else*, breaking normal public-CA verification for + any endpoint that isn't actually intercepted. That's what `combined-ca-bundle.pem` above is for: + the host's own already-trusted public bundle with Zscaler's root appended, the same outcome + Debian's `update-ca-certificates` gives the custom connectors at build time, just assembled once + on the host instead of inside a Dockerfile. + +`docker-compose.yml` mounts one of these two files (read-only) into each service that has a real +external upstream, and sets the matching env var(s) — see the services under `OPENCTI CONNECTORS`/ +`OPENCTI DEFAULT DATA` for the concrete pattern. A few services were deliberately left alone: + +- Internal-only services (`worker`, `connector-export-file-*`, `connector-import-file-*`, + `connector-import-document`, `connector-analysis`) only ever talk to `opencti` over the internal + Docker network (plain `http://opencti:8080`) — no external upstream to intercept. +- `xtm-one`/`xtm-one-worker` almost certainly also need this -- their `postgresql+asyncpg` + `DATABASE_URL` strongly implies Python (SQLAlchemy's async-Postgres dialect naming, a + Python-only convention), and `PLATFORM_REGISTRATION_TOKEN` suggests an outbound + licensing/registration call to Filigran. Applied the same Python treatment on that inference -- + no public Dockerfile found for `xtmone/platform`/`xtmone/worker` to confirm it directly the way + `opencti`/`worker`/the connectors above were, so this one's worth confirming against real + container logs rather than trusting as firmly as the rest of this section. +- `xtm-composer` (`filigran/xtm-composer`) was left alone entirely -- no comparable signal at all + (no `DATABASE_URL`-style clue, no public Dockerfile), and its own config here shows no external + URL besides the internal `http://opencti:8080` (it manages the `xtm-one`/`xtm-one-worker` + containers via the mounted Docker socket, not outbound HTTPS of its own). Worth revisiting if it + turns out to make outbound calls this config doesn't show. +- `connector-qualys-cve-enrichment` previously worked around this exact problem with + `QUALYS_SSL_VERIFY=false` — disabling verification entirely rather than trusting the + intercepting CA properly. That flag should come back out now that this connector has a real + trust path instead. + +## Known gaps / things to verify before this is production-ready + +- **Sector/Location collision risk (§7d/§6):** this connector currently creates + `Identity(class="class")`/`Location` objects unconditionally. `connector-opencti` (OpenCTI's + built-in "Datasets" seeder) likely already seeded a default sector/location taxonomy — + `connector.py`'s `_process()` has a `TODO` marking where a look-up-by-name pass needs to go + before this is safe to run against a real instance with real data. +- **State growth:** the `part_of_relationship_ids`/`located_at_relationship_ids`/ + `cidr_relationship_ids` maps in connector state grow with the number of distinct external keys + ever seen and never shrink. Fine at VS's scale (hundreds of orgs, thousands of CIDRs at most), + but worth knowing if that assumption ever needs revisiting. +- Not yet run against a real OpenCTI instance — everything up through `OpenCTIConnectorHelper` + initialization and the GraphQL health check has been verified locally (it fails, correctly, at + "no OpenCTI reachable" rather than anywhere in this connector's own code); the actual + send-and-verify-in-OpenCTI step is next, once the local stack (§9d) exists. diff --git a/open-cti/connectors/vs-org-bootstrap/config.yml.sample b/open-cti/connectors/vs-org-bootstrap/config.yml.sample new file mode 100644 index 000000000..d4b484bf4 --- /dev/null +++ b/open-cti/connectors/vs-org-bootstrap/config.yml.sample @@ -0,0 +1,57 @@ +# Connector D — VS Organization & CIDR Bootstrap +# +# See ../../OpenCTI-connector.md §7d for the design this implements, and §10 for the discipline +# (idempotency, marking/attribution, scoping) this config exists to enforce. +# +# Copy to config.yml for local runs (docker-compose.yml drives the containerized version purely +# from env vars instead — see README.md). + +opencti: + url: 'http://localhost:8080' + token: 'ChangeMe' + +connector: + id: 'ChangeMe-generate-a-uuid4' + name: 'VS Organization & CIDR Bootstrap' + scope: 'identity,location' + type: 'EXTERNAL_IMPORT' + confidence_level: 75 + log_level: 'info' + # ISO 8601 duration. Deliberately slower than Connectors A/C -- org/CIDR data churns far less + # than scan findings (OpenCTI-connector.md §7d). Start conservative; tighten once vs_org_sync's + # own real cadence is confirmed (§6). + duration_period: 'PT6H' + +vs_org_bootstrap: + # --- mini_data_lake connectivity (OpenCTI-connector.md §2) --- + db_host: 'ChangeMe' + db_port: 5432 + db_name: 'cyhy_mini_data_lake_staging' + db_user: 'open_cti' + # IAM database authentication, not a static password -- see §2. When true, db_password is + # ignored and a short-lived token is generated per connection via boto3/STS. + db_use_iam_auth: true + db_password: '' + aws_region: 'us-gov-west-1' + + # --- dev/test discipline (§9) --- + # When true, skip mini_data_lake entirely and load tests/fixtures/*.json instead. Mirrors the + # IS_LOCAL convention already used throughout backend/src/xfd_django/xfd_api/tasks/. + is_local: false + local_fixture_dir: './tests/fixtures' + + # Comma-separated org acronym allowlist. REQUIRED to be non-empty outside of a deliberate + # production run -- see §9c. Empty string here means "no orgs," not "all orgs": this connector + # refuses to run unscoped by accident (see src/config.py). + org_acronym_allowlist: '' + # Explicit opt-in to remove the allowlist requirement for a real production run. + allow_unscoped_run: false + max_rows_per_run: 5000 + + # --- attribution / marking (§10c) --- + author_name: 'CISA VulnScanning' + # Deliberately left unset. This is a real, undecided policy question (§10c) -- the connector + # refuses to start without an explicit value here rather than silently defaulting to something + # like TLP:CLEAR that's very unlikely to be right for this data. Set to a marking definition + # name/STIX ID once the policy call is made. + tlp_marking: '' diff --git a/open-cti/connectors/vs-org-bootstrap/requirements.txt b/open-cti/connectors/vs-org-bootstrap/requirements.txt new file mode 100644 index 000000000..17d9c9b7f --- /dev/null +++ b/open-cti/connectors/vs-org-bootstrap/requirements.txt @@ -0,0 +1,5 @@ +boto3==1.43.81 +psycopg2-binary==2.9.9 +pycti==7.260824.0 +python-dateutil==2.9.0.post0 +stix2==3.0.2 diff --git a/open-cti/connectors/vs-org-bootstrap/src/__init__.py b/open-cti/connectors/vs-org-bootstrap/src/__init__.py new file mode 100644 index 000000000..fc87dfc0c --- /dev/null +++ b/open-cti/connectors/vs-org-bootstrap/src/__init__.py @@ -0,0 +1 @@ +"""VS Organization & CIDR Bootstrap connector -- see ../README.md and ../../OpenCTI-connector.md.""" diff --git a/open-cti/connectors/vs-org-bootstrap/src/config.py b/open-cti/connectors/vs-org-bootstrap/src/config.py new file mode 100644 index 000000000..4be25ade4 --- /dev/null +++ b/open-cti/connectors/vs-org-bootstrap/src/config.py @@ -0,0 +1,164 @@ +"""Config loading for the VS Organization & CIDR Bootstrap connector. + +Precedence is env var > config.yml > default, via pycti's own get_config_variable() -- see +OpenCTI-connector.md §10b ("confirm rather than assume" pycti behavior; this one's verified +against the installed pycti==7.260824.0 source). + +Fails closed on two things deliberately, rather than silently defaulting to something wrong: + - tlp_marking: unset in this repo on purpose (§10c is a real, undecided policy question this + doc can't answer). Refusing to start beats guessing TLP:CLEAR for stakeholder vuln data. + - org_acronym_allowlist: empty means "scoped to nothing," not "all orgs" (§9c) -- a connector + run has to opt in to being unscoped via allow_unscoped_run, not fall into it by omission. +""" + +# Standard Python Libraries +import os +from typing import Dict, List, Optional + +# Third-Party Libraries +from pycti import get_config_variable +import yaml + + +class ConfigError(RuntimeError): + """Raised when required configuration is missing -- fail closed, not silent default.""" + + +def load_yaml_config() -> Dict: + """Load config.yml next to this connector if present; docker-compose drives env vars only.""" + here = os.path.dirname(os.path.abspath(__file__)) + config_file_path = os.path.join(here, "..", "config.yml") + if os.path.isfile(config_file_path): + with open(config_file_path, encoding="utf-8") as f: + return yaml.safe_load(f) or {} + return {} + + +class Config: + """Typed view over this connector's configuration.""" + + def __init__(self, raw: Optional[Dict] = None): + """Load config from `raw` if given (tests), else config.yml/env vars, then validate.""" + self.raw: Dict = raw if raw is not None else load_yaml_config() + + # --- mini_data_lake connectivity (OpenCTI-connector.md §2) --- + self.db_host = get_config_variable( + "VS_ORG_BOOTSTRAP_DB_HOST", ["vs_org_bootstrap", "db_host"], self.raw + ) + self.db_port = get_config_variable( + "VS_ORG_BOOTSTRAP_DB_PORT", + ["vs_org_bootstrap", "db_port"], + self.raw, + isNumber=True, + default=5432, + ) + self.db_name = get_config_variable( + "VS_ORG_BOOTSTRAP_DB_NAME", + ["vs_org_bootstrap", "db_name"], + self.raw, + default="cyhy_mini_data_lake_staging", + ) + self.db_user = get_config_variable( + "VS_ORG_BOOTSTRAP_DB_USER", + ["vs_org_bootstrap", "db_user"], + self.raw, + default="open_cti", + ) + self.db_use_iam_auth = get_config_variable( + "VS_ORG_BOOTSTRAP_DB_USE_IAM_AUTH", + ["vs_org_bootstrap", "db_use_iam_auth"], + self.raw, + default=True, + ) + self.db_password = get_config_variable( + "VS_ORG_BOOTSTRAP_DB_PASSWORD", + ["vs_org_bootstrap", "db_password"], + self.raw, + default="", + ) + self.aws_region = get_config_variable( + "VS_ORG_BOOTSTRAP_AWS_REGION", ["vs_org_bootstrap", "aws_region"], self.raw + ) + + # --- dev/test discipline (§9) --- + self.is_local = get_config_variable( + "IS_LOCAL", ["vs_org_bootstrap", "is_local"], self.raw, default=False + ) + self.local_fixture_dir = get_config_variable( + "VS_ORG_BOOTSTRAP_LOCAL_FIXTURE_DIR", + ["vs_org_bootstrap", "local_fixture_dir"], + self.raw, + default="./tests/fixtures", + ) + + org_allowlist_raw = get_config_variable( + "VS_ORG_BOOTSTRAP_ORG_ACRONYM_ALLOWLIST", + ["vs_org_bootstrap", "org_acronym_allowlist"], + self.raw, + default="", + ) + self.org_acronym_allowlist: List[str] = [ + acronym.strip() + for acronym in (org_allowlist_raw or "").split(",") + if acronym.strip() + ] + self.allow_unscoped_run = get_config_variable( + "VS_ORG_BOOTSTRAP_ALLOW_UNSCOPED_RUN", + ["vs_org_bootstrap", "allow_unscoped_run"], + self.raw, + default=False, + ) + self.max_rows_per_run = get_config_variable( + "VS_ORG_BOOTSTRAP_MAX_ROWS_PER_RUN", + ["vs_org_bootstrap", "max_rows_per_run"], + self.raw, + isNumber=True, + default=5000, + ) + + # --- attribution / marking (§10c) --- + self.author_name = get_config_variable( + "VS_ORG_BOOTSTRAP_AUTHOR_NAME", + ["vs_org_bootstrap", "author_name"], + self.raw, + default="CISA VulnScanning", + ) + self.tlp_marking = get_config_variable( + "VS_ORG_BOOTSTRAP_TLP_MARKING", + ["vs_org_bootstrap", "tlp_marking"], + self.raw, + ) + + # Deliberately slower than Connectors A/C (§7d) -- org/CIDR data churns far less than + # scan findings. ISO 8601 duration, read the same way pycti's own connectors do. + self.duration_period = get_config_variable( + "CONNECTOR_DURATION_PERIOD", + ["connector", "duration_period"], + self.raw, + default="PT6H", + ) + + self._validate() + + def _validate(self) -> None: + """Fail closed on the two decisions this doc can't make silently. See module docstring.""" + if not self.tlp_marking: + raise ConfigError( + "vs_org_bootstrap.tlp_marking is not set. This is a real, undecided policy " + "question (OpenCTI-connector.md §10c) -- refusing to start rather than default " + "to something like TLP:CLEAR for stakeholder vulnerability data. Set " + "VS_ORG_BOOTSTRAP_TLP_MARKING once the data-handling policy call is made." + ) + if not self.org_acronym_allowlist and not self.allow_unscoped_run: + raise ConfigError( + "vs_org_bootstrap.org_acronym_allowlist is empty and allow_unscoped_run is not " + "set. An empty allowlist means 'scoped to nothing', not 'all orgs' -- see " + "OpenCTI-connector.md §9c. Set VS_ORG_BOOTSTRAP_ORG_ACRONYM_ALLOWLIST for dev, " + "or explicitly set VS_ORG_BOOTSTRAP_ALLOW_UNSCOPED_RUN=true for a deliberate " + "full-scope production run." + ) + if not self.is_local and not self.db_host: + raise ConfigError( + "vs_org_bootstrap.db_host is not set and is_local is false -- nothing to connect " + "to. Set VS_ORG_BOOTSTRAP_DB_HOST, or IS_LOCAL=true to use fixtures instead." + ) diff --git a/open-cti/connectors/vs-org-bootstrap/src/connector.py b/open-cti/connectors/vs-org-bootstrap/src/connector.py new file mode 100644 index 000000000..38832646b --- /dev/null +++ b/open-cti/connectors/vs-org-bootstrap/src/connector.py @@ -0,0 +1,306 @@ +"""Orchestration for the VS Organization & CIDR Bootstrap connector (OpenCTI-connector.md §7d). + +Root of the four-connector dependency chain (§8) -- Connectors A and C both resolve their org +Identity by acronym against what this connector creates. Build/deploy order: this one first. +""" + +# Standard Python Libraries +import datetime +import logging +from typing import Dict, Optional + +# Third-Party Libraries +from pycti import OpenCTIConnectorHelper +import stix2 + +from . import mapping +from .config import Config +from .db import OrgBootstrapData, VsOrgBootstrapRepository + +LOGGER = logging.getLogger("vs_org_bootstrap") + +# Well-known, platform-recognized TLP marking-definition STIX ids (stix2 ships these with the +# fixed ids OpenCTI already seeds by default -- no lookup/create needed, just reference them). +# TLP:CLEAR is the STIX 2.1 successor name for TLP:WHITE; both map to the same marking here since +# this stix2 version only ships the pre-2.1 WHITE/GREEN/AMBER/RED set. +_TLP_MARKINGS = { + "TLP:CLEAR": stix2.TLP_WHITE, + "TLP:WHITE": stix2.TLP_WHITE, + "TLP:GREEN": stix2.TLP_GREEN, + "TLP:AMBER": stix2.TLP_AMBER, + "TLP:RED": stix2.TLP_RED, +} + + +def resolve_marking(tlp_marking: str) -> stix2.MarkingDefinition: + """Resolve a TLP string to its well-known marking-definition object. + + Config validation (config.py) already refuses an empty value -- this refuses an unrecognized + one too, rather than silently falling back to something permissive. + """ + marking = _TLP_MARKINGS.get(tlp_marking.strip().upper()) + if marking is None: + raise ValueError( + f"Unrecognized tlp_marking {tlp_marking!r}. Expected one of: " + f"{', '.join(_TLP_MARKINGS)} (see OpenCTI-connector.md §10c)." + ) + return marking + + +class VsOrgBootstrapConnector: + """Polls org/CIDR data from mini_data_lake and upserts it into OpenCTI. + + Produces Identity/Location/IPv4-Addr(CIDR) objects and their relationships. + """ + + def __init__(self, config: Optional[Config] = None, helper=None): + """Build the connector, injecting `config`/`helper` for tests instead of the real thing. + + `config`/`helper` injection points exist for tests (§9b Loop 2 dry-run) -- production + (main.py) always calls VsOrgBootstrapConnector() with neither, building the real thing. + """ + self.config = config or Config() + self.helper = helper or OpenCTIConnectorHelper(self.config.raw) + self.repository = VsOrgBootstrapRepository(self.config) + + self.author = mapping.build_author(self.config.author_name) + self.marking = resolve_marking(self.config.tlp_marking) + + LOGGER.info( + "VS Org Bootstrap connector initialized (is_local=%s, org_allowlist=%s)", + self.config.is_local, + self.config.org_acronym_allowlist or "", + ) + + def run(self) -> None: + """Entry point pycti's scheduler calls each duration_period tick.""" + self.helper.connector_logger.info("Starting VS Org Bootstrap run") + state = self.helper.get_state() or {} + + try: + data = self.repository.fetch( + since_updated_at=state.get("last_updated_at_watermark") + ) + new_watermark = self._process(data, state) + except Exception as e: # pylint: disable=broad-except + # §10e: don't let one bad run corrupt state -- log and let the next scheduled tick + # retry from the last confirmed watermark rather than partially advancing it. + # connector_logger is pycti's own AppLogger, not a stdlib logging.Logger -- verified + # against the installed source after this call itself crashed with "unexpected + # keyword argument 'exc_info'" while trying to log a real error (worse than not + # catching at all, since it masked the original exception). AppLogger.error() takes + # (message, meta=None) and always passes exc_info=1 to the underlying logger + # internally, so the traceback is captured automatically either way. + self.helper.connector_logger.error( + "VS Org Bootstrap run failed", meta={"error": str(e)} + ) + return + + # §10f: only advance the watermark after the bundle has actually been accepted by + # OpenCTI, not before -- new_watermark is None (unchanged) whenever nothing was sent. + if new_watermark is not None: + state["last_updated_at_watermark"] = new_watermark + self.helper.set_state(state) + + self.helper.connector_logger.info("VS Org Bootstrap run complete") + + # ------------------------------------------------------------------ + + def _process(self, data: OrgBootstrapData, state: Dict) -> Optional[str]: + """Build the STIX bundle for one run and send it. + + Returns the new watermark value to persist, or None if there was nothing to send (so + run() leaves state untouched). + """ + if not data.organizations: + self.helper.connector_logger.info("No organizations in scope for this run") + return None + + objects = [self.author, self.marking] + + # --- Sectors first (org->sector relationships need the Sector Identity to exist) --- + # §7d / §6: real collision risk with connector-opencti's default sector taxonomy. + # TODO: look up existing Identity(class=sector) by name via self.helper.api before + # deciding to mint a new one -- deferred here pending the ground-truthing pass + # (OpenCTI-connector.md §10, next steps) that checks the actual overlap. + sector_stix_by_acronym = {} + for sector_row in data.sectors: + sector_obj = mapping.map_sector(sector_row, self.author.id, self.marking.id) + sector_stix_by_acronym[sector_row["acronym"]] = sector_obj.id + objects.append(sector_obj) + + # --- Locations, keyed by mini_data_lake location id --- + location_stix_by_id = {} + for location_id, location_row in data.locations_by_id.items(): + location_obj = mapping.map_location( + location_row, self.author.id, self.marking.id + ) + if location_obj is not None: + location_stix_by_id[location_id] = location_obj.id + objects.append(location_obj) + + # --- Organizations + their part-of/located-at relationships --- + org_stix_by_acronym = {} + org_row_by_acronym = {} + for org_row in data.organizations: + org_obj = mapping.map_organization(org_row, self.author.id, self.marking.id) + org_stix_by_acronym[org_row["acronym"]] = org_obj.id + org_row_by_acronym[org_row["acronym"]] = org_row + objects.append(org_obj) + + if ( + org_row.get("location_id") + and org_row["location_id"] in location_stix_by_id + ): + objects.append( + self._located_at_relationship( + state, + org_row["acronym"], + org_obj.id, + location_stix_by_id[org_row["location_id"]], + ) + ) + + # Parent/child org relationships -- needs both orgs resolved first, hence a second pass. + for acronym, org_row in org_row_by_acronym.items(): + parent_id = org_row.get("parent_id") + if not parent_id: + continue + parent_acronym = next( + (a for a, r in org_row_by_acronym.items() if r["id"] == parent_id), None + ) + if parent_acronym and parent_acronym in org_stix_by_acronym: + objects.append( + self._part_of_relationship( + state, + f"org:{acronym}->org:{parent_acronym}", + org_stix_by_acronym[acronym], + org_stix_by_acronym[parent_acronym], + ) + ) + + # Sector membership -- from the M2M join, not a column on organization itself. + for sector_acronym, member_acronyms in data.sector_memberships.items(): + if sector_acronym not in sector_stix_by_acronym: + continue + for member_acronym in member_acronyms: + if member_acronym not in org_stix_by_acronym: + continue + objects.append( + self._part_of_relationship( + state, + f"org:{member_acronym}->sector:{sector_acronym}", + org_stix_by_acronym[member_acronym], + sector_stix_by_acronym[sector_acronym], + ) + ) + + # --- CIDRs + ownership relationships --- + # §10e says one bad row shouldn't sink an entire poll's bundle -- this loop is where + # that was actually just tested for real: a single malformed CIDR (a same-day + # open+close, see mapping.build_owns_cidr) previously aborted the *whole* run, at full + # production scale (13k+ orgs), not just that one org. Isolated per-row instead of + # relying on run()'s top-level catch, which only protects state from a failed run, not + # one run's success from a single bad row within it. + for cidr_row in data.cidrs: + org_acronym = cidr_row.get("organization_acronym") + if org_acronym not in org_stix_by_acronym: + continue + try: + cidr_obj = mapping.map_cidr_observable(cidr_row["network"]) + relationship = self._cidr_relationship( + state, org_stix_by_acronym[org_acronym], cidr_obj.id, cidr_row + ) + except Exception as e: # pylint: disable=broad-except + self.helper.connector_logger.warning( + f"Skipping CIDR {cidr_row.get('network')!r} for org {org_acronym!r}: {e}" + ) + continue + objects.append(cidr_obj) + objects.append(relationship) + + objects = mapping.dedupe_bundle_objects(objects) + bundle = self.helper.stix2_create_bundle(objects) + self.helper.send_stix2_bundle(bundle, update=True) + + # connector_logger is pycti's AppLogger, not a stdlib logging.Logger -- it takes + # (message, meta=None), not lazy %-style formatting with extra positional args (same + # class of mistake as the exc_info bug above; found the same way, by it crashing on a + # real run). Pre-format the message into one string instead. + self.helper.connector_logger.info( + f"Sent {len(objects)} objects " + f"({len(data.organizations)} orgs, {len(data.sectors)} sectors, " + f"{len(data.cidrs)} CIDRs)" + ) + + watermark = max( + (org["updated_at"] for org in data.organizations if org.get("updated_at")), + default=None, + ) + # Same live-vs-fixture type gap as mapping.normalize_timestamp() -- psycopg2 returns a + # real datetime.datetime for organization.updated_at (a timestamptz column), but our + # IS_LOCAL fixtures store it as a JSON/ISO string already. pycti's set_state() does a + # plain json.dumps(state) with no datetime support, so a raw datetime here crashes for + # real on a live run ("Object of type datetime is not JSON serializable") -- not caught + # by fixture-based tests for the same reason the UUID/date bugs weren't. + # isinstance, not hasattr -- mypy can verify the None branch is unreachable through + # isinstance narrowing, which it can't do for a hasattr() guard (a real limitation, not + # a runtime bug: hasattr(None, "isoformat") is correctly False at runtime either way, + # proven by the test suite, but mypy flagged this as a real pre-commit blocker). + return ( + watermark.isoformat() + if isinstance(watermark, datetime.datetime) + else watermark + ) + + # ------------------------------------------------------------------ + # §10a idempotency: reuse a previously-recorded relationship id rather than trusting + # OpenCTI's fuzzy ±30-day type/source/target/time dedup window to find the right one, + # especially for the null->set stop_time transition that isn't clearly documented there. + # ------------------------------------------------------------------ + + def _located_at_relationship( + self, state: Dict, org_acronym: str, org_id: str, location_id: str + ): + rel_map = state.setdefault("located_at_relationship_ids", {}) + rel = mapping.build_located_at( + org_id, + location_id, + self.author.id, + self.marking.id, + existing_id=rel_map.get(org_acronym), + ) + rel_map[org_acronym] = rel.id + return rel + + def _part_of_relationship( + self, state: Dict, key: str, child_id: str, parent_id: str + ): + rel_map = state.setdefault("part_of_relationship_ids", {}) + rel = mapping.build_part_of( + child_id, + parent_id, + self.author.id, + self.marking.id, + existing_id=rel_map.get(key), + ) + rel_map[key] = rel.id + return rel + + def _cidr_relationship( + self, state: Dict, org_id: str, cidr_id: str, cidr_row: Dict + ): + rel_map = state.setdefault("cidr_relationship_ids", {}) + key = f"{org_id}:{cidr_row['network']}" + stop_time = None if cidr_row.get("current") else cidr_row.get("last_seen") + rel = mapping.build_owns_cidr( + org_id, + cidr_id, + self.author.id, + self.marking.id, + first_seen=cidr_row.get("first_seen"), + last_seen_or_stop=stop_time, + existing_id=rel_map.get(key), + ) + rel_map[key] = rel.id + return rel diff --git a/open-cti/connectors/vs-org-bootstrap/src/db.py b/open-cti/connectors/vs-org-bootstrap/src/db.py new file mode 100644 index 000000000..5ffe84566 --- /dev/null +++ b/open-cti/connectors/vs-org-bootstrap/src/db.py @@ -0,0 +1,239 @@ +"""mini_data_lake access for the VS Organization & CIDR Bootstrap connector. + +Two implementations behind one interface (VsOrgBootstrapRepository): a real Postgres-backed one +and an IS_LOCAL fixture-backed one, mirroring the convention already used throughout +backend/src/xfd_django/xfd_api/tasks/utils/query_redshift.py's load_test_data(). See +OpenCTI-connector.md §9a. + +Table/column names below are taken from backend/src/xfd_django/xfd_mini_dl/models.py -- see +OpenCTI-connector.md §7d for the line references. Sector.organizations is a plain Django +ManyToManyField with no `through` model (models.py ~L2450), so its join table was initially a +guess at Django's default naming convention rather than something read off an explicit model +class -- confirmed against the real schema (2026-08-26) as `sector_organizations`, columns +`sector_id`/`organization_id`, exactly as guessed. +""" + +# Standard Python Libraries +from dataclasses import dataclass, field +import json +import logging +import os +from typing import Dict, List, Optional + +# Third-Party Libraries +import boto3 +import psycopg2 +import psycopg2.extras + +from .config import Config + +LOGGER = logging.getLogger("vs_org_bootstrap.db") + + +@dataclass +class OrgBootstrapData: + """Everything one connector run needs, regardless of where it came from.""" + + organizations: List[Dict] = field(default_factory=list) + sectors: List[Dict] = field(default_factory=list) + # sector acronym -> list of member org acronyms + sector_memberships: Dict[str, List[str]] = field(default_factory=dict) + # location id -> location row + locations_by_id: Dict[str, Dict] = field(default_factory=dict) + # list of {cidr fields..., organization_acronym, first_seen, last_seen, current} + cidrs: List[Dict] = field(default_factory=list) + + +def _generate_iam_auth_token(config: Config) -> str: + """Generate a short-lived RDS IAM auth token. + + See OpenCTI-connector.md §2 -- ~15min TTL, so this is called fresh per connection, never + cached across the connector's lifetime. + """ + client = boto3.client("rds", region_name=config.aws_region) + return client.generate_db_auth_token( + DBHostname=config.db_host, + Port=config.db_port, + DBUsername=config.db_user, + ) + + +def get_connection(config: Config): + """Open a psycopg2 connection to mini_data_lake, IAM-auth or password per config.""" + password = ( + _generate_iam_auth_token(config) + if config.db_use_iam_auth + else config.db_password + ) + return psycopg2.connect( + host=config.db_host, + port=config.db_port, + dbname=config.db_name, + user=config.db_user, + password=password, + sslmode="require", # IAM auth requires TLS regardless of rds.force_ssl -- §2 + cursor_factory=psycopg2.extras.RealDictCursor, + connect_timeout=10, + ) + + +class VsOrgBootstrapRepository: + """Fetches org/sector/CIDR/location data, scoped and capped per config (§9c).""" + + def __init__(self, config: Config): + """Hold the config; connections are opened per-call, not cached (§10d).""" + self.config = config + + def fetch(self, since_updated_at: Optional[str]) -> OrgBootstrapData: + """Fetch one run's worth of data, from fixtures (IS_LOCAL) or mini_data_lake.""" + if self.config.is_local: + return self._fetch_local() + return self._fetch_live(since_updated_at) + + # ------------------------------------------------------------------ + # Live path + # ------------------------------------------------------------------ + + def _fetch_live(self, since_updated_at: Optional[str]) -> OrgBootstrapData: + conn = get_connection(self.config) + try: + organizations = self._fetch_organizations_live(conn, since_updated_at) + org_acronyms = [org["acronym"] for org in organizations] + location_ids = [ + org["location_id"] for org in organizations if org.get("location_id") + ] + + return OrgBootstrapData( + organizations=organizations, + sectors=self._fetch_sectors_live(conn), + sector_memberships=self._fetch_sector_memberships_live( + conn, org_acronyms + ), + locations_by_id=self._fetch_locations_live(conn, location_ids), + cidrs=self._fetch_cidrs_live(conn, org_acronyms), + ) + finally: + conn.close() + + def _fetch_organizations_live( + self, conn, since_updated_at: Optional[str] + ) -> List[Dict]: + # Explicit column projection (§10d) -- only what mapping.py actually maps to STIX. + # Filtered by the org acronym allowlist (§9c) and, when present, Organization.updated_at + # (§7d -- the one table in this whole architecture with a real watermark column, no + # LatestPortScan/Ticket-style gotchas). + # `acronym = ANY(NULL)` evaluates to NULL (not TRUE) in Postgres -- verified against a + # real instance -- so without the explicit "IS NULL OR" branch, an unscoped run + # (ALLOW_UNSCOPED_RUN=true, empty allowlist -> acronyms param is None) would silently + # return zero organizations instead of all of them. Found before this was ever actually + # exercised, not from a live failure. + query = """ + SELECT id, acronym, name, retired, stakeholder, vs_stakeholder, type, + state, state_name, county, county_fips, state_fips, country, country_name, + location_id, parent_id, updated_at + FROM organization + WHERE (%(acronyms)s IS NULL OR acronym = ANY(%(acronyms)s)) + AND (%(since)s IS NULL OR updated_at > %(since)s) + ORDER BY updated_at + LIMIT %(limit)s + """ + with conn.cursor() as cur: + cur.execute( + query, + { + "acronyms": self.config.org_acronym_allowlist or None, + "since": since_updated_at, + "limit": self.config.max_rows_per_run, + }, + ) + return [dict(row) for row in cur.fetchall()] + + def _fetch_sectors_live(self, conn) -> List[Dict]: + # Small, low-cardinality table (~20 rows, CISA's sector taxonomy) -- full poll every run + # rather than tracking incrementally, per §7d. + with conn.cursor() as cur: + cur.execute("SELECT id, acronym, name, retired FROM sector") + return [dict(row) for row in cur.fetchall()] + + def _fetch_sector_memberships_live( + self, conn, org_acronyms: List[str] + ) -> Dict[str, List[str]]: + if not org_acronyms: + return {} + # sector_organizations/sector_id/organization_id confirmed against the real schema + # (see module docstring) -- Django's default M2M join-table naming, as guessed. + query = """ + SELECT s.acronym AS sector_acronym, o.acronym AS org_acronym + FROM sector_organizations so + JOIN sector s ON s.id = so.sector_id + JOIN organization o ON o.id = so.organization_id + WHERE o.acronym = ANY(%(acronyms)s) + """ + memberships: Dict[str, List[str]] = {} + with conn.cursor() as cur: + cur.execute(query, {"acronyms": org_acronyms}) + for row in cur.fetchall(): + memberships.setdefault(row["sector_acronym"], []).append( + row["org_acronym"] + ) + return memberships + + def _fetch_locations_live(self, conn, location_ids: List[str]) -> Dict[str, Dict]: + if not location_ids: + return {} + # location.id is a native Postgres uuid column; psycopg2 adapts a Python list of + # id strings to a text[] array by default, and Postgres won't implicitly compare + # uuid = ANY(text[]) -- "operator does not exist: uuid = text", hit for real on a live + # run, not caught by the mapping-only tests (fixtures never round-trip through + # psycopg2's actual type adaptation). Explicit cast needed; the other three ANY() + # queries in this file all compare acronym (varchar), which doesn't have this problem. + query = """ + SELECT id, name, country_abrv, country, county, county_fips, gnis_id, + state_abrv, state + FROM location + WHERE id = ANY(%(ids)s::uuid[]) + """ + with conn.cursor() as cur: + cur.execute(query, {"ids": location_ids}) + return {str(row["id"]): dict(row) for row in cur.fetchall()} + + def _fetch_cidrs_live(self, conn, org_acronyms: List[str]) -> List[Dict]: + if not org_acronyms: + return [] + query = """ + SELECT c.network, c.retired AS cidr_retired, + co.first_seen, co.last_seen, co.current, + o.acronym AS organization_acronym + FROM cidr_orgs co + JOIN cidr c ON c.id = co.cidr_id + JOIN organization o ON o.id = co.organization_id + WHERE o.acronym = ANY(%(acronyms)s) + LIMIT %(limit)s + """ + with conn.cursor() as cur: + cur.execute( + query, + {"acronyms": org_acronyms, "limit": self.config.max_rows_per_run}, + ) + return [dict(row) for row in cur.fetchall()] + + # ------------------------------------------------------------------ + # IS_LOCAL fixture path (§9a) -- same shape as the live path, zero DB/network. + # ------------------------------------------------------------------ + + def _fetch_local(self) -> OrgBootstrapData: + directory = self.config.local_fixture_dir + LOGGER.info("IS_LOCAL=true -- loading fixtures from %s", directory) + return OrgBootstrapData( + organizations=self._load_json(directory, "organizations.json"), + sectors=self._load_json(directory, "sectors.json"), + sector_memberships=self._load_json(directory, "sector_memberships.json"), + locations_by_id=self._load_json(directory, "locations_by_id.json"), + cidrs=self._load_json(directory, "cidrs.json"), + ) + + @staticmethod + def _load_json(directory: str, filename: str): + path = os.path.join(directory, filename) + with open(path, encoding="utf-8") as f: + return json.load(f) diff --git a/open-cti/connectors/vs-org-bootstrap/src/main.py b/open-cti/connectors/vs-org-bootstrap/src/main.py new file mode 100644 index 000000000..89aacaf6e --- /dev/null +++ b/open-cti/connectors/vs-org-bootstrap/src/main.py @@ -0,0 +1,27 @@ +"""Entry point. See OpenCTI-connector.md §7d for design, README.md for how to run this locally.""" + +# Standard Python Libraries +import sys +import time + +from .config import ConfigError +from .connector import VsOrgBootstrapConnector + +if __name__ == "__main__": + try: + connector = VsOrgBootstrapConnector() + except ConfigError as e: + # Fail closed and loud (§10c/§9c) -- not a stack trace to dig through. + print(f"[vs-org-bootstrap] refusing to start: {e}", file=sys.stderr) + sys.exit(1) + + try: + connector.helper.schedule_iso( + message_callback=connector.run, + duration_period=connector.config.duration_period, + ) + except KeyboardInterrupt: + connector.helper.connector_logger.info("Stopping VS Org Bootstrap connector") + connector.helper.stop() + time.sleep(1) + sys.exit(0) diff --git a/open-cti/connectors/vs-org-bootstrap/src/mapping.py b/open-cti/connectors/vs-org-bootstrap/src/mapping.py new file mode 100644 index 000000000..a479539a0 --- /dev/null +++ b/open-cti/connectors/vs-org-bootstrap/src/mapping.py @@ -0,0 +1,280 @@ +"""Pure row -> STIX mapping functions for the VS Organization & CIDR Bootstrap connector. + +Deliberately dependency-free beyond stix2/pycti and the standard library -- no DB, no OpenCTI +API, no network. This is "Loop 1" from OpenCTI-connector.md §9b: the fastest possible test loop, +and where most of a connector's actual bugs live. See tests/test_mapping.py. + +STIX mapping decisions implemented here are documented in OpenCTI-connector.md §7d; the +idempotency pattern (why relationships take an `existing_id` param) is §10a. +""" + +# Standard Python Libraries +import datetime +from ipaddress import ip_network +from typing import Dict, List, Optional + +# Third-Party Libraries +from pycti import Identity as OpenCTIIdentity +from pycti import Location as OpenCTILocation +from pycti import StixCoreRelationship +import stix2 + +VS_EXTERNAL_SOURCE = "VS" + + +def build_author(name: str) -> stix2.Identity: + """Build the single system Identity every VS-sourced object should cite as createdBy (§10c).""" + return stix2.Identity( + id=OpenCTIIdentity.generate_id(name, "system"), + name=name, + identity_class="system", + ) + + +def map_organization( + org: Dict, + author_id: str, + marking_id: str, + parent_stix_id: Optional[str] = None, +) -> stix2.Identity: + """Organization row -> Identity(class=organization). + + `acronym` is the durable join key (OpenCTI-connector.md §7d) -- carried as an External + Reference for our own/analysts' traceability. It is NOT what makes this idempotent: Identity + objects get OpenCTI's own content-derived standard_id from (name, identity_class) via + Identity.generate_id(), independent of anything we attach -- see §10a for why that's *not* + true of relationships, which is the part of this file that actually needs care. + """ + labels = [] + if org.get("retired"): + labels.append("vs-retired") + if org.get("stakeholder"): + labels.append("vs-stakeholder") + if org.get("vs_stakeholder"): + labels.append("vs-vs-stakeholder") + + extra = {"x_opencti_aliases": [org["acronym"]]} if org.get("acronym") else {} + + return stix2.Identity( + id=OpenCTIIdentity.generate_id(org["name"], "organization"), + name=org["name"], + identity_class="organization", + labels=labels or None, + created_by_ref=author_id, + object_marking_refs=[marking_id], + external_references=[ + stix2.ExternalReference( + source_name=VS_EXTERNAL_SOURCE, external_id=org["acronym"] + ) + ] + if org.get("acronym") + else None, + custom_properties=extra, + ) + + +def map_sector(sector: Dict, author_id: str, marking_id: str) -> stix2.Identity: + """Map a sector row to Identity(class="class"). + + "class" is STIX 2.1's open-vocab term for a categorical (non-org/individual/system/group) + identity, which is how OpenCTI represents sectors. Collision risk with connector-opencti's + default sector taxonomy is real -- see OpenCTI-connector.md §7d/§6; look-up-before-create + against that taxonomy happens in connector.py, not here (this module has no OpenCTI API + access by design). + """ + return stix2.Identity( + id=OpenCTIIdentity.generate_id(sector["name"], "class"), + name=sector["name"], + identity_class="class", + created_by_ref=author_id, + object_marking_refs=[marking_id], + external_references=[ + stix2.ExternalReference( + source_name=VS_EXTERNAL_SOURCE, external_id=sector["acronym"] + ) + ] + if sector.get("acronym") + else None, + ) + + +def map_location( + location: Dict, author_id: str, marking_id: str +) -> Optional[stix2.Location]: + """Map a location row to a country-level Location. + + Deliberately simplified to country only for this first pass (not state/county) -- see + OpenCTI-connector.md §7d. Returns None when there's no usable country field, rather than + emitting an empty/garbage Location. + """ + country_name = location.get("country") or location.get("country_abrv") + if not country_name: + return None + + return stix2.Location( + id=OpenCTILocation.generate_id(country_name, "Country"), + name=country_name, + country=location.get("country_abrv") or country_name, + created_by_ref=author_id, + object_marking_refs=[marking_id], + custom_properties={"x_opencti_location_type": "Country"}, + ) + + +def map_cidr_observable(network: str): + """CIDR string -> IPv4Address/IPv6Address SCO using CIDR notation directly as `value`. + + STIX 2.1 explicitly permits a CIDR block as an address SCO's value (no custom observable type + needed -- OpenCTI-connector.md §7d). SCOs get the STIX spec's own deterministic, value-derived + ID automatically; nothing here needs pycti's generate_id helpers. + """ + parsed = ip_network(network, strict=False) + if parsed.version == 4: + return stix2.IPv4Address(value=network) + return stix2.IPv6Address(value=network) + + +def normalize_timestamp(value): + """Normalize a DateField/DateTimeField value to something stix2's TimestampProperty accepts. + + CidrOrgs.first_seen/last_seen are DateField (date-only) in mini_data_lake -- see + OpenCTI-connector.md §7d -- and psycopg2 hands those back as bare datetime.date objects (or + bare "YYYY-MM-DD" strings in the IS_LOCAL JSON fixtures). stix2's TimestampProperty rejects + both; it needs a full datetime. Found by actually running the dry-run test against fixture + data (tests/test_connector_dry_run.py), not by inspection -- worth remembering as a class of + bug this codebase's DateField-vs-DateTimeField columns will keep producing elsewhere too. + """ + if value is None: + return None + if isinstance(value, datetime.datetime): + return value + if isinstance(value, datetime.date): + return datetime.datetime( + value.year, value.month, value.day, tzinfo=datetime.timezone.utc + ) + if isinstance(value, str): + if "T" in value: + return value + return f"{value}T00:00:00Z" + return value + + +def _relationship_id( + relationship_type: str, + source_ref: str, + target_ref: str, + start_time, + stop_time, + existing_id: Optional[str], +) -> str: + """Apply the §10a idempotency pattern: reuse a known ID rather than always recomputing one. + + Recomputing via generate_id() every time hashes in start_time/stop_time and would therefore + produce a *different* ID the moment stop_time changes (e.g. a CIDR being retired). Only call + generate_id() the first time this external key is seen; every later reference to it passes + back the ID connector.py already has recorded in state, so OpenCTI is told "update this exact + object" rather than left to its fuzzy ±30-day type/source/target/time window match + (docs.opencti.io/latest/usage/deduplication/) to find it on its own. + """ + if existing_id: + return existing_id + return StixCoreRelationship.generate_id( + relationship_type, source_ref, target_ref, start_time, stop_time + ) + + +def build_part_of( + child_id: str, + parent_id: str, + author_id: str, + marking_id: str, + existing_id: Optional[str] = None, +) -> stix2.Relationship: + """Org -> Sector, or child-org -> parent-org.""" + rel_id = _relationship_id("part-of", child_id, parent_id, None, None, existing_id) + return stix2.Relationship( + id=rel_id, + relationship_type="part-of", + source_ref=child_id, + target_ref=parent_id, + created_by_ref=author_id, + object_marking_refs=[marking_id], + ) + + +def build_located_at( + org_id: str, + location_id: str, + author_id: str, + marking_id: str, + existing_id: Optional[str] = None, +) -> stix2.Relationship: + """Org -> Location.""" + rel_id = _relationship_id( + "located-at", org_id, location_id, None, None, existing_id + ) + return stix2.Relationship( + id=rel_id, + relationship_type="located-at", + source_ref=org_id, + target_ref=location_id, + created_by_ref=author_id, + object_marking_refs=[marking_id], + ) + + +def build_owns_cidr( + org_id: str, + cidr_observable_id: str, + author_id: str, + marking_id: str, + first_seen=None, + last_seen_or_stop=None, + existing_id: Optional[str] = None, +) -> stix2.Relationship: + """Build the org -> CIDR observable relationship. + + Carries the CidrOrgs.first_seen/current lifecycle (§7d) as native start_time/stop_time -- + stop_time set only once CidrOrgs.current has flipped to False (retired). Same idempotency + caveat as every relationship in this file: see _relationship_id. + + CidrOrgs.first_seen/last_seen are DateField (date-only, see mapping.normalize_timestamp) -- + a CIDR first observed and marked non-current within the same calendar day normalizes to the + *same* midnight-UTC timestamp for both. STIX 2.1 requires stop_time strictly later than + start_time (verified: equal values raise ValueError, not just stop < start), so that case + would otherwise crash the entire run over one CIDR -- caught for real at full production + scale (13k+ orgs), not a theoretical edge case. Date-only precision genuinely can't + distinguish "opened and closed same day" from "still open," so dropping stop_time (treating + it as not-yet-confirmed-closed) is the honest choice here, not a workaround. + """ + start_time = normalize_timestamp(first_seen) + stop_time = normalize_timestamp(last_seen_or_stop) + if start_time is not None and stop_time is not None and stop_time <= start_time: + stop_time = None + rel_id = _relationship_id( + "related-to", org_id, cidr_observable_id, start_time, stop_time, existing_id + ) + return stix2.Relationship( + id=rel_id, + relationship_type="related-to", + source_ref=org_id, + target_ref=cidr_observable_id, + start_time=start_time, + stop_time=stop_time, + created_by_ref=author_id, + object_marking_refs=[marking_id], + ) + + +def dedupe_bundle_objects(objects: List) -> List: + """Collapse a list of STIX objects to one per id, keeping the last occurrence of each. + + Bundles built across many orgs/sectors in one run will repeat the same Identity/Location many + times (e.g. every org in the same sector references that Sector Identity). stix2's Bundle + doesn't dedupe for you -- collapse by id before sending, both to keep bundles smaller (§10b) + and because stix2.Bundle construction is order-sensitive about duplicate IDs. + """ + seen = {} + for obj in objects: + seen[obj["id"]] = obj + return list(seen.values()) diff --git a/open-cti/connectors/vs-org-bootstrap/tests/__init__.py b/open-cti/connectors/vs-org-bootstrap/tests/__init__.py new file mode 100644 index 000000000..ad726214f --- /dev/null +++ b/open-cti/connectors/vs-org-bootstrap/tests/__init__.py @@ -0,0 +1 @@ +"""Tests for the VS Organization & CIDR Bootstrap connector -- see README.md for the two loops.""" diff --git a/open-cti/connectors/vs-org-bootstrap/tests/fixtures/cidrs.json b/open-cti/connectors/vs-org-bootstrap/tests/fixtures/cidrs.json new file mode 100644 index 000000000..549481456 --- /dev/null +++ b/open-cti/connectors/vs-org-bootstrap/tests/fixtures/cidrs.json @@ -0,0 +1,18 @@ +[ + { + "cidr_retired": false, + "current": true, + "first_seen": "2025-01-01", + "last_seen": "2026-08-20", + "network": "198.51.100.0/24", + "organization_acronym": "TESTORG1" + }, + { + "cidr_retired": true, + "current": false, + "first_seen": "2024-06-01", + "last_seen": "2025-02-01", + "network": "2001:db8::/32", + "organization_acronym": "TESTORG1" + } +] diff --git a/open-cti/connectors/vs-org-bootstrap/tests/fixtures/locations_by_id.json b/open-cti/connectors/vs-org-bootstrap/tests/fixtures/locations_by_id.json new file mode 100644 index 000000000..045600478 --- /dev/null +++ b/open-cti/connectors/vs-org-bootstrap/tests/fixtures/locations_by_id.json @@ -0,0 +1,13 @@ +{ + "22222222-2222-2222-2222-222222222222": { + "country": "United States", + "country_abrv": "US", + "county": "Fairfax", + "county_fips": 59, + "gnis_id": "1480098", + "id": "22222222-2222-2222-2222-222222222222", + "name": "Fairfax, VA", + "state": "Virginia", + "state_abrv": "VA" + } +} diff --git a/open-cti/connectors/vs-org-bootstrap/tests/fixtures/organizations.json b/open-cti/connectors/vs-org-bootstrap/tests/fixtures/organizations.json new file mode 100644 index 000000000..eb7c08cd8 --- /dev/null +++ b/open-cti/connectors/vs-org-bootstrap/tests/fixtures/organizations.json @@ -0,0 +1,40 @@ +[ + { + "acronym": "TESTORG1", + "country": "US", + "country_name": "United States", + "county": "Fairfax", + "county_fips": 59, + "id": "11111111-1111-1111-1111-111111111111", + "location_id": "22222222-2222-2222-2222-222222222222", + "name": "Test Organization One", + "parent_id": null, + "retired": false, + "stakeholder": true, + "state": "VA", + "state_fips": 51, + "state_name": "Virginia", + "type": "FEDERAL", + "updated_at": "2026-08-20T12:00:00+00:00", + "vs_stakeholder": true + }, + { + "acronym": "TESTORG2-CHILD", + "country": "US", + "country_name": "United States", + "county": "Fairfax", + "county_fips": 59, + "id": "33333333-3333-3333-3333-333333333333", + "location_id": "22222222-2222-2222-2222-222222222222", + "name": "Test Organization Two (Child)", + "parent_id": "11111111-1111-1111-1111-111111111111", + "retired": false, + "stakeholder": true, + "state": "VA", + "state_fips": 51, + "state_name": "Virginia", + "type": "FEDERAL", + "updated_at": "2026-08-21T09:30:00+00:00", + "vs_stakeholder": true + } +] diff --git a/open-cti/connectors/vs-org-bootstrap/tests/fixtures/sector_memberships.json b/open-cti/connectors/vs-org-bootstrap/tests/fixtures/sector_memberships.json new file mode 100644 index 000000000..4eb55fca0 --- /dev/null +++ b/open-cti/connectors/vs-org-bootstrap/tests/fixtures/sector_memberships.json @@ -0,0 +1,5 @@ +{ + "IT": [ + "TESTORG1" + ] +} diff --git a/open-cti/connectors/vs-org-bootstrap/tests/fixtures/sectors.json b/open-cti/connectors/vs-org-bootstrap/tests/fixtures/sectors.json new file mode 100644 index 000000000..cae156f4c --- /dev/null +++ b/open-cti/connectors/vs-org-bootstrap/tests/fixtures/sectors.json @@ -0,0 +1,8 @@ +[ + { + "acronym": "IT", + "id": "44444444-4444-4444-4444-444444444444", + "name": "Information Technology", + "retired": false + } +] diff --git a/open-cti/connectors/vs-org-bootstrap/tests/test_connector_dry_run.py b/open-cti/connectors/vs-org-bootstrap/tests/test_connector_dry_run.py new file mode 100644 index 000000000..9710e56d8 --- /dev/null +++ b/open-cti/connectors/vs-org-bootstrap/tests/test_connector_dry_run.py @@ -0,0 +1,247 @@ +"""Loop 2 (OpenCTI-connector.md §9b): the real poll->map->bundle pipeline against fixtures. + +Uses a stub in place of OpenCTIConnectorHelper so no network/queue is touched at all -- this *is* +the dry-run pattern §9b describes, made concrete and actually exercised in CI/dev. + +Also the executable proof behind the §10a idempotency claim: run the pipeline twice in a row and +assert relationship ids are stable across runs when nothing changed, the way OpenCTI needs them +to be for updates (like a CIDR retiring) to land on the same object instead of a duplicate. +""" + +# Standard Python Libraries +import os +import sys + +sys.path.insert(0, os.path.join(os.path.dirname(__file__), "..")) + +# Third-Party Libraries +from pycti.utils.opencti_logger import logger as pycti_logger + +# First-Party +from src.config import Config # noqa: E402 +from src.connector import VsOrgBootstrapConnector # noqa: E402 + + +class StubHelper: + """Just enough of OpenCTIConnectorHelper's surface for _process() to run against. + + send_stix2_bundle captures instead of transmitting -- the network/queue boundary itself. + + connector_logger deliberately uses pycti's *real* AppLogger, not a plain stdlib + logging.getLogger() -- a stdlib logger would have silently accepted the old + connector_logger.error(msg, exc_info=True) and connector_logger.info(msg, %s-args) calls + that crashed for real on a live run (AppLogger.error/info only take (message, meta=None), no + lazy %%-formatting, no exc_info kwarg). Using the real class here means this dry-run test + would have caught both bugs before they ever reached a real connector run. + """ + + def __init__(self): + """Start with an empty capture list -- nothing has been "sent" yet.""" + self.connector_logger = pycti_logger(level=20, json_logging=False)( + "stub_helper" + ) + self.sent_bundles = [] + + @staticmethod + def stix2_create_bundle(items): + """Delegate to pycti's real bundle serialization rather than reimplementing it.""" + # Real pycti behavior (verified against the installed version) -- reuse it directly + # rather than reimplementing bundle serialization ourselves. + # Third-Party Libraries + from pycti import OpenCTIConnectorHelper + + return OpenCTIConnectorHelper.stix2_create_bundle(items) + + def send_stix2_bundle(self, bundle, **kwargs): + """Capture the bundle instead of sending it -- this *is* the dry-run boundary.""" + self.sent_bundles.append(bundle) + return [bundle] + + +def build_test_connector(): + """Build a VsOrgBootstrapConnector wired to fixtures and a StubHelper, for both tests below.""" + config = Config( + raw={ + "vs_org_bootstrap": { + "org_acronym_allowlist": "TESTORG1,TESTORG2-CHILD", + "is_local": True, + "local_fixture_dir": os.path.join( + os.path.dirname(__file__), "fixtures" + ), + "tlp_marking": "TLP:AMBER", + } + } + ) + return VsOrgBootstrapConnector(config=config, helper=StubHelper()) + + +def test_first_run_produces_expected_objects(): + """A single run against the fixtures should produce exactly the objects §7d's design expects.""" + connector = build_test_connector() + data = connector.repository.fetch(since_updated_at=None) + state = {} + + watermark = connector._process(data, state) + + assert ( + watermark == "2026-08-21T09:30:00+00:00" + ) # max updated_at across both fixture orgs + assert len(connector.helper.sent_bundles) == 1 + + # Standard Python Libraries + import json + + bundle = json.loads(connector.helper.sent_bundles[0]) + types = [obj["type"] for obj in bundle["objects"]] + assert types.count("identity") == 4 # author + 2 orgs + 1 sector + assert types.count("location") == 1 + assert types.count("ipv4-addr") == 1 # 198.51.100.0/24 + assert types.count("ipv6-addr") == 1 # 2001:db8::/32 + # org->sector, child->parent, 2x org->location (both orgs share one location), 2x org->cidr + assert types.count("relationship") == 6 + assert types.count("marking-definition") == 1 + + # The retired IPv6 CIDR's relationship should carry a stop_time; the active v4 one shouldn't. + cidr_rels = [ + obj + for obj in bundle["objects"] + if obj["type"] == "relationship" and obj["relationship_type"] == "related-to" + ] + assert any("stop_time" in rel for rel in cidr_rels) + assert any("stop_time" not in rel for rel in cidr_rels) + + +def test_relationship_ids_are_stable_across_consecutive_runs(): + """Prove the §10a claim executably: re-running against unchanged data must be a no-op. + + It must produce the *same* relationship ids, not new ones -- otherwise every poll would + duplicate the graph. + """ + connector = build_test_connector() + data = connector.repository.fetch(since_updated_at=None) + + state = {} + connector._process(data, state) + first_run_cidr_rel_ids = dict(state["cidr_relationship_ids"]) + first_run_part_of_ids = dict(state["part_of_relationship_ids"]) + + # Second run, same data, state carried forward exactly as run() would do it. + connector._process(data, state) + + assert state["cidr_relationship_ids"] == first_run_cidr_rel_ids + assert state["part_of_relationship_ids"] == first_run_part_of_ids + + +def test_cidr_relationship_id_stays_pinned_when_current_flips_but_stop_time_updates(): + """Prove the actual §10a contract at the connector level, not just the raw mapping function. + + test_mapping.py correctly shows generate_id() alone WOULD change the id -- the point of + connector.py's state tracking is to prevent exactly that: once an external key has a + recorded relationship id, every later write reuses that id verbatim, and the *new* stop_time + is what constitutes the update OpenCTI applies to the existing object -- confirmed directly + against send_stix2_bundle()'s own docstring ("OpenCTI always upserts data by standard id/hash + regardless of [the update] flag"). + """ + connector = build_test_connector() + data = connector.repository.fetch(since_updated_at=None) + active_cidr = next(c for c in data.cidrs if c["current"]) + org_stix_id = mapping_org_id(connector, data, active_cidr["organization_acronym"]) + rel_key = f"{org_stix_id}:{active_cidr['network']}" + + state = {} + connector._process(data, state) + open_id = state["cidr_relationship_ids"][rel_key] + + # Simulate the CIDR retiring on the next poll. + active_cidr["current"] = False + active_cidr["last_seen"] = "2026-08-25" + connector._process(data, state) + + closed_id = state["cidr_relationship_ids"][rel_key] + assert closed_id == open_id # pinned, as designed + + # Standard Python Libraries + import json + + bundle = json.loads(connector.helper.sent_bundles[-1]) + rel_obj = next(obj for obj in bundle["objects"] if obj.get("id") == closed_id) + assert ( + "stop_time" in rel_obj + ) # ...and the content that changed rides along on that same id + + +def test_process_watermark_is_json_serializable_for_real_datetime_objects(): + """Prove the watermark fix against what psycopg2 actually returns, not just fixture strings. + + organization.updated_at is a timestamptz column -- psycopg2 hands back a real + datetime.datetime for it on a live run, but every fixture in this suite stores it as an + already-JSON-safe ISO string, so this specific bug ("Object of type datetime is not JSON + serializable" from pycti's set_state() -> json.dumps()) was invisible to every other test + here. Override the fixture data with a real datetime, the way live data actually looks. + """ + # Standard Python Libraries + import datetime + import json + + connector = build_test_connector() + data = connector.repository.fetch(since_updated_at=None) + for org in data.organizations: + org["updated_at"] = datetime.datetime( + 2026, 8, 21, 9, 30, tzinfo=datetime.timezone.utc + ) + + watermark = connector._process(data, {}) + + assert isinstance(watermark, str) + json.dumps({"last_updated_at_watermark": watermark}) # must not raise + + +def test_one_bad_cidr_does_not_abort_the_whole_run(): + """§10e's row-isolation promise, actually exercised. + + One malformed CIDR previously aborted _process() entirely -- at full production scale, a + single bad row meant *zero* orgs got sent that run, not just the offending one. Uses an + invalid network string here (a different failure mode than the stop_time bug that first + surfaced this) specifically to prove the isolation is general-purpose, not a fix narrowly + scoped to that one bug. + """ + # Standard Python Libraries + import json + + connector = build_test_connector() + data = connector.repository.fetch(since_updated_at=None) + data.cidrs.append( + { + "network": "not-a-valid-cidr", + "organization_acronym": data.organizations[0]["acronym"], + "first_seen": "2026-01-01", + "last_seen": "2026-01-01", + "current": True, + } + ) + + watermark = connector._process(data, {}) + + assert watermark is not None # the run completed, it didn't raise + bundle = json.loads(connector.helper.sent_bundles[-1]) + networks_sent = [ + obj["value"] + for obj in bundle["objects"] + if obj["type"] in ("ipv4-addr", "ipv6-addr") + ] + assert "not-a-valid-cidr" not in networks_sent # the bad one was skipped + assert "198.51.100.0/24" in networks_sent # the good ones still went out + + +def mapping_org_id(connector, data, acronym): + """Recompute an org's Identity STIX id the same way connector.py does. + + Just for building the expected state-dict key in the test above. + """ + # Third-Party Libraries + from src import mapping + + org_row = next(o for o in data.organizations if o["acronym"] == acronym) + return mapping.map_organization( + org_row, connector.author.id, connector.marking.id + ).id diff --git a/open-cti/connectors/vs-org-bootstrap/tests/test_mapping.py b/open-cti/connectors/vs-org-bootstrap/tests/test_mapping.py new file mode 100644 index 000000000..54b431ce9 --- /dev/null +++ b/open-cti/connectors/vs-org-bootstrap/tests/test_mapping.py @@ -0,0 +1,245 @@ +"""Loop 1 (OpenCTI-connector.md §9b): pure unit tests against the fixtures in tests/fixtures/. + +No DB, no OpenCTI, no queue -- run with: pytest tests/test_mapping.py -v +""" + +# Standard Python Libraries +import json +import os +import sys +import uuid + +# Third-Party Libraries +import pytest + +sys.path.insert(0, os.path.join(os.path.dirname(__file__), "..")) + +# Third-Party Libraries +# First-Party +from src import mapping # noqa: E402 + +FIXTURES = os.path.join(os.path.dirname(__file__), "fixtures") + + +def load_fixture(name): + """Load one tests/fixtures/*.json file by name.""" + with open(os.path.join(FIXTURES, name), encoding="utf-8") as f: + return json.load(f) + + +@pytest.fixture +def author_id(): + """Provide a real author Identity id for tests that need one.""" + return mapping.build_author("CISA VulnScanning").id + + +@pytest.fixture +def marking_id(): + """Provide a throwaway but validly-shaped marking-definition id for tests.""" + # Real value comes from config once the §10c policy decision is made. + return "marking-definition--f88d31f6-486f-44da-b317-01333bde0b82" + + +def test_build_author_is_a_system_identity(): + """The author Identity must be class=system and deterministic by name.""" + author = mapping.build_author("CISA VulnScanning") + assert author.identity_class == "system" + assert author.name == "CISA VulnScanning" + # Deterministic: building it twice gives the same id (content-derived, not random). + assert author.id == mapping.build_author("CISA VulnScanning").id + + +def test_map_organization_basic_fields(author_id, marking_id): + """An Organization row maps to an Identity carrying the right name/class/labels/reference.""" + orgs = load_fixture("organizations.json") + org = mapping.map_organization(orgs[0], author_id, marking_id) + + assert org.name == "Test Organization One" + assert org.identity_class == "organization" + assert org.created_by_ref == author_id + assert marking_id in org.object_marking_refs + assert "vs-stakeholder" in org.labels + assert "vs-vs-stakeholder" in org.labels + assert "vs-retired" not in org.labels # retired=false in the fixture + assert org.external_references[0].external_id == "TESTORG1" + + +def test_map_organization_is_deterministic_by_name(author_id, marking_id): + """Mapping the same org row twice must produce the same Identity id.""" + orgs = load_fixture("organizations.json") + first = mapping.map_organization(orgs[0], author_id, marking_id) + second = mapping.map_organization(orgs[0], author_id, marking_id) + assert first.id == second.id # same name -> same Identity.generate_id() result + + +def test_map_organization_retired_label(author_id, marking_id): + """A retired org row must carry the vs-retired label.""" + org_row = dict(load_fixture("organizations.json")[0]) + org_row["retired"] = True + org = mapping.map_organization(org_row, author_id, marking_id) + assert "vs-retired" in org.labels + + +def test_map_sector(author_id, marking_id): + """A Sector row maps to a class="class" Identity with its acronym as an external reference.""" + sector_row = load_fixture("sectors.json")[0] + sector = mapping.map_sector(sector_row, author_id, marking_id) + assert sector.identity_class == "class" + assert sector.name == "Information Technology" + assert sector.external_references[0].external_id == "IT" + + +def test_map_location_country_only(author_id, marking_id): + """A Location row with country info maps to a country-level Location.""" + location_row = list(load_fixture("locations_by_id.json").values())[0] + location = mapping.map_location(location_row, author_id, marking_id) + assert location is not None + assert location.name == "United States" + assert location.x_opencti_location_type == "Country" + + +def test_map_location_returns_none_without_country(author_id, marking_id): + """A Location row with no usable country field maps to None, not a garbage Location.""" + assert mapping.map_location({"name": "nowhere"}, author_id, marking_id) is None + + +def test_map_cidr_observable_ipv4(): + """An IPv4 CIDR string maps to an IPv4-Addr observable using CIDR notation as its value.""" + obs = mapping.map_cidr_observable("198.51.100.0/24") + assert obs.value == "198.51.100.0/24" + assert obs.type == "ipv4-addr" + + +def test_map_cidr_observable_ipv6(): + """An IPv6 CIDR string maps to an IPv6-Addr observable using CIDR notation as its value.""" + obs = mapping.map_cidr_observable("2001:db8::/32") + assert obs.value == "2001:db8::/32" + assert obs.type == "ipv6-addr" + + +def test_cidr_observable_ids_are_deterministic(): + """Mapping the same CIDR twice must produce the same observable id.""" + # SCOs get the STIX spec's own value-derived id -- no pycti generate_id needed (§7d/mapping.py + # docstring). Confirming this actually holds for our exact usage, not just trusting the spec. + first = mapping.map_cidr_observable("198.51.100.0/24") + second = mapping.map_cidr_observable("198.51.100.0/24") + assert first.id == second.id + + +@pytest.fixture +def org_stix_id(author_id, marking_id): + """Provide the mapped Identity id for the first fixture organization.""" + return mapping.map_organization( + load_fixture("organizations.json")[0], author_id, marking_id + ).id + + +@pytest.fixture +def cidr_stix_id(): + """Provide the mapped observable id for a fixed test CIDR.""" + return mapping.map_cidr_observable("198.51.100.0/24").id + + +def test_part_of_relationship_without_existing_id_is_generated( + author_id, marking_id, org_stix_id +): + """With no existing_id, a part-of relationship gets a freshly generated STIX id.""" + sector_id = mapping.map_sector( + load_fixture("sectors.json")[0], author_id, marking_id + ).id + rel = mapping.build_part_of(org_stix_id, sector_id, author_id, marking_id) + assert rel.relationship_type == "part-of" + assert rel.source_ref == org_stix_id + assert rel.target_ref == sector_id + assert rel.id.startswith("relationship--") + + +def test_relationship_reuses_existing_id_when_given( + author_id, marking_id, org_stix_id, cidr_stix_id +): + """Test the core §10a behavior: a known relationship id must be reused verbatim. + + Recomputing instead, with a changed stop_time, would silently produce a *different* id + (StixCoreRelationship.generate_id hashes start/stop time in) -- exactly the failure mode + this pattern exists to avoid. + """ + known_id = f"relationship--{uuid.uuid4()}" + rel = mapping.build_owns_cidr( + org_stix_id, + cidr_stix_id, + author_id, + marking_id, + first_seen="2025-01-01T00:00:00Z", + last_seen_or_stop="2026-08-20T00:00:00Z", + existing_id=known_id, + ) + assert rel.id == known_id + + +def test_relationship_id_changes_with_stop_time_when_not_pinned( + author_id, marking_id, org_stix_id, cidr_stix_id +): + """Demonstrate *why* §10a's existing_id pattern is necessary. + + Without it, closing out a CIDR (setting stop_time) changes the generated id, which is + precisely what would defeat naive "just resubmit the relationship" idempotency. + """ + open_rel = mapping.build_owns_cidr( + org_stix_id, + cidr_stix_id, + author_id, + marking_id, + first_seen="2025-01-01T00:00:00Z", + last_seen_or_stop=None, + ) + closed_rel = mapping.build_owns_cidr( + org_stix_id, + cidr_stix_id, + author_id, + marking_id, + first_seen="2025-01-01T00:00:00Z", + last_seen_or_stop="2026-08-20T00:00:00Z", + ) + assert open_rel.id != closed_rel.id + + +def test_build_owns_cidr_drops_stop_time_when_not_strictly_after_start_time( + author_id, marking_id, org_stix_id, cidr_stix_id +): + """A same-day open+close must not crash. + + CidrOrgs.first_seen/last_seen are DateField (date-only) -- caught for real at full + production scale (13k+ orgs, 2026-08-27), where STIX 2.1's strict stop_time > start_time + constraint rejected the equal-timestamps case outright. + """ + same_day = mapping.build_owns_cidr( + org_stix_id, + cidr_stix_id, + author_id, + marking_id, + first_seen="2026-08-27", + last_seen_or_stop="2026-08-27", + ) + assert not hasattr(same_day, "stop_time") + assert same_day.start_time is not None # start_time itself is still preserved + + stop_before_start = mapping.build_owns_cidr( + org_stix_id, + cidr_stix_id, + author_id, + marking_id, + first_seen="2026-08-27", + last_seen_or_stop="2026-08-20", + ) + assert not hasattr(stop_before_start, "stop_time") + + +def test_dedupe_bundle_objects_collapses_repeated_ids(author_id, marking_id): + """Two STIX objects sharing an id must collapse to one in dedupe_bundle_objects().""" + org_row = load_fixture("organizations.json")[0] + same_org_twice = [ + mapping.map_organization(org_row, author_id, marking_id), + mapping.map_organization(org_row, author_id, marking_id), + ] + deduped = mapping.dedupe_bundle_objects(same_org_twice) + assert len(deduped) == 1 diff --git a/open-cti/connectors/vs-port-inventory/Dockerfile b/open-cti/connectors/vs-port-inventory/Dockerfile new file mode 100644 index 000000000..180e7853b --- /dev/null +++ b/open-cti/connectors/vs-port-inventory/Dockerfile @@ -0,0 +1,29 @@ +FROM python:3.12-slim + +WORKDIR /opt/connector + +# Zscaler TLS-inspects outbound HTTPS on this network -- identical setup to the other three VS +# connectors' Dockerfiles (see their comments/READMEs "TLS interception"). Reused verbatim: this +# is a property of the *network*, not of any one connector. +# +# Pulled from a named build context (docker-compose.yml's additional_contexts, pointed at +# /opt/open-cti/tls -- runtime state, never this repo checkout), never COPYed from anywhere +# inside this directory -- the cert never touches git. +COPY --from=zscaler_cert zscaler.pem /usr/local/share/ca-certificates/zscaler.crt +RUN apt-get update \ + && apt-get install -y --no-install-recommends ca-certificates \ + && update-ca-certificates \ + && rm -rf /var/lib/apt/lists/* +ENV PIP_CERT=/etc/ssl/certs/ca-certificates.crt \ + REQUESTS_CA_BUNDLE=/etc/ssl/certs/ca-certificates.crt \ + AWS_CA_BUNDLE=/etc/ssl/certs/ca-certificates.crt \ + SSL_CERT_FILE=/etc/ssl/certs/ca-certificates.crt + +COPY requirements.txt . +RUN pip install --no-cache-dir -r requirements.txt + +COPY src/ src/ +COPY tests/ tests/ + +# Relative imports in src/main.py require running as a module, not a script -- see README.md. +ENTRYPOINT ["python3", "-m", "src.main"] diff --git a/open-cti/connectors/vs-port-inventory/README.md b/open-cti/connectors/vs-port-inventory/README.md new file mode 100644 index 000000000..5469b1dc1 --- /dev/null +++ b/open-cti/connectors/vs-port-inventory/README.md @@ -0,0 +1,156 @@ +# VS Port/Service Inventory Connector + +Connector C in the four-connector VS architecture — see +[`../../OpenCTI-connector.md`](../../OpenCTI-connector.md) §7c for the full design and §10 for +the implementation discipline this code follows. + +Depends on [connector D](../vs-org-bootstrap/) (org Identity must exist) — otherwise independent +of connectors A/B. + +## What it does — and the one thing that makes it structurally different from A/D + +Reads `LatestPortScan` — already deduped to one row per `(organization, ip, port, protocol)` by +the sync task's own upsert, i.e. "current state of the world," not a raw scan log — and produces +`IPv4-Addr`/`IPv6-Addr` (reused from connector A), `Network-Traffic`, and (when scanner-reported +product/version/cpe detail exists) `Software` objects, plus the relationships tying them to their +owning org. + +**No timestamp watermark is possible here (§7c's core gotcha).** +`mark_stale_latest_port_scans()` (`backend/.../vs_port_scans.py:771`) flips a port's `current` to +`False` after 14 days via a plain `UPDATE ... SET current = FALSE WHERE time_scanned < cutoff` — +confirmed by reading that function directly — and it never touches `time_scanned` itself. A +`WHERE time_scanned > watermark` poll would never see that transition and would report a closed +port as open forever. So this connector polls the **entire in-scope table every run** and diffs +it against a `(org, ip, port, protocol) -> last-known-state` map kept in connector state — closer +to a reconciliation loop than to connector A's incremental polls. + +## Structure + +```text +src/ + config.py # env/yaml config loading, fails closed on tlp_marking + org scope + db.py # mini_data_lake access + IS_LOCAL fixture fallback -- one full-table query + mapping.py # pure row -> STIX mapping functions (no I/O) -- Loop 1 test target + connector.py # orchestration: full poll -> diff against state -> bundle only what changed + main.py # entry point +tests/ + fixtures/ # small synthetic sample rows, shaped like real mini_data_lake queries + test_mapping.py # Loop 1: pure unit tests, no DB/OpenCTI/queue at all + test_connector_dry_run.py # Loop 2: full reconciliation pipeline against fixtures +``` + +Run the fast loops constantly: + +```bash +pip install -r requirements.txt +python3 -m pytest tests/ -v +``` + +## Two things this connector refuses to start without + +- **`VS_PORT_INVENTORY_TLP_MARKING`** — must match the other three connectors' value once the + TLP/marking policy decision (§10c) is actually made. +- **`VS_PORT_INVENTORY_ORG_ACRONYM_ALLOWLIST`** (or explicit + `VS_PORT_INVENTORY_ALLOW_UNSCOPED_RUN=true`) — matters *more* here than for connectors A/D: since + this connector always does a full poll (never an incremental one), an accidentally unscoped run + means re-pulling and re-diffing the entire `LatestPortScan` table on every single manual dev + run, not just a bounded incremental slice. + +## A STIX id gotcha found and steered around before it could bite, not after + +`stix2.NetworkTraffic`'s `start`/`end` are genuine ID Contributing Properties — verified directly +against the installed `stix2` library that two otherwise-identical `NetworkTraffic` objects with +different `start` values get *different* ids. Putting this port's open/stale lifecycle directly +on the `NetworkTraffic` SCO (the STIX-native-looking choice) would have fragmented one port's +history into a new object every time it changed, instead of updating one object in place — the +third time this exact bug class has shown up across the four connectors (Notes' content-hash for +Connector B, relationships' `start_time`/`stop_time` for Connector A/D). The `NetworkTraffic` SCO +here is built from stable fields only; the lifecycle lives entirely on a separate, +`§10a`-pinned `Relationship` (org → Network-Traffic), the same idempotency pattern every other +connector already uses. + +## Service/state summary on the Network-Traffic observable + +The Network-Traffic SCO carries a service/state summary readable straight off the observable, +without following its relationship to the owning org — as OpenCTI **Labels**, not the +`x_opencti_service`/`x_opencti_open` custom properties an earlier version of this connector tried. + +**Why not `x_opencti_*` custom properties:** verified directly against this platform's real +GraphQL schema (`__type(name: "NetworkTraffic") { fields { name } }`) that it only recognizes a +fixed set of `x_opencti_*` keys for this entity type (`x_opencti_stix_ids`, +`x_opencti_modified_at`, `x_opencti_inferences`, `x_opencti_score`, `x_opencti_description`). +Anything else — `x_opencti_service`, `x_opencti_open` — gets **silently dropped on ingest**, not +stored anywhere, not just hidden in some "Additional information" panel. Confirmed with a real +throwaway bundle sent through a live instance and read back via GraphQL: `x_opencti_description` +survives (already relied on below), an unrecognized `x_opencti_*` key does not. + +**What actually works, verified the same way:** a bare `labels` custom property (no `x_opencti_` +prefix) *does* land as `objectLabel` — a real, visible field on this entity type. So +`mapping.network_traffic_labels()` puts three labels on the SCO: + +- **`vs-service-`** — the scanner-reported service name (e.g. `vs-service-https`), when + known. `x_opencti_description` (OpenCTI's built-in Description field) still also gets the raw + service name, unchanged from before and confirmed working independently of this label. +- **`vs-state-`** — the scanner's raw, most-recently-confirmed `LatestPortScan.state` + (`open`/`closed`/`filtered`/`open|filtered`/...) for this exact port. The exact same value + `lifecycle_labels()` already puts on the relationship, for the same row. +- **`vs-open`** — present only when `state == "open"` (`mapping.is_port_state_open()`), a + presence-only tag for a quick visual filter. + +This is **not** this connector's own recency/staleness tracking (`current`, driven by +`time_scanned` vs `latest_port_scan_cutoff_days`) — an earlier version of this code conflated the +two; that was wrong and got corrected before shipping. `state` is genuinely overwritten on every +rescan (`insert_port_scans_sql()`'s `ON CONFLICT ... DO UPDATE`), and the platform's own +`PortScanSummary` rollup already treats `state = 'open'` as the authoritative "is this port open" +check, so this connector matches that same convention. + +Labels are safe to vary between polls without fragmenting the SCO into a new object each time — +verified directly against the installed `stix2` library that custom properties (including +`labels`) don't feed `NetworkTraffic`'s ID Contributing Properties (same verification already done +for `start`/`end`, see the STIX id gotcha section above), and confirmed end-to-end that `labels` +itself is a real accepted field for this SCO type. + +**Deliberately not touched by the aging sweep.** `_close_locally()` (a port aging out with no +fresh row on hand) updates only the lifecycle relationship's `current`/`stop_time` — never these +labels on the SCO. Going unobserved for `latest_port_scan_cutoff_days` is evidence this connector +no longer trusts the record enough to keep tracking it as active, not evidence the port itself +closed; the last scanner-confirmed state stays exactly as it was until an actual rescan says +otherwise. + +## Running it + +Same two modes as the other three connectors — see connector D's README for the fuller +explanation of each. `IS_LOCAL=true` loads `tests/fixtures/port_scans.json`; against real +`mini_data_lake` + OpenCTI, copy `config.yml.sample` to `config.yml`, fill in `db_host` and a +real `org_acronym_allowlist` scoped to one or two low-volume test orgs, then `python3 -m src.main`. + +## TLS interception (Zscaler) + +Identical setup to the other three connectors — see +[connector D's README](../vs-org-bootstrap/README.md#tls-interception-zscaler--required-manual-per-instance-setup-not-git-managed). +Already set up once for this network. + +## Known gaps / things to verify before this is production-ready + +- **A key that vanishes from the source entirely (not just `current` flipping to `False`) is + left untouched in state, not revoked.** `state["port_scan_state"]` is only ever added to or + updated for rows actually seen in a given poll — never wholesale-replaced — specifically so a + partial/empty poll (a transient DB issue, or a deliberately narrowed org scope) can't silently + erase previously-recorded history. The tradeoff: a genuine hard delete of a `LatestPortScan` + row (unconfirmed whether this ever happens — `mark_stale_latest_port_scans()` only sets + `current=False`, never deletes) would leave a stale relationship un-revoked forever. Same + category of known gap as connector B's un-revoked stale Note. +- **Row-count growth is structurally different from connector A's backlog.** `LatestPortScan` + rows never get deleted when a port goes stale, only flagged — so the in-scope row count only + ever grows over time, unlike a ticket backlog a watermark eventually catches up past. Real row + counts for this table are still unconfirmed (`OpenCTI-connector.md` §6); size + `max_vulnscan_rows_per_entity`'s sibling here, `max_rows_per_run`, off real numbers once known, + the same way connector A's was resized off its own real backlog count. +- **Reopened-port semantics (clear `stop_time`, keep the original `start_time`) are a documented + judgment call, not something the design doc explicitly specifies** — see + `connector.py`'s `_resolve_lifecycle()` docstring for the reasoning. +- Not yet run against a real OpenCTI instance — the fixture-based dry-run pipeline (Loop 2, §9b) + and a throwaway real-Postgres check (Loop 4 — confirmed `current` returns a real Python `bool`, + `time_scanned` a real `datetime`, and the join/scoping query plan works as expected) are both + done; the actual send-and-verify-in-OpenCTI step is next, the same order every other connector + in this set went through. diff --git a/open-cti/connectors/vs-port-inventory/config.yml.sample b/open-cti/connectors/vs-port-inventory/config.yml.sample new file mode 100644 index 000000000..274007a5d --- /dev/null +++ b/open-cti/connectors/vs-port-inventory/config.yml.sample @@ -0,0 +1,51 @@ +# Connector C — VS Port/Service Inventory +# +# See ../../OpenCTI-connector.md §7c for the design this implements, and §10 for the discipline +# this config exists to enforce. +# +# Copy to config.yml for local runs (docker-compose.yml drives the containerized version purely +# from env vars instead — see README.md). + +opencti: + url: 'http://localhost:8080' + token: 'ChangeMe' + +connector: + id: 'ChangeMe-generate-a-uuid4' + name: 'VS Port/Service Inventory' + scope: 'ipv4-addr,ipv6-addr,network-traffic,software' + type: 'EXTERNAL_IMPORT' + confidence_level: 75 + log_level: 'info' + duration_period: 'P1D' + +vs_port_inventory: + # --- mini_data_lake connectivity (OpenCTI-connector.md §2) --- + db_host: 'ChangeMe' + db_port: 5432 + db_name: 'cyhy_mini_data_lake_staging' + db_user: 'open_cti' + db_use_iam_auth: true + db_password: '' + aws_region: 'us-gov-west-1' + + # --- dev/test discipline (§9) --- + is_local: false + local_fixture_dir: './tests/fixtures' + + # §9c -- REQUIRED to be non-empty outside of a deliberate production run, same reason as + # connectors A/D. + org_acronym_allowlist: '' + allow_unscoped_run: false + # Bounds both the bootstrap poll (which can still be very large -- every row the source + # considers `current`, regardless of age) and every steady-state poll after it. Size this off + # real numbers once known -- even a plain aggregate query against this table timed out in the + # bastion Lambda (2026-08-31). + max_rows_per_run: 5000 + # This connector's own copy of LATEST_PORT_SCAN_CUTOFF (backend/.../vs_port_scans.py, default + # 14) -- staleness is computed locally from this instead of re-polling for it. See db.py. + latest_port_scan_cutoff_days: 14 + + # --- attribution / marking (§10c) --- + author_name: 'CISA VulnScanning' + tlp_marking: '' diff --git a/open-cti/connectors/vs-port-inventory/requirements.txt b/open-cti/connectors/vs-port-inventory/requirements.txt new file mode 100644 index 000000000..17d9c9b7f --- /dev/null +++ b/open-cti/connectors/vs-port-inventory/requirements.txt @@ -0,0 +1,5 @@ +boto3==1.43.81 +psycopg2-binary==2.9.9 +pycti==7.260824.0 +python-dateutil==2.9.0.post0 +stix2==3.0.2 diff --git a/open-cti/connectors/vs-port-inventory/src/__init__.py b/open-cti/connectors/vs-port-inventory/src/__init__.py new file mode 100644 index 000000000..a07651154 --- /dev/null +++ b/open-cti/connectors/vs-port-inventory/src/__init__.py @@ -0,0 +1 @@ +"""VS Port/Service Inventory connector -- see ../README.md and ../../OpenCTI-connector.md.""" diff --git a/open-cti/connectors/vs-port-inventory/src/config.py b/open-cti/connectors/vs-port-inventory/src/config.py new file mode 100644 index 000000000..49ef35f3a --- /dev/null +++ b/open-cti/connectors/vs-port-inventory/src/config.py @@ -0,0 +1,190 @@ +"""Config loading for the VS Port/Service Inventory connector. + +Same fail-closed shape as connectors A/D's config.py (§10i). Same +`org_acronym_allowlist`/`allow_unscoped_run` pair A/D use to guard against an accidentally +unscoped run (§9c). + +**Revised (2026-08-31): windowed/watermark polling, matching connector A's shape -- see db.py's +module docstring for the full reasoning.** The original full-poll-every-run design turned out to +be too expensive against the real table size (a plain `GROUP BY` aggregate over it timed out even +in a Lambda with its own execution budget) -- replaced with an ordinary watermark poll plus a +locally-computed staleness rule, so this connector no longer needs to re-read the entire in-scope +table just to detect a port going stale. `latest_port_scan_cutoff_days` below is this connector's +own copy of that rule. +""" + +# Standard Python Libraries +import os +from typing import Dict, List, Optional + +# Third-Party Libraries +from pycti import get_config_variable +import yaml + + +class ConfigError(RuntimeError): + """Raised when required configuration is missing -- fail closed, not silent default.""" + + +def load_yaml_config() -> Dict: + """Load config.yml next to this connector if present; docker-compose drives env vars only.""" + here = os.path.dirname(os.path.abspath(__file__)) + config_file_path = os.path.join(here, "..", "config.yml") + if os.path.isfile(config_file_path): + with open(config_file_path, encoding="utf-8") as f: + return yaml.safe_load(f) or {} + return {} + + +class Config: + """Typed view over this connector's configuration.""" + + def __init__(self, raw: Optional[Dict] = None): + """Load config from `raw` if given (tests), else config.yml/env vars, then validate.""" + self.raw: Dict = raw if raw is not None else load_yaml_config() + + # --- mini_data_lake connectivity (OpenCTI-connector.md §2) --- + self.db_host = get_config_variable( + "VS_PORT_INVENTORY_DB_HOST", ["vs_port_inventory", "db_host"], self.raw + ) + self.db_port = get_config_variable( + "VS_PORT_INVENTORY_DB_PORT", + ["vs_port_inventory", "db_port"], + self.raw, + isNumber=True, + default=5432, + ) + self.db_name = get_config_variable( + "VS_PORT_INVENTORY_DB_NAME", + ["vs_port_inventory", "db_name"], + self.raw, + default="cyhy_mini_data_lake_staging", + ) + self.db_user = get_config_variable( + "VS_PORT_INVENTORY_DB_USER", + ["vs_port_inventory", "db_user"], + self.raw, + default="open_cti", + ) + self.db_use_iam_auth = get_config_variable( + "VS_PORT_INVENTORY_DB_USE_IAM_AUTH", + ["vs_port_inventory", "db_use_iam_auth"], + self.raw, + default=True, + ) + self.db_password = get_config_variable( + "VS_PORT_INVENTORY_DB_PASSWORD", + ["vs_port_inventory", "db_password"], + self.raw, + default="", + ) + self.aws_region = get_config_variable( + "VS_PORT_INVENTORY_AWS_REGION", + ["vs_port_inventory", "aws_region"], + self.raw, + ) + + # --- dev/test discipline (§9) --- + self.is_local = get_config_variable( + "IS_LOCAL", ["vs_port_inventory", "is_local"], self.raw, default=False + ) + self.local_fixture_dir = get_config_variable( + "VS_PORT_INVENTORY_LOCAL_FIXTURE_DIR", + ["vs_port_inventory", "local_fixture_dir"], + self.raw, + default="./tests/fixtures", + ) + + org_allowlist_raw = get_config_variable( + "VS_PORT_INVENTORY_ORG_ACRONYM_ALLOWLIST", + ["vs_port_inventory", "org_acronym_allowlist"], + self.raw, + default="", + ) + self.org_acronym_allowlist: List[str] = [ + acronym.strip() + for acronym in (org_allowlist_raw or "").split(",") + if acronym.strip() + ] + self.allow_unscoped_run = get_config_variable( + "VS_PORT_INVENTORY_ALLOW_UNSCOPED_RUN", + ["vs_port_inventory", "allow_unscoped_run"], + self.raw, + default=False, + ) + # Same role as connector A's own cap: bounds both the bootstrap poll (which can still be + # "monumental" -- the source may consider a huge number of rows `current = TRUE` at once, + # same shape as A's 413,925-stale-open-ticket discovery) and every steady-state poll + # after it. Real row counts for this table are still unconfirmed (OpenCTI-connector.md + # §6) -- even a plain aggregate query against it timed out in the bastion Lambda + # (2026-08-31) -- start conservative, same as A/D did before their own real numbers were + # known, and size this up deliberately once real numbers exist. + self.max_rows_per_run = get_config_variable( + "VS_PORT_INVENTORY_MAX_ROWS_PER_RUN", + ["vs_port_inventory", "max_rows_per_run"], + self.raw, + isNumber=True, + default=5000, + ) + # This connector's own copy of LATEST_PORT_SCAN_CUTOFF + # (backend/.../vs_port_scans.py:36, default 14) -- see db.py's module docstring for why + # staleness is computed locally from this instead of re-polling for it. A mismatch + # between this and whatever the sync task actually has deployed only causes soft drift + # in exactly when we mark something stale, not silent data loss -- anything genuinely + # rescanned still reconciles correctly via the ordinary watermark poll regardless. + self.latest_port_scan_cutoff_days = get_config_variable( + "VS_PORT_INVENTORY_CUTOFF_DAYS", + ["vs_port_inventory", "latest_port_scan_cutoff_days"], + self.raw, + isNumber=True, + default=14, + ) + + # --- attribution / marking (§10c) --- + self.author_name = get_config_variable( + "VS_PORT_INVENTORY_AUTHOR_NAME", + ["vs_port_inventory", "author_name"], + self.raw, + default="CISA VulnScanning", + ) + self.tlp_marking = get_config_variable( + "VS_PORT_INVENTORY_TLP_MARKING", + ["vs_port_inventory", "tlp_marking"], + self.raw, + ) + + # §7c doesn't depend on VulnScanningSync's ticket-specific cadence the way A does, but + # still shouldn't poll tighter than the source data actually changes. Same P1D starting + # point as A, for the same reason -- tune once real `VulnScanningSync` cadence is known. + self.duration_period = get_config_variable( + "CONNECTOR_DURATION_PERIOD", + ["connector", "duration_period"], + self.raw, + default="P1D", + ) + + self._validate() + + def _validate(self) -> None: + """Fail closed on the two decisions this doc can't make silently. See module docstring.""" + if not self.tlp_marking: + raise ConfigError( + "vs_port_inventory.tlp_marking is not set. This is a real, undecided policy " + "question (OpenCTI-connector.md §10c) -- refusing to start rather than default " + "to something like TLP:CLEAR for stakeholder vulnerability data. Set " + "VS_PORT_INVENTORY_TLP_MARKING once the data-handling policy call is made." + ) + if not self.org_acronym_allowlist and not self.allow_unscoped_run: + raise ConfigError( + "vs_port_inventory.org_acronym_allowlist is empty and allow_unscoped_run is not " + "set. An empty allowlist means 'scoped to nothing', not 'all orgs' -- see " + "OpenCTI-connector.md §9c. Set VS_PORT_INVENTORY_ORG_ACRONYM_ALLOWLIST for dev, " + "or explicitly set VS_PORT_INVENTORY_ALLOW_UNSCOPED_RUN=true for a deliberate " + "full-scope production run." + ) + if not self.is_local and not self.db_host: + raise ConfigError( + "vs_port_inventory.db_host is not set and is_local is false -- nothing to " + "connect to. Set VS_PORT_INVENTORY_DB_HOST, or IS_LOCAL=true to use fixtures " + "instead." + ) diff --git a/open-cti/connectors/vs-port-inventory/src/connector.py b/open-cti/connectors/vs-port-inventory/src/connector.py new file mode 100644 index 000000000..34bba3be2 --- /dev/null +++ b/open-cti/connectors/vs-port-inventory/src/connector.py @@ -0,0 +1,382 @@ +"""Orchestration for the VS Port/Service Inventory connector (OpenCTI-connector.md §7c). + +Depends on connector D (org Identity must exist) -- independent of connectors A/B otherwise. + +**Revised (2026-08-31): watermark-windowed polling, matching connector A's shape.** The original +version of this connector polled the *entire* in-scope `LatestPortScan` table every run, because +`mark_stale_latest_port_scans()` flips `current` to `False` without ever touching `time_scanned` +(verified directly against that function's source -- see db.py) -- a plain `WHERE time_scanned > +watermark` poll can't see that transition. That turned out to be too expensive against the real +table size (a plain aggregate query against it timed out even in a Lambda, 2026-08-31). + +The fix isn't a bigger row cap, it's not needing a query to observe that transition at all: +`_run_aging_sweep()` below applies the *exact same* staleness rule +(`time_scanned` older than `config.latest_port_scan_cutoff_days`) locally, against connector +state, on every run -- independent of whatever this run's DB query happened to return. That turns +this back into an ordinary watermark poll (`db.py`'s `since_last_seen`/`include_current`, the +same shape as connector A's `since_last_seen`/`include_stale_open`), with staleness detection +handled entirely in-process instead of by re-reading the source. + +**Revised (2026-09-02):** the Network-Traffic SCO now also carries a service/state summary as +OpenCTI Labels -- `vs-state-`/`vs-open`/`vs-service-` (see mapping.py's +`network_traffic_labels()` and its module docstring for why labels, not the `x_opencti_service`/ +`x_opencti_open` custom properties an earlier version of this tried, which turned out not to be +real fields on this platform's Network-Traffic schema and were being silently dropped on ingest). +The underlying state is still the scanner's own confirmed port state (`LatestPortScan.state == +"open"`), not this connector's recency tracking -- so unlike the lifecycle relationship, +`_close_locally()` below deliberately leaves the SCO untouched when a port ages out locally; going +unobserved for 14+ days isn't evidence a port closed. +""" + +# Standard Python Libraries +import datetime +import logging +from typing import Dict, List, Optional + +# Third-Party Libraries +from pycti import OpenCTIConnectorHelper +import stix2 + +from . import mapping +from .config import Config +from .db import PortInventoryData, VsPortInventoryRepository + +LOGGER = logging.getLogger("vs_port_inventory") + +# Same well-known TLP marking-definition ids as connectors A/B/D. +_TLP_MARKINGS = { + "TLP:CLEAR": stix2.TLP_WHITE, + "TLP:WHITE": stix2.TLP_WHITE, + "TLP:GREEN": stix2.TLP_GREEN, + "TLP:AMBER": stix2.TLP_AMBER, + "TLP:RED": stix2.TLP_RED, +} + + +def resolve_marking(tlp_marking: str) -> stix2.MarkingDefinition: + """Resolve a TLP string to its well-known marking-definition object. Identical to A/B/D.""" + marking = _TLP_MARKINGS.get(tlp_marking.strip().upper()) + if marking is None: + raise ValueError( + f"Unrecognized tlp_marking {tlp_marking!r}. Expected one of: " + f"{', '.join(_TLP_MARKINGS)} (see OpenCTI-connector.md §10c)." + ) + return marking + + +class VsPortInventoryConnector: + """Polls an incremental window of LatestPortScan and locally ages out stale entries. + + Produces IPv4-Addr/IPv6-Addr, Network-Traffic, and (when present) Software objects, plus the + relationships tying them to their owning org -- with the port's open/stale lifecycle carried + on a pinned relationship, never on the Network-Traffic SCO itself (see mapping.py). + """ + + def __init__(self, config: Optional[Config] = None, helper=None): + """Build the connector, injecting `config`/`helper` for tests instead of the real thing.""" + self.config = config or Config() + self.helper = helper or OpenCTIConnectorHelper(self.config.raw) + self.repository = VsPortInventoryRepository(self.config) + + self.author = mapping.build_author(self.config.author_name) + self.marking = resolve_marking(self.config.tlp_marking) + + LOGGER.info( + "VS Port Inventory connector initialized (is_local=%s, org_allowlist=%s)", + self.config.is_local, + self.config.org_acronym_allowlist or "", + ) + + def run(self) -> None: + """Entry point pycti's scheduler calls each duration_period tick.""" + self.helper.connector_logger.info("Starting VS Port Inventory run") + state = self.helper.get_state() or {} + + # is_first_run gates the bootstrap `include_current` fetch -- the same role + # is_first_run/include_stale_open play in connector A's run(), applied to this table's + # own "don't permanently miss something currently active" case. + since = state.get("last_seen_watermark") + is_first_run = since is None + + try: + data = self.repository.fetch( + since_last_seen=since, include_current=is_first_run + ) + new_watermark = self._process(data, state) + except Exception as e: # pylint: disable=broad-except + # §10e/§10i: don't let one bad run corrupt state. connector_logger is pycti's + # AppLogger, not a stdlib logging.Logger -- (message, meta=None) only. + self.helper.connector_logger.error( + "VS Port Inventory run failed", meta={"error": str(e)} + ) + return + + if new_watermark is not None: + state["last_seen_watermark"] = new_watermark + # Persisted regardless of whether the watermark advanced -- the aging sweep can mutate + # state (marking something locally stale) even on a run that fetched nothing new. + self.helper.set_state(state) + self.helper.connector_logger.info("VS Port Inventory run complete") + + # ------------------------------------------------------------------ + + def _process(self, data: PortInventoryData, state: Dict) -> Optional[str]: + """Process this run's fetched window, then age out anything state alone says is stale. + + `state["port_scan_state"]` is only ever added to or updated for keys actually seen this + run, or found stale by the aging sweep -- never wholesale-replaced -- so a partial or + empty poll (a transient DB issue, or a deliberately narrowed org scope) can't silently + erase previously-recorded history for keys it simply didn't touch. A key that vanishes + from the source entirely (as opposed to aging past the cutoff, which this connector now + detects on its own) is a known, documented gap -- see README.md "Known gaps," same + category as connector B's un-revoked stale Note. + + Returns the new watermark to persist (max `time_scanned` actually fetched this run), or + None if nothing was fetched -- mirrors connector A's watermark contract exactly. + """ + objects: List = [self.author, self.marking] + port_scan_state = state.setdefault("port_scan_state", {}) + org_ip_rel_map = state.setdefault("org_ip_relationship_ids", {}) + run_time = datetime.datetime.now(datetime.timezone.utc) + + seen_keys = set() + # §10e/§10i: one bad row must not sink the whole run -- built in from the start. + for row in data.port_scans: + key = self._key(row) + seen_keys.add(key) + try: + self._process_row( + row, key, port_scan_state, org_ip_rel_map, run_time, objects + ) + except Exception as e: # pylint: disable=broad-except + self.helper.connector_logger.warning(f"Skipping port scan {key!r}: {e}") + continue + + self._run_aging_sweep(port_scan_state, seen_keys, run_time, objects) + + # §9 queue discipline: a poll where nothing actually changed for any row has nothing + # new to send -- author+marking alone shouldn't hit the queue. + if len(objects) > 2: + objects = mapping.dedupe_bundle_objects(objects) + bundle = self.helper.stix2_create_bundle(objects) + self.helper.send_stix2_bundle(bundle, update=True) + self.helper.connector_logger.info( + f"Sent {len(objects)} objects ({len(data.port_scans)} port scans fetched this run)" + ) + else: + self.helper.connector_logger.info( + f"Nothing changed ({len(data.port_scans)} port scans fetched this run)" + ) + + return self._compute_watermark(data.port_scans) + + def _process_row( + self, row, key, port_scan_state, org_ip_rel_map, run_time, objects + ) -> None: + """Map, diff, and (if changed) queue one freshly-fetched LatestPortScan row's objects.""" + time_scanned = row.get("time_scanned") + if not time_scanned: + raise ValueError("LatestPortScan row has no time_scanned") + + org_id = mapping.resolve_org_identity_id(row["organization_name"]) + ip_obj = mapping.map_ip_observable(row["ip_string"]) + nt_obj = mapping.build_network_traffic(row, ip_obj.id) + + prev = port_scan_state.get(key) + # Locally computed, deliberately ignoring row["current"] -- see module docstring. Using + # our own rule uniformly (whether a row was just fetched or aged between polls) means a + # bootstrap-caught row that already looks stale by our own clock behaves identically to + # one the aging sweep would catch next run, rather than two different sources of truth. + # Note this is recency ("current"), a wholly separate concept from nt_obj's own + # vs-state-*/vs-open labels above -- see mapping.py's module docstring for why those + # must not be conflated (an earlier version of this code got that wrong). + is_current = self._is_within_cutoff(time_scanned, run_time) + start_time, stop_time = self._resolve_lifecycle(prev, row, is_current, run_time) + + labels = mapping.lifecycle_labels(row) + external_id = mapping.lifecycle_external_id(row) + lifecycle_rel = mapping.build_lifecycle_relationship_from_parts( + org_id, + nt_obj.id, + self.author.id, + self.marking.id, + start_time=start_time, + stop_time=stop_time, + labels=labels, + external_id=external_id, + existing_id=(prev or {}).get("relationship_id"), + ) + + org_ip_key = f"{org_id}:{ip_obj.id}" + org_ip_rel = mapping.build_org_owns_ip( + org_id, + ip_obj.id, + self.author.id, + self.marking.id, + existing_id=org_ip_rel_map.get(org_ip_key), + ) + org_ip_rel_map[org_ip_key] = org_ip_rel.id + + software_obj = mapping.build_software(row) + software_rel = None + if software_obj is not None: + software_rel = mapping.build_software_relationship( + nt_obj.id, + software_obj.id, + self.author.id, + self.marking.id, + existing_id=(prev or {}).get("software_relationship_id"), + ) + + new_state = { + "relationship_id": lifecycle_rel.id, + "network_traffic_id": nt_obj.id, + "org_id": org_id, + "labels": labels, + "external_id": external_id, + "start_time": _serialize(start_time), + "stop_time": _serialize(stop_time), + "current": is_current, + "time_scanned": _serialize(mapping.normalize_timestamp(time_scanned)), + "service_name": row.get("service_name"), + # The scanner-confirmed state, cached so a pure open<->closed flip (no other field + # changing) is still detected below and resent -- see mapping.py's module docstring + # for why this is not the same thing as "current". + "port_state_open": mapping.is_port_state_open(row.get("state")), + "software_relationship_id": software_rel.id if software_rel else None, + } + changed = prev is None or any( + prev.get(field) != new_state[field] + for field in ( + "current", + "service_name", + "port_state_open", + "software_relationship_id", + "stop_time", + "labels", + ) + ) + port_scan_state[key] = new_state + + if changed: + objects.append(ip_obj) + objects.append(nt_obj) + objects.append(lifecycle_rel) + objects.append(org_ip_rel) + if software_obj is not None: + objects.append(software_obj) + objects.append(software_rel) + + def _run_aging_sweep(self, port_scan_state, seen_keys, run_time, objects) -> None: + """Locally close out anything state alone says has aged past the cutoff. + + The whole point of this connector's redesign (see module docstring): no DB query here, + just the same rule `mark_stale_latest_port_scans()` applies, evaluated against what + connector state already knows. Only ever touches entries this run's fetch didn't already + update -- a freshly-fetched row's own currency was already decided in _process_row(). + """ + for key, entry in port_scan_state.items(): + if key in seen_keys or not entry.get("current"): + continue + last_scanned = mapping.normalize_timestamp(entry.get("time_scanned")) + if last_scanned is None: + continue # defensive -- don't let one malformed entry crash the whole sweep + if run_time - last_scanned < datetime.timedelta( + days=self.config.latest_port_scan_cutoff_days + ): + continue + self._close_locally(key, entry, run_time, objects) + + def _close_locally(self, key, entry, run_time, objects) -> None: + """Build an updated lifecycle relationship for one aged-out entry, from state alone. + + Deliberately does NOT touch the Network-Traffic SCO's own vs-state-*/vs-open labels -- + those track the scanner's last-confirmed state (mapping.py's module docstring), and going + stale here means "unobserved for 14+ days," not "confirmed closed." Only the + relationship's own current/stop_time -- this connector's tracking of whether the entry is + still fresh enough to trust -- changes when nothing but time has passed. + """ + rel = mapping.build_lifecycle_relationship_from_parts( + entry["org_id"], + entry["network_traffic_id"], + self.author.id, + self.marking.id, + start_time=mapping.normalize_timestamp(entry.get("start_time")), + stop_time=run_time, + labels=entry.get("labels") or [], + external_id=entry["external_id"], + existing_id=entry["relationship_id"], + ) + entry["current"] = False + entry["stop_time"] = _serialize(run_time) + objects.append(rel) + self.helper.connector_logger.info( + f"Marked {key!r} stale locally " + f"(time_scanned aged past {self.config.latest_port_scan_cutoff_days}d cutoff)" + ) + + def _is_within_cutoff(self, time_scanned, run_time) -> bool: + """Apply this connector's own copy of LATEST_PORT_SCAN_CUTOFF to a raw time_scanned value.""" + last_scanned = mapping.normalize_timestamp(time_scanned) + return (run_time - last_scanned) < datetime.timedelta( + days=self.config.latest_port_scan_cutoff_days + ) + + @staticmethod + def _resolve_lifecycle(prev, row, is_current, run_time): + """§7c: preserve start_time once set; pin stop_time at first-observed-closed, not later. + + A port that reopens after being marked stale clears its stop_time (the same relationship + goes active again) but keeps its *original* start_time -- resetting start_time on reopen + would overstate how novel that exposure is. Not explicitly spelled out in the design doc; + a documented judgment call, consistent with how "preserve start_time, never overwrite" + is stated there for the first-seen case. + """ + if prev is None: + start_time = ( + mapping.normalize_timestamp(row.get("time_scanned")) or run_time + ) + stop_time = None if is_current else run_time + else: + start_time = mapping.normalize_timestamp(prev.get("start_time")) + if is_current: + stop_time = None + else: + stop_time = ( + mapping.normalize_timestamp(prev["stop_time"]) + if prev.get("stop_time") + else run_time + ) + return start_time, stop_time + + @staticmethod + def _compute_watermark(port_scans) -> Optional[str]: + """Compute the max time_scanned across everything actually fetched this run. + + Connector A's exact watermark contract, just a single timestamp column instead of a + COALESCE of two. + """ + values = [ + mapping.normalize_timestamp(row.get("time_scanned")) for row in port_scans + ] + watermark = max((v for v in values if v is not None), default=None) + return _serialize(watermark) + + @staticmethod + def _key(row: Dict) -> str: + """Build the reconciliation key this connector diffs on: (org, ip, port, protocol).""" + return ( + f"{row.get('organization_acronym')}:{row.get('ip_string')}:" + f"{row.get('port')}:{row.get('protocol')}" + ) + + +def _serialize(value) -> Optional[str]: + """isoformat() a datetime for JSON-safe state storage; pass through None/strings as-is. + + Same live-vs-fixture type gap as connectors A/D's watermark serialization (§10i): + psycopg2/normalize_timestamp hand back real datetime.datetime objects, but pycti's + set_state() does a plain json.dumps() with no datetime support. isinstance, not hasattr -- + same mypy-narrowing reason documented in connector A's connector.py. + """ + return value.isoformat() if isinstance(value, datetime.datetime) else value diff --git a/open-cti/connectors/vs-port-inventory/src/db.py b/open-cti/connectors/vs-port-inventory/src/db.py new file mode 100644 index 000000000..14fa27bee --- /dev/null +++ b/open-cti/connectors/vs-port-inventory/src/db.py @@ -0,0 +1,196 @@ +"""mini_data_lake access for the VS Port/Service Inventory connector. + +Same two-implementation shape as connectors A/D's db.py (real Postgres + IS_LOCAL fixtures, see +OpenCTI-connector.md §9a) -- kept structurally identical rather than reinvented (§10i). + +Table/column names taken from backend/src/xfd_django/xfd_mini_dl/models.py -- `LatestPortScan` +(L3116) has `db_table = "latest_port_scan"` and, unlike `VulnScan` (Connector B), *does* have +real indexes: a unique constraint plus explicit indexes on +`(organization, ip, port, protocol)`, `(ip, current, time_scanned)`, `time_scanned`, and `state` +-- confirmed by reading the model's own `Meta.indexes`, not assumed. + +**Revised design (2026-08-31): windowed/watermark polling, not a full-table poll every run.** +The original version of this file polled the entire in-scope table on every run, specifically +because `mark_stale_latest_port_scans()` +(`backend/src/xfd_django/xfd_api/tasks/utils/vs_port_scans.py:771`) flips `current` to `False` +via a plain `UPDATE ... SET current = FALSE WHERE time_scanned < cutoff`, never touching +`time_scanned` -- confirmed by reading that function directly. A plain `WHERE time_scanned > +watermark` poll can never see that transition, on the first run or any later one, since nothing +ever bumps the timestamp when it happens. + +Full-table-every-run turned out to be too expensive against the real row count, though -- so the +fix isn't a bigger `max_rows_per_run`, it's not needing to observe that transition via a query at +all. `connector.py` now computes staleness itself, locally, from `time_scanned` and a known +cutoff -- the exact same rule `mark_stale_latest_port_scans()` applies, just evaluated in our own +process instead of re-derived from a fresh row every time. That turns this back into an ordinary +watermark poll: + +- `since_last_seen`/`include_current` here are connector A's `since_last_seen`/ + `include_stale_open` pattern, applied to this table: the bootstrap poll (no watermark yet) + also pulls in every row the source still marks `current = TRUE`, regardless of how old + `time_scanned` is, so a currently-open port scanned long ago isn't permanently missed the same + way connector A's stale-open tickets were. Bounded by `max_rows_per_run` + `ORDER BY + time_scanned` exactly like connector A's bootstrap -- a "monumental" first run still safely + paginates across however many polls it takes, oldest-relevant-first, instead of ever pulling + everything in one shot. +- Every poll after that is a plain `time_scanned > watermark` query -- new ports and anything + actually rescanned. Nothing here ever needs to re-derive "did this go stale," because + connector.py already tracks that itself from data it already has. +""" + +# Standard Python Libraries +from dataclasses import dataclass, field +import json +import logging +import os +from typing import Dict, List, Optional + +# Third-Party Libraries +import boto3 +import psycopg2 +import psycopg2.extras + +from .config import Config + +LOGGER = logging.getLogger("vs_port_inventory.db") + + +@dataclass +class PortInventoryData: + """Everything one connector run needs, regardless of where it came from.""" + + port_scans: List[Dict] = field(default_factory=list) + + +def _generate_iam_auth_token(config: Config) -> str: + """Generate a short-lived RDS IAM auth token. Same pattern as connectors A/D -- see their db.py.""" + client = boto3.client("rds", region_name=config.aws_region) + return client.generate_db_auth_token( + DBHostname=config.db_host, + Port=config.db_port, + DBUsername=config.db_user, + ) + + +def get_connection(config: Config): + """Open a psycopg2 connection to mini_data_lake, IAM-auth or password per config.""" + password = ( + _generate_iam_auth_token(config) + if config.db_use_iam_auth + else config.db_password + ) + return psycopg2.connect( + host=config.db_host, + port=config.db_port, + dbname=config.db_name, + user=config.db_user, + password=password, + sslmode="require", # IAM auth requires TLS regardless of rds.force_ssl -- §2 + cursor_factory=psycopg2.extras.RealDictCursor, + connect_timeout=10, + ) + + +# One complete literal query, matching connectors A/B/D's convention -- no f-string/`.format()` +# assembly of query text (Connector B's db.py hit a real bandit B608 finding over exactly that +# pattern; avoided here from the start instead of fixed after the fact). +# +# Explicit column projection (§10d) -- only what mapping.py actually maps to STIX. `current` is +# still selected -- used only to decide what to *fetch* during the bootstrap poll (see module +# docstring), never trusted directly for the state connector.py actually records; that's derived +# locally from `time_scanned` + the configured cutoff instead, uniformly for every row regardless +# of how it was matched. +# +# (%(acronyms)s IS NULL OR ...) guard built in from the start (§10i) -- same acronym = ANY(NULL) +# lesson connector A's db.py already applies. +# +# The since/include_current condition is deliberately *not* "since IS NULL OR ..." the way +# connector A's is -- verified directly against a real postgres:17 container (§9b Loop 4) that +# copying that shape here was a real bug: on a bootstrap poll (since IS NULL), an unconditional +# "IS NULL" branch makes the whole OR true regardless of `current`, silently pulling in +# already-stale rows a bootstrap poll should never touch. `(since IS NOT NULL AND time_scanned > +# since) OR (include_current AND current IS TRUE)` means the first branch only ever fires once a +# real watermark exists, so a bootstrap poll (since IS NULL) is governed by `include_current` +# alone -- confirmed against the same container that this correctly excludes a stale row with +# current=false, and excludes everything once include_current=False in steady state. +# +# organization is a direct FK here (unlike VulnScan's denormalized `owner` string) -- joined +# server-side and scoped by acronym (varchar), the same structural choice connector A's db.py +# made to sidestep ever needing a `uuid = ANY(list)` cast for organization_id. +_QUERY = """ + SELECT lps.id, lps.ip_string, lps.port, lps.protocol, lps.state, lps.time_scanned, + lps.service_name, lps.service_cpe, lps.service_product, lps.service_version, + lps.source, lps.nmi_service_group, lps.risky_service_group, lps.current, + lps.port_scan_id, o.acronym AS organization_acronym, o.name AS organization_name + FROM latest_port_scan lps + JOIN organization o ON o.id = lps.organization_id + WHERE (%(acronyms)s IS NULL OR o.acronym = ANY(%(acronyms)s)) + AND ( + (%(since)s IS NOT NULL AND lps.time_scanned > %(since)s) + OR (%(include_current)s AND lps.current IS TRUE) + ) + ORDER BY lps.time_scanned + LIMIT %(limit)s +""" + + +class VsPortInventoryRepository: + """Fetches one incremental window of LatestPortScan, capped per config (§9c).""" + + def __init__(self, config: Config): + """Hold the config; connections are opened per-call, not cached (§10d).""" + self.config = config + + def fetch( + self, since_last_seen: Optional[str], include_current: bool = False + ) -> PortInventoryData: + """Fetch one run's worth of data, from fixtures (IS_LOCAL) or mini_data_lake. + + `include_current` is the bootstrap-only safety net (see connector.py's `run()` for where + it's computed) -- see the module docstring for why it exists. + """ + if self.config.is_local: + return self._fetch_local() + return self._fetch_live(since_last_seen, include_current) + + # ------------------------------------------------------------------ + # Live path + # ------------------------------------------------------------------ + + def _fetch_live( + self, since_last_seen: Optional[str], include_current: bool + ) -> PortInventoryData: + conn = get_connection(self.config) + try: + with conn.cursor() as cur: + cur.execute( + _QUERY, + { + "acronyms": self.config.org_acronym_allowlist or None, + "since": since_last_seen, + "include_current": include_current, + "limit": self.config.max_rows_per_run, + }, + ) + return PortInventoryData( + port_scans=[dict(row) for row in cur.fetchall()] + ) + finally: + conn.close() + + # ------------------------------------------------------------------ + # IS_LOCAL fixture path (§9a) -- same shape as the live path, zero DB/network. + # ------------------------------------------------------------------ + + def _fetch_local(self) -> PortInventoryData: + directory = self.config.local_fixture_dir + LOGGER.info("IS_LOCAL=true -- loading fixtures from %s", directory) + return PortInventoryData( + port_scans=self._load_json(directory, "port_scans.json") + ) + + @staticmethod + def _load_json(directory: str, filename: str): + path = os.path.join(directory, filename) + with open(path, encoding="utf-8") as f: + return json.load(f) diff --git a/open-cti/connectors/vs-port-inventory/src/main.py b/open-cti/connectors/vs-port-inventory/src/main.py new file mode 100644 index 000000000..61dd0c4cf --- /dev/null +++ b/open-cti/connectors/vs-port-inventory/src/main.py @@ -0,0 +1,27 @@ +"""Entry point. See OpenCTI-connector.md §7c for design, README.md for how to run this locally.""" + +# Standard Python Libraries +import sys +import time + +from .config import ConfigError +from .connector import VsPortInventoryConnector + +if __name__ == "__main__": + try: + connector = VsPortInventoryConnector() + except ConfigError as e: + # Fail closed and loud (§10c/§9c) -- not a stack trace to dig through. + print(f"[vs-port-inventory] refusing to start: {e}", file=sys.stderr) + sys.exit(1) + + try: + connector.helper.schedule_iso( + message_callback=connector.run, + duration_period=connector.config.duration_period, + ) + except KeyboardInterrupt: + connector.helper.connector_logger.info("Stopping VS Port Inventory connector") + connector.helper.stop() + time.sleep(1) + sys.exit(0) diff --git a/open-cti/connectors/vs-port-inventory/src/mapping.py b/open-cti/connectors/vs-port-inventory/src/mapping.py new file mode 100644 index 000000000..ee111b2e8 --- /dev/null +++ b/open-cti/connectors/vs-port-inventory/src/mapping.py @@ -0,0 +1,399 @@ +"""Pure row -> STIX mapping functions for the VS Port/Service Inventory connector. + +Deliberately dependency-free beyond stix2/pycti and the standard library -- no DB, no OpenCTI +API, no network. This is "Loop 1" from OpenCTI-connector.md §9b. See tests/test_mapping.py. + +STIX mapping decisions here are documented in OpenCTI-connector.md §7c. The idempotency pattern +(§10a) applies to the lifecycle relationship the same way it did for connectors A/D -- but this +connector also had to verify, and steer clear of, a *third* instance of the same underlying bug +class: putting `start`/`end` directly on the `NetworkTraffic` SCO itself. Verified directly +against the installed stix2 library that `start`/`end` are among `NetworkTraffic`'s own +ID-contributing properties -- two NetworkTraffic objects built with the same +src_ref/src_port/protocols but *different* `start` values get *different* ids. Putting the +lifecycle there instead of on a separate, pinned relationship would have fragmented one port's +history into a new SCO every time it changed, not updated one object in place -- so the +NetworkTraffic SCO here is built from stable fields only, and the lifecycle lives entirely on +`build_lifecycle_relationship()`/`build_lifecycle_relationship_from_parts()`, pinned in connector +state exactly like connectors A/D already do. + +Revised (2026-08-31, see db.py's module docstring): connector.py now computes port staleness +locally from `time_scanned` + a known cutoff, rather than re-polling for it, which means it +sometimes needs to build an updated lifecycle relationship with no fresh DB row on hand -- +`build_lifecycle_relationship_from_parts()` exists for exactly that case. + +Revised (2026-09-02): the Network-Traffic SCO carries a service/state summary readable straight +off the observable, without following the relationship to the org for it -- but **not** via +`x_opencti_service`/`x_opencti_open` custom properties, as an earlier version of this file tried. +Verified directly against this platform's real GraphQL schema (`__type(name: "NetworkTraffic")`) +that those two keys aren't recognized fields at all -- OpenCTI's Network-Traffic schema only +knows a fixed set of `x_opencti_*` keys (`x_opencti_stix_ids`, `x_opencti_modified_at`, +`x_opencti_inferences`, `x_opencti_score`, `x_opencti_description`), so anything else gets +silently dropped on ingest. Confirmed with a real, throwaway bundle sent through a live instance +and read back via GraphQL: `x_opencti_description` survives (already relied on below), an +unrecognized `x_opencti_*` key does not, and a bare (non-`x_opencti_`-prefixed) `labels` custom +property *does* land as `objectLabel` -- so that's what carries this connector's own +service/state summary now, the same mechanism `lifecycle_labels()` already uses on the +relationship, applied here to the SCO itself. + +The state label is **not** this connector's locally-computed recency flag (`current`/ +`is_within_cutoff`) -- an earlier version of this code conflated the two, which was wrong. It's +`LatestPortScan.state == "open"` (`is_port_state_open()` below), the scanner's own confirmed +finding for this exact port on its most recent scan (`models.py`'s help text: "State of the port, +as reported by the scanner; see nmap states" -- and the platform's own `PortScanSummary` rollup +treats it the same way, filtering `WHERE ps.state = 'open'`). `state` gets overwritten in place on +every rescan (`insert_port_scans_sql()`'s `ON CONFLICT ... DO UPDATE`), so a port that was open +and is later rescanned closed genuinely flips this label -- independent of whether that rescan +happened to also be within the freshness cutoff. Consequence: the aging sweep (connector.py, no +fresh row on hand) must never touch these labels -- going stale (unobserved for 14+ days) is not +evidence a port closed, just that its last-known state is unconfirmed by anything more recent. +""" + +# Standard Python Libraries +import datetime +from ipaddress import ip_address +from typing import Dict, List, Optional + +# Third-Party Libraries +from dateutil import parser as dateutil_parser +from pycti import Identity as OpenCTIIdentity +from pycti import StixCoreRelationship +import stix2 + +VS_EXTERNAL_SOURCE = "VS" + + +def build_author(name: str) -> stix2.Identity: + """Build the single system Identity every VS-sourced object should cite as createdBy (§10c). + + Identical construction to connectors A/B/D's build_author() -- all four connectors' + Identity(class="system") objects resolve to the one shared, content-derived author. + """ + return stix2.Identity( + id=OpenCTIIdentity.generate_id(name, "system"), + name=name, + identity_class="system", + ) + + +def resolve_org_identity_id(organization_name: str) -> str: + """Resolve an organization's existing Identity id -- look up, don't blindly create (§7c). + + Identical reasoning to connector A's mapping.resolve_org_identity_id() -- connector D owns + creating Identity(class="organization"); this recomputes the same content-derived id rather + than a live API lookup per row, given this connector's own full-poll volume (§9c). + """ + return OpenCTIIdentity.generate_id(organization_name, "organization") + + +def build_org_owns_ip( + org_id: str, + ip_observable_id: str, + author_id: str, + marking_id: str, + existing_id: Optional[str] = None, +) -> stix2.Relationship: + """Org -> IP observable, built identically to connector A's equivalent mapping function. + + Deliberately identical -- same relationship_type/source/target/no-time-component means the + same deterministic id, so this connector's and connector A's org->IP edges for the same + (org, ip) pair collide into the *same* OpenCTI relationship rather than each connector + creating its own separate copy of "this org is associated with this host." + """ + rel_id = _relationship_id( + "related-to", org_id, ip_observable_id, None, None, existing_id + ) + return stix2.Relationship( + id=rel_id, + relationship_type="related-to", + source_ref=org_id, + target_ref=ip_observable_id, + created_by_ref=author_id, + object_marking_refs=[marking_id], + ) + + +def map_ip_observable(ip_string: str): + """IP string -> IPv4Address/IPv6Address SCO. + + LatestPortScan.ip_string is a single scanned host's address, same shape as connector A's + Ticket.ip_string -- ip_address(), not ip_network(). Raises ValueError on anything malformed; + left to propagate so connector.py's per-row try/except (§10i) can skip just that one row. + """ + parsed = ip_address(ip_string) + if parsed.version == 4: + return stix2.IPv4Address(value=ip_string) + return stix2.IPv6Address(value=ip_string) + + +def normalize_timestamp(value): + """Normalize a DateTimeField value, or a value read back from connector state, to a datetime. + + Same helper connector A's mapping.py already proved out, reused verbatim here rather than + reinvented (§10i). Two distinct sources feed this: `LatestPortScan.time_scanned` (a real + psycopg2 datetime on a live run, an ISO string in IS_LOCAL fixtures) *and* values read back + out of connector state (always JSON strings, since state round-trips through + pycti's set_state()/get_state()). Parsing every string through dateutil and always returning + a real datetime object closes the `+00:00`-offset-string gap connector A found the hard way, + for both sources at once rather than trusting either one to already be in a stix2-safe shape. + """ + if value is None: + return None + if isinstance(value, datetime.datetime): + return value + if isinstance(value, datetime.date): + return datetime.datetime( + value.year, value.month, value.day, tzinfo=datetime.timezone.utc + ) + if isinstance(value, str): + parsed = dateutil_parser.isoparse(value) + if parsed.tzinfo is None: + parsed = parsed.replace(tzinfo=datetime.timezone.utc) + return parsed + return value + + +def _normalize_protocol(protocol: Optional[str]) -> str: + """Lower-case the scanned protocol for STIX's `protocols` list. + + LatestPortScan.protocol is documented as "'tcp' or 'udp'" (models.py) -- deliberately not + validated against a fixed enum beyond that, since stix2's own StringProperty doesn't + constrain `protocols` to a fixed vocabulary either. Raises on a missing value rather than + guessing "tcp" -- a port scan with no protocol at all is a malformed row, not a default case. + """ + if not protocol: + raise ValueError("LatestPortScan row has no protocol") + return protocol.strip().lower() + + +def is_port_state_open(state: Optional[str]) -> bool: + """Reduce LatestPortScan.state (nmap-style: open/closed/filtered/...) to a plain boolean. + + Anything other than a literal "open" (case/whitespace-insensitive) -- closed, filtered, + open|filtered, missing entirely -- reads as "not confirmed open". Feeds `network_traffic_labels()` + below, not `current`/recency -- see module docstring for why those are different things. + """ + return (state or "").strip().lower() == "open" + + +def _label_safe(value: str) -> str: + """Normalize a raw value into a plain OpenCTI label token: lower-case, hyphenated.""" + return "-".join(value.strip().lower().split()) + + +def network_traffic_labels(row: Dict) -> List[str]: + """Compute this row's service/state summary as OpenCTI Labels on the SCO itself. + + Not `x_opencti_service`/`x_opencti_open` -- verified those aren't real fields on this + platform's Network-Traffic schema and get silently dropped (see module docstring). `labels` + (bare, not `x_opencti_`-prefixed) is the confirmed-working mechanism -- verified end-to-end + against a real instance that it lands as `objectLabel`. `vs-state-` mirrors the exact + label `lifecycle_labels()` already puts on the relationship, for the same row; `vs-open` is + an additional presence-only tag for a quick visual filter, since "is this thing open" is + exactly the question this connector exists to answer. + """ + labels: List[str] = [] + state = row.get("state") + if state: + labels.append(f"vs-state-{_label_safe(state)}") + if is_port_state_open(state): + labels.append("vs-open") + service_name = row.get("service_name") + if service_name: + labels.append(f"vs-service-{_label_safe(service_name)}") + return labels + + +def build_network_traffic(row: Dict, ip_observable_id: str) -> stix2.NetworkTraffic: + """Build the Network-Traffic SCO for one (ip, port, protocol) row. + + Deliberately built from stable fields only -- see module docstring for why `start`/`end`/ + `is_active` never go here. `service_name` is often not a valid STIX protocol string (§7c), so + it's carried as a custom property instead of crammed into `protocols`. + """ + protocol = _normalize_protocol(row.get("protocol")) + port = row.get("port") + if not port: + raise ValueError("LatestPortScan row has no port") + # Verified against the installed stix2 library that custom_properties do not feed + # NetworkTraffic's own id computation (unlike start/end) -- safe to vary between polls + # without fragmenting the SCO the way start/end would have. This includes `labels` (verified + # separately, end-to-end against a real instance, that it's a real accepted field for this + # SCO type and not itself ID-contributing). + custom_properties: Dict[str, object] = {} + labels = network_traffic_labels(row) + if labels: + custom_properties["labels"] = labels + service_name = row.get("service_name") + if service_name: + custom_properties["x_opencti_description"] = service_name + return stix2.NetworkTraffic( + src_ref=ip_observable_id, + src_port=int(port), + protocols=[protocol], + custom_properties=custom_properties, + ) + + +def build_software(row: Dict) -> Optional[stix2.Software]: + """Build a Software SCO when the row actually has product/version/cpe detail. + + Content-derived id is the *correct* behavior here, unlike NetworkTraffic's lifecycle -- a + different product/version genuinely is a different Software entity, not an update to the + same one, so no pinning is needed (same reasoning as connector D's IPv4-Addr/IPv6-Addr reuse: + SCOs with real ID Contributing Properties don't need §10a's pinning pattern at all). + """ + if not ( + row.get("service_product") + or row.get("service_version") + or row.get("service_cpe") + ): + return None + return stix2.Software( + name=row.get("service_product") or row.get("service_name") or "unknown", + version=row.get("service_version"), + cpe=row.get("service_cpe"), + ) + + +def _relationship_id( + relationship_type: str, + source_ref: str, + target_ref: str, + start_time, + stop_time, + existing_id: Optional[str], +) -> str: + """Apply the §10a idempotency pattern -- identical reasoning to connectors A/D's mapping.py.""" + if existing_id: + return existing_id + return StixCoreRelationship.generate_id( + relationship_type, source_ref, target_ref, start_time, stop_time + ) + + +def lifecycle_labels(row: Dict) -> List[str]: + """Compute this row's labels once, so connector state can cache and reuse them. + + Needed for the state-only relationship rebuild in build_lifecycle_relationship_from_parts() + when there's no fresh row on hand. + """ + labels: List[str] = [] + if row.get("source"): + labels.append(f"vs-source-{row['source']}") + if row.get("risky_service_group"): + labels.append(f"vs-risky-service-{row['risky_service_group']}") + if row.get("nmi_service_group"): + labels.append(f"vs-nmi-service-{row['nmi_service_group']}") + if row.get("state"): + labels.append(f"vs-state-{row['state']}") + return labels + + +def lifecycle_external_id(row: Dict) -> str: + """Return the bookkeeping id carried on the lifecycle relationship's External Reference.""" + return row.get("port_scan_id") or row["id"] + + +def build_lifecycle_relationship_from_parts( + org_id: str, + network_traffic_id: str, + author_id: str, + marking_id: str, + start_time, + stop_time, + labels: List[str], + external_id: str, + existing_id: Optional[str] = None, +) -> stix2.Relationship: + """Build the org -> Network-Traffic lifecycle relationship from already-resolved parts. + + The lower-level builder both `build_lifecycle_relationship()` (fresh row on hand) and + connector.py's locally-computed staleness sweep (no fresh row -- state only, see db.py's + module docstring for why that's a real, deliberate case now) share. `start_time`/`stop_time` + are passed in already-resolved, not computed here -- §10a's idempotency depends on connector + state (what was recorded last run), which this dependency-free module deliberately has no + access to. + """ + rel_id = _relationship_id( + "related-to", org_id, network_traffic_id, start_time, stop_time, existing_id + ) + return stix2.Relationship( + id=rel_id, + relationship_type="related-to", + source_ref=org_id, + target_ref=network_traffic_id, + start_time=start_time, + stop_time=stop_time, + labels=labels or None, + created_by_ref=author_id, + object_marking_refs=[marking_id], + external_references=[ + stix2.ExternalReference( + source_name=VS_EXTERNAL_SOURCE, external_id=external_id + ) + ], + ) + + +def build_lifecycle_relationship( + row: Dict, + org_id: str, + network_traffic_id: str, + author_id: str, + marking_id: str, + start_time, + stop_time, + existing_id: Optional[str] = None, +) -> stix2.Relationship: + """Build the org -> Network-Traffic relationship carrying this port's open/stale lifecycle. + + Thin wrapper around build_lifecycle_relationship_from_parts() for the common case: a fresh + row is on hand, so labels/external_id are computed from it directly. + """ + return build_lifecycle_relationship_from_parts( + org_id, + network_traffic_id, + author_id, + marking_id, + start_time, + stop_time, + labels=lifecycle_labels(row), + external_id=lifecycle_external_id(row), + existing_id=existing_id, + ) + + +def build_software_relationship( + network_traffic_id: str, + software_id: str, + author_id: str, + marking_id: str, + existing_id: Optional[str] = None, +) -> stix2.Relationship: + """Network-Traffic -> Software, with no lifecycle of its own. + + Presence/absence tracks with Software's own content-derived id instead of a separate + start/stop window. + """ + rel_id = _relationship_id( + "related-to", network_traffic_id, software_id, None, None, existing_id + ) + return stix2.Relationship( + id=rel_id, + relationship_type="related-to", + source_ref=network_traffic_id, + target_ref=software_id, + created_by_ref=author_id, + object_marking_refs=[marking_id], + ) + + +def dedupe_bundle_objects(objects: List) -> List: + """Collapse a list of STIX objects to one per id, keeping the last occurrence of each. + + Same rationale as connectors A/B/D -- many rows share the same IP/org across one poll. + """ + seen = {} + for obj in objects: + seen[obj["id"]] = obj + return list(seen.values()) diff --git a/open-cti/connectors/vs-port-inventory/tests/__init__.py b/open-cti/connectors/vs-port-inventory/tests/__init__.py new file mode 100644 index 000000000..398b31136 --- /dev/null +++ b/open-cti/connectors/vs-port-inventory/tests/__init__.py @@ -0,0 +1 @@ +"""Tests for the VS Port/Service Inventory connector -- see README.md for the two loops.""" diff --git a/open-cti/connectors/vs-port-inventory/tests/fixtures/port_scans.json b/open-cti/connectors/vs-port-inventory/tests/fixtures/port_scans.json new file mode 100644 index 000000000..0221ebac6 --- /dev/null +++ b/open-cti/connectors/vs-port-inventory/tests/fixtures/port_scans.json @@ -0,0 +1,59 @@ +[ + { + "current": true, + "id": "11111111-1111-1111-1111-111111111111", + "ip_string": "198.51.100.5", + "nmi_service_group": "web", + "organization_acronym": "TESTORG1", + "organization_name": "Test Organization One", + "port": 443, + "port_scan_id": "ps-0001", + "protocol": "tcp", + "risky_service_group": null, + "service_cpe": "cpe:/a:apache:http_server:2.4.41", + "service_name": "https", + "service_product": "Apache httpd", + "service_version": "2.4.41", + "source": "nmap", + "state": "open", + "time_scanned": "2026-08-20T12:00:00+00:00" + }, + { + "current": false, + "id": "22222222-2222-2222-2222-222222222222", + "ip_string": "198.51.100.5", + "nmi_service_group": null, + "organization_acronym": "TESTORG1", + "organization_name": "Test Organization One", + "port": 23, + "port_scan_id": "ps-0002", + "protocol": "tcp", + "risky_service_group": "telnet", + "service_cpe": null, + "service_name": "telnet", + "service_product": null, + "service_version": null, + "source": "nmap", + "state": "open", + "time_scanned": "2026-07-01T00:00:00+00:00" + }, + { + "current": true, + "id": "33333333-3333-3333-3333-333333333333", + "ip_string": "2001:db8::5", + "nmi_service_group": null, + "organization_acronym": "TESTORG2-CHILD", + "organization_name": "Test Organization Two (Child)", + "port": 22, + "port_scan_id": "ps-0003", + "protocol": "tcp", + "risky_service_group": null, + "service_cpe": null, + "service_name": "ssh", + "service_product": "OpenSSH", + "service_version": "8.9", + "source": "nmap", + "state": "open", + "time_scanned": "2026-08-21T00:00:00+00:00" + } +] diff --git a/open-cti/connectors/vs-port-inventory/tests/test_connector_dry_run.py b/open-cti/connectors/vs-port-inventory/tests/test_connector_dry_run.py new file mode 100644 index 000000000..ad86fa1e9 --- /dev/null +++ b/open-cti/connectors/vs-port-inventory/tests/test_connector_dry_run.py @@ -0,0 +1,388 @@ +"""Loop 2 (OpenCTI-connector.md §9b): the real poll->diff->bundle pipeline against fixtures. + +Uses a stub in place of OpenCTIConnectorHelper so no network/queue is touched at all. Proves the +redesigned watermark + local-aging-sweep loop (2026-08-31, see connector.py's module docstring): +a bootstrap poll pulls in currently-active rows regardless of age, a second identical poll sends +nothing, a port aging past the cutoff *without ever being refetched* still gets closed by the +aging sweep alone, and a port reopening clears stop_time while keeping its original start_time. + +Timestamps here are computed relative to test-execution time (`_now()`), not hardcoded calendar +dates -- this connector's own logic depends on elapsed time relative to `time_scanned`, so a +fixed date would silently drift out of the cutoff window as real time passes and make these +tests flaky months later, exactly the kind of live-vs-fixture gap §10i keeps calling out. +""" + +# Standard Python Libraries +import datetime +import json +import os +import sys + +sys.path.insert(0, os.path.join(os.path.dirname(__file__), "..")) + +# Third-Party Libraries +from pycti.utils.opencti_logger import logger as pycti_logger + +# First-Party +from src.config import Config # noqa: E402 +from src.connector import VsPortInventoryConnector # noqa: E402 +from src.db import PortInventoryData # noqa: E402 + + +class StubHelper: + """Just enough of OpenCTIConnectorHelper's surface for run()/_process() to run against. + + connector_logger deliberately uses pycti's *real* AppLogger, not a plain stdlib + logging.getLogger() -- same reasoning as connectors A/B/D's stubs (§10i). + """ + + def __init__(self): + """Start with empty capture state -- nothing sent or stored yet.""" + self.connector_logger = pycti_logger(level=20, json_logging=False)( + "stub_helper" + ) + self.sent_bundles = [] + self._state = {} + + def get_state(self): + """Return the current stored state, mirroring the real helper's equivalent method.""" + return self._state + + def set_state(self, state): + """Persist state, mirroring the real helper's equivalent method.""" + self._state = state + + @staticmethod + def stix2_create_bundle(items): + """Delegate to pycti's real bundle serialization rather than reimplementing it.""" + # Third-Party Libraries + from pycti import OpenCTIConnectorHelper + + return OpenCTIConnectorHelper.stix2_create_bundle(items) + + def send_stix2_bundle(self, bundle, **kwargs): + """Capture the bundle instead of sending it -- this *is* the dry-run boundary.""" + self.sent_bundles.append(bundle) + return [bundle] + + +def build_test_connector(cutoff_days=14): + """Build a VsPortInventoryConnector wired to fixtures and a StubHelper.""" + config = Config( + raw={ + "vs_port_inventory": { + "org_acronym_allowlist": "TESTORG1,TESTORG2-CHILD", + "is_local": True, + "local_fixture_dir": os.path.join( + os.path.dirname(__file__), "fixtures" + ), + "tlp_marking": "TLP:AMBER", + "latest_port_scan_cutoff_days": str(cutoff_days), + } + } + ) + return VsPortInventoryConnector(config=config, helper=StubHelper()) + + +def _now(): + return datetime.datetime.now(datetime.timezone.utc) + + +def _row(**overrides): + row = { + "id": "ps-1", + "port_scan_id": "ps-1", + "ip_string": "198.51.100.5", + "port": 443, + "protocol": "tcp", + "state": "open", + "time_scanned": _now().isoformat(), + "service_name": "https", + "service_cpe": None, + "service_product": "Apache httpd", + "service_version": "2.4.41", + "source": "nmap", + "nmi_service_group": "web", + "risky_service_group": None, + "current": True, + "organization_acronym": "TESTORG1", + "organization_name": "Test Organization One", + } + row.update(overrides) + return row + + +def test_first_run_against_fixtures_produces_expected_object_shape(): + """A single run against the fixtures should produce the object types §7c's design expects.""" + connector = build_test_connector() + data = connector.repository.fetch(since_last_seen=None, include_current=True) + state = {} + + connector._process(data, state) + + assert len(connector.helper.sent_bundles) == 1 + bundle = json.loads(connector.helper.sent_bundles[0]) + types = [obj["type"] for obj in bundle["objects"]] + assert ( + types.count("identity") == 1 + ) # author only -- org Identity is looked up, not created + assert types.count("marking-definition") == 1 + assert types.count("ipv4-addr") == 1 # 198.51.100.5, shared by two fixture rows + assert types.count("ipv6-addr") == 1 # 2001:db8::5 + assert ( + types.count("network-traffic") == 3 + ) # 443/tcp, 23/tcp, 22/tcp -- all distinct + assert ( + types.count("software") == 2 + ) # Apache httpd, OpenSSH -- the telnet row has none + # 3 lifecycle relationships + 2 deduped org->ip relationships + 2 network-traffic->software + assert types.count("relationship") == 7 + # The "vs-open" label must mirror the fixtures' own `state` field, not recency -- all three + # fixture rows report state="open" (port 23's telnet row is old enough to be locally + # non-"current", fetched via include_current, but the scanner still last confirmed it open). + nt_by_port = { + obj["src_port"]: "vs-open" in obj.get("labels", []) + for obj in bundle["objects"] + if obj["type"] == "network-traffic" + } + assert nt_by_port == {443: True, 23: True, 22: True} + + +def test_a_pure_state_flip_still_triggers_a_resend(): + """A rescan that only changes state=open->closed (nothing else) must still be sent. + + port_state_open is in _process_row()'s changed-fields check specifically so this doesn't + silently get swallowed by the "nothing changed" queue-discipline shortcut. + """ + connector = build_test_connector() + state = {} + connector._process(PortInventoryData(port_scans=[_row(state="open")]), state) + sent_before = len(connector.helper.sent_bundles) + + connector._process(PortInventoryData(port_scans=[_row(state="closed")]), state) + + assert len(connector.helper.sent_bundles) == sent_before + 1 + key = connector._key(_row()) + assert state["port_scan_state"][key]["port_state_open"] is False + bundle = json.loads(connector.helper.sent_bundles[-1]) + nt_id = state["port_scan_state"][key]["network_traffic_id"] + nt_obj = next(obj for obj in bundle["objects"] if obj.get("id") == nt_id) + assert "vs-open" not in nt_obj.get("labels", []) + + +def test_second_identical_run_sends_no_bundle(): + """§9 queue discipline: nothing changed means nothing worth putting on the queue.""" + connector = build_test_connector() + data = PortInventoryData(port_scans=[_row()]) + state = {} + connector._process(data, state) + + connector._process(data, state) + + assert ( + len(connector.helper.sent_bundles) == 1 + ) # only the first run actually sent anything + + +def test_is_current_is_computed_locally_not_trusted_from_the_row(): + """The row's own `current` field must be ignored -- only time_scanned + cutoff decide. + + A row claiming current=False but scanned moments ago must be treated as open; a row + claiming current=True but scanned long before the cutoff must be treated as stale. + """ + connector = build_test_connector(cutoff_days=14) + stale_but_claims_current = _row( + id="ps-old", + current=True, + time_scanned=(_now() - datetime.timedelta(days=30)).isoformat(), + ) + fresh_but_claims_closed = _row( + id="ps-new", + port=22, + current=False, + time_scanned=_now().isoformat(), + ) + state = {} + + connector._process(PortInventoryData(port_scans=[stale_but_claims_current]), state) + connector._process(PortInventoryData(port_scans=[fresh_but_claims_closed]), state) + + old_key = connector._key(stale_but_claims_current) + new_key = connector._key(fresh_but_claims_closed) + assert state["port_scan_state"][old_key]["current"] is False + assert state["port_scan_state"][old_key]["stop_time"] is not None + assert state["port_scan_state"][new_key]["current"] is True + assert state["port_scan_state"][new_key]["stop_time"] is None + + +def test_watermark_is_the_max_time_scanned_across_fetched_rows(): + """The new watermark must be the latest time_scanned actually seen this run.""" + connector = build_test_connector() + earlier = _row( + id="ps-1", + port=443, + time_scanned=(_now() - datetime.timedelta(days=1)).isoformat(), + ) + later = _row(id="ps-2", port=22, time_scanned=_now().isoformat()) + + watermark = connector._process(PortInventoryData(port_scans=[earlier, later]), {}) + + assert watermark == connector._compute_watermark([earlier, later]) + assert watermark == later["time_scanned"] + + +def test_watermark_is_none_when_nothing_was_fetched(): + """No fetched rows means nothing to advance the watermark past.""" + connector = build_test_connector() + assert connector._process(PortInventoryData(port_scans=[]), {}) is None + + +def test_aging_sweep_closes_a_port_that_ages_past_cutoff_without_ever_being_refetched(): + """The core point of the redesign: no fresh row is needed to detect staleness. + + §7c's original gotcha (mark_stale_latest_port_scans() never bumps time_scanned) is solved by + computing the same rule locally instead of re-polling for it -- this proves that path. + """ + connector = build_test_connector(cutoff_days=14) + row = _row() + state = {} + connector._process(PortInventoryData(port_scans=[row]), state) + key = connector._key(row) + assert state["port_scan_state"][key]["current"] is True + assert state["port_scan_state"][key]["stop_time"] is None + open_id = state["port_scan_state"][key]["relationship_id"] + + # Simulate real elapsed time without a fresh row -- push the stored time_scanned back past + # the cutoff directly in state, the same way waiting 15 real days would. + long_ago = _now() - datetime.timedelta(days=15) + state["port_scan_state"][key]["time_scanned"] = long_ago.isoformat() + + watermark = connector._process(PortInventoryData(port_scans=[]), state) + + assert ( + watermark is None + ) # nothing was fetched -- the closure came from the aging sweep alone + assert state["port_scan_state"][key]["current"] is False + assert state["port_scan_state"][key]["stop_time"] is not None + assert ( + state["port_scan_state"][key]["relationship_id"] == open_id + ) # pinned, as designed + # Scanner-confirmed state is untouched by going stale -- see mapping.py's module docstring. + assert state["port_scan_state"][key]["port_state_open"] is True + bundle = json.loads(connector.helper.sent_bundles[-1]) + rel_obj = next(obj for obj in bundle["objects"] if obj.get("id") == open_id) + assert "stop_time" in rel_obj + # The SCO itself must NOT be resent by the aging sweep alone -- its vs-state-*/vs-open + # labels track the scanner's confirmed state, not recency, so there's nothing new to say + # about it here. + nt_id = state["port_scan_state"][key]["network_traffic_id"] + assert not any(obj.get("id") == nt_id for obj in bundle["objects"]) + + +def test_aging_sweep_leaves_a_still_fresh_port_alone(): + """A port well within the cutoff window must not get touched by the aging sweep.""" + connector = build_test_connector(cutoff_days=14) + row = _row() + state = {} + connector._process(PortInventoryData(port_scans=[row]), state) + sent_before = len(connector.helper.sent_bundles) + + connector._process(PortInventoryData(port_scans=[]), state) + + key = connector._key(row) + assert state["port_scan_state"][key]["current"] is True + assert ( + len(connector.helper.sent_bundles) == sent_before + ) # nothing to send -- still current + + +def test_reopened_port_clears_stop_time_but_keeps_original_start_time(): + """A port that reopens after going stale must clear stop_time without resetting start_time.""" + connector = build_test_connector(cutoff_days=14) + row = _row() + state = {} + + connector._process(PortInventoryData(port_scans=[row]), state) + key = connector._key(row) + original_start = state["port_scan_state"][key]["start_time"] + + # Age it out locally, same as the dedicated aging-sweep test. + state["port_scan_state"][key]["time_scanned"] = ( + _now() - datetime.timedelta(days=15) + ).isoformat() + connector._process(PortInventoryData(port_scans=[]), state) + assert state["port_scan_state"][key]["stop_time"] is not None + + # Reopens: a fresh row for the same key, rescanned just now. + reopened_row = _row(time_scanned=_now().isoformat()) + connector._process(PortInventoryData(port_scans=[reopened_row]), state) + + assert state["port_scan_state"][key]["stop_time"] is None + assert ( + state["port_scan_state"][key]["start_time"] == original_start + ) # unchanged, not reset + + +def test_run_threads_since_and_include_current_like_connector_as_bootstrap_flag(): + """run() must pass include_current=True only on the first poll. + + Matches connector A's is_first_run/include_stale_open pattern applied to this table. + """ + connector = build_test_connector() + calls = [] + original_fetch = connector.repository.fetch + + def spying_fetch(since_last_seen, include_current=False): + calls.append((since_last_seen, include_current)) + return original_fetch(since_last_seen, include_current=include_current) + + connector.repository.fetch = spying_fetch + + connector.run() # no watermark yet -- bootstrap + connector.run() # watermark now set -- steady-state + + assert calls[0] == (None, True) + assert calls[1][1] is False + + +def test_state_persists_even_when_a_run_sends_nothing(): + """State must persist every successful poll, not just when something changed.""" + connector = build_test_connector() + + connector.run() + first_state_keys = set(connector.helper.get_state().get("port_scan_state", {})) + connector.run() + + assert ( + set(connector.helper.get_state().get("port_scan_state", {})) == first_state_keys + ) + assert connector.helper.get_state() != {} + + +def test_one_bad_row_does_not_abort_the_whole_run(): + """A malformed row must not sink every other row in the same poll. + + §10e/§10i row-isolation, built in from the start. + """ + connector = build_test_connector() + good_row = _row() + bad_row = _row( + id="bad-row", + port_scan_id="ps-bad", + ip_string="198.51.100.9", + protocol=None, # malformed -- mapping.build_network_traffic() must raise on this + ) + state = {} + + connector._process( + PortInventoryData(port_scans=[good_row, bad_row]), state + ) # must not raise + + bundle = json.loads(connector.helper.sent_bundles[-1]) + ips_sent = [ + obj["value"] + for obj in bundle["objects"] + if obj["type"] in ("ipv4-addr", "ipv6-addr") + ] + assert "198.51.100.9" not in ips_sent # the bad one was skipped + assert "198.51.100.5" in ips_sent # the good one still went out diff --git a/open-cti/connectors/vs-port-inventory/tests/test_mapping.py b/open-cti/connectors/vs-port-inventory/tests/test_mapping.py new file mode 100644 index 000000000..72c9eab04 --- /dev/null +++ b/open-cti/connectors/vs-port-inventory/tests/test_mapping.py @@ -0,0 +1,320 @@ +"""Loop 1 (OpenCTI-connector.md §9b): pure mapping unit tests, no DB/OpenCTI/queue at all.""" + +# Standard Python Libraries +import datetime +import os +import sys + +sys.path.insert(0, os.path.join(os.path.dirname(__file__), "..")) + +# Third-Party Libraries +from pycti import Identity as OpenCTIIdentity # noqa: E402 +from pycti import StixCoreRelationship # noqa: E402 +import pytest # noqa: E402 + +# First-Party +from src import mapping # noqa: E402 +import stix2 # noqa: E402 + +_AUTHOR_ID = OpenCTIIdentity.generate_id("Test Author", "system") +_MARKING_ID = stix2.TLP_AMBER.id +_ORG_ID = OpenCTIIdentity.generate_id("Test Organization One", "organization") +_IP_ID = mapping.map_ip_observable("198.51.100.5").id +_NETWORK_TRAFFIC_ID = stix2.NetworkTraffic( + src_ref=_IP_ID, src_port=443, protocols=["tcp"] +).id + + +def _row(**overrides): + row = { + "id": "ps-1", + "port_scan_id": "ps-1", + "ip_string": "198.51.100.5", + "port": 443, + "protocol": "tcp", + "state": "open", + "time_scanned": "2026-08-20T12:00:00+00:00", + "service_name": "https", + "service_cpe": None, + "service_product": None, + "service_version": None, + "source": "nmap", + "nmi_service_group": None, + "risky_service_group": None, + "current": True, + } + row.update(overrides) + return row + + +def test_resolve_org_identity_id_matches_connector_ds_own_identity_construction(): + """§7c: 'look up, don't blindly create' -- must land on connector D's own id.""" + assert mapping.resolve_org_identity_id("Test Organization One") == _ORG_ID + + +def test_map_ip_observable_v4_and_v6(): + """A host address maps to the matching IPv4Address/IPv6Address SCO type.""" + assert mapping.map_ip_observable("198.51.100.5").type == "ipv4-addr" + assert mapping.map_ip_observable("2001:db8::5").type == "ipv6-addr" + + +def test_build_network_traffic_uses_normalized_lowercase_protocol(): + """Protocol strings get lower-cased regardless of how the scanner reported them.""" + nt = mapping.build_network_traffic(_row(protocol="TCP"), _IP_ID) + assert nt.protocols == ["tcp"] + assert nt.src_port == 443 + assert nt.src_ref == _IP_ID + + +def test_build_network_traffic_raises_without_a_port(): + """A row with no port number is malformed, not a default-able case.""" + with pytest.raises(ValueError): + mapping.build_network_traffic(_row(port=None), _IP_ID) + + +def test_build_network_traffic_raises_without_a_protocol(): + """A row with no protocol is malformed, not a default-able case.""" + with pytest.raises(ValueError): + mapping.build_network_traffic(_row(protocol=None), _IP_ID) + + +def test_build_network_traffic_id_is_stable_when_service_name_changes(): + """The id must not drift just because scanner-reported service metadata changes. + + §10i-style verification, done proactively: only src_ref/src_port/protocols are + ID-contributing (verified against the installed stix2 library). + """ + nt1 = mapping.build_network_traffic(_row(service_name="https"), _IP_ID) + nt2 = mapping.build_network_traffic(_row(service_name="something else"), _IP_ID) + assert nt1.id == nt2.id + + +def test_build_network_traffic_id_is_stable_when_state_changes(): + """The id must not drift when the scanner-reported state flips open<->closed either. + + Same reasoning as service_name above -- the vs-state-*/vs-open labels must be free to change + on a rescan without fragmenting the SCO into a new object. + """ + nt_open = mapping.build_network_traffic(_row(state="open"), _IP_ID) + nt_closed = mapping.build_network_traffic(_row(state="closed"), _IP_ID) + assert nt_open.id == nt_closed.id + + +def test_build_network_traffic_open_reflects_scanner_state_not_recency(): + """The "vs-open" label must come from LatestPortScan.state == "open", not any recency notion. + + connector.py's own `current`/staleness tracking is a wholly separate concept -- see + mapping.py's module docstring for why an earlier version of this code got that wrong. Also + verifies the module docstring's own claim: x_opencti_open/x_opencti_service aren't real + fields on this platform's Network-Traffic schema (confirmed end-to-end against a live + instance) -- must not even be present on the built object. + """ + open_nt = mapping.build_network_traffic(_row(state="open"), _IP_ID) + assert "vs-open" in open_nt.labels + assert "x_opencti_open" not in open_nt + assert "x_opencti_service" not in open_nt + for closed_state in ("closed", "filtered", "open|filtered", None, " "): + nt = mapping.build_network_traffic(_row(state=closed_state), _IP_ID) + assert "vs-open" not in (nt.get("labels") or []), closed_state + + +def test_build_network_traffic_sets_service_label_and_description(): + """service_name must land as both a vs-service-* label and the existing description field. + + x_opencti_description was already confirmed working before this change; the label is new. + """ + nt = mapping.build_network_traffic(_row(service_name="https"), _IP_ID) + assert "vs-service-https" in nt.labels + assert nt.x_opencti_description == "https" + + +def test_build_network_traffic_labels_include_raw_state(): + """vs-state- must mirror the exact state lifecycle_labels() puts on the relationship.""" + nt = mapping.build_network_traffic(_row(state="open|filtered"), _IP_ID) + assert "vs-state-open|filtered" in nt.labels + + +def test_is_port_state_open_reduces_nmap_style_states_to_a_plain_boolean(): + """Only a literal "open" (case/whitespace-insensitive) counts -- everything else is False.""" + assert mapping.is_port_state_open("open") is True + assert mapping.is_port_state_open("Open ") is True + assert mapping.is_port_state_open("closed") is False + assert mapping.is_port_state_open("filtered") is False + assert mapping.is_port_state_open(None) is False + assert mapping.is_port_state_open("") is False + + +def test_build_software_returns_none_without_product_version_or_cpe(): + """A row with no software detail at all produces no Software SCO.""" + assert mapping.build_software(_row()) is None + + +def test_build_software_maps_product_version_and_cpe(): + """service_product/version/cpe map onto Software's name/version/cpe respectively.""" + software = mapping.build_software( + _row( + service_product="Apache httpd", + service_version="2.4.41", + service_cpe="cpe:/a:x", + ) + ) + assert software.name == "Apache httpd" + assert software.version == "2.4.41" + assert software.cpe == "cpe:/a:x" + + +def test_build_software_id_changes_with_version(): + """A genuinely different version is a genuinely different Software entity. + + Content-derived id drift here is correct, unlike NetworkTraffic's lifecycle fields. + """ + a = mapping.build_software( + _row(service_product="Apache httpd", service_version="2.4.41") + ) + b = mapping.build_software( + _row(service_product="Apache httpd", service_version="2.4.62") + ) + assert a.id != b.id + + +def test_normalize_timestamp_parses_offset_and_z_suffixed_strings_to_the_same_instant(): + """A bare +00:00-offset string and a Z-suffixed string must parse to the same instant. + + §10i: stix2's TimestampProperty rejects a bare +00:00-offset string outright -- this helper + must always hand back a real datetime regardless of which ISO variant it's given. + """ + a = mapping.normalize_timestamp("2026-08-20T12:00:00+00:00") + b = mapping.normalize_timestamp("2026-08-20T12:00:00Z") + assert isinstance(a, datetime.datetime) + assert a == b + + +def test_build_lifecycle_relationship_reuses_existing_id_when_pinned(): + """A pinned id must be returned verbatim, not recomputed from source/target/time.""" + pinned_id = StixCoreRelationship.generate_id( + "related-to", _ORG_ID, "network-traffic--x", None, None + ) + rel = mapping.build_lifecycle_relationship( + _row(), + _ORG_ID, + _NETWORK_TRAFFIC_ID, + _AUTHOR_ID, + _MARKING_ID, + start_time=None, + stop_time=None, + existing_id=pinned_id, + ) + assert rel.id == pinned_id + + +def test_build_lifecycle_relationship_labels_reflect_row_fields(): + """source/risky_service_group/nmi_service_group/state must all become labels.""" + rel = mapping.build_lifecycle_relationship( + _row( + source="nmap", + risky_service_group="telnet", + nmi_service_group="web", + state="open", + ), + _ORG_ID, + _NETWORK_TRAFFIC_ID, + _AUTHOR_ID, + _MARKING_ID, + start_time=None, + stop_time=None, + ) + assert set(rel.labels) == { + "vs-source-nmap", + "vs-risky-service-telnet", + "vs-nmi-service-web", + "vs-state-open", + } + + +def test_lifecycle_labels_matches_what_build_lifecycle_relationship_produces(): + """This must be the exact same label computation build_lifecycle_relationship() uses. + + It's cached in connector state for the aging sweep (db.py's module docstring), so a drift + here would silently change labels on a locally-closed relationship. + """ + row = _row( + source="nmap", + risky_service_group="telnet", + nmi_service_group="web", + state="open", + ) + assert set(mapping.lifecycle_labels(row)) == { + "vs-source-nmap", + "vs-risky-service-telnet", + "vs-nmi-service-web", + "vs-state-open", + } + + +def test_lifecycle_external_id_prefers_port_scan_id_over_id(): + """port_scan_id is preferred; the row's own id is only a fallback.""" + assert ( + mapping.lifecycle_external_id(_row(port_scan_id="ps-real", id="fallback")) + == "ps-real" + ) + assert ( + mapping.lifecycle_external_id(_row(port_scan_id=None, id="fallback")) + == "fallback" + ) + + +def test_build_lifecycle_relationship_from_parts_reuses_existing_id_when_pinned(): + """The lower-level, state-only builder (no fresh row) must also honor a pinned id.""" + pinned_id = StixCoreRelationship.generate_id( + "related-to", _ORG_ID, "network-traffic--x", None, None + ) + rel = mapping.build_lifecycle_relationship_from_parts( + _ORG_ID, + _NETWORK_TRAFFIC_ID, + _AUTHOR_ID, + _MARKING_ID, + start_time=None, + stop_time=None, + labels=["vs-source-nmap"], + external_id="ps-1", + existing_id=pinned_id, + ) + assert rel.id == pinned_id + + +def test_build_lifecycle_relationship_and_from_parts_produce_identical_relationships(): + """The row-driven wrapper must be a pure pass-through to the parts-based builder. + + Same id, same labels, same external reference, for the same underlying data. + """ + row = _row(source="nmap", risky_service_group="telnet") + via_row = mapping.build_lifecycle_relationship( + row, + _ORG_ID, + _NETWORK_TRAFFIC_ID, + _AUTHOR_ID, + _MARKING_ID, + start_time=None, + stop_time=None, + ) + via_parts = mapping.build_lifecycle_relationship_from_parts( + _ORG_ID, + _NETWORK_TRAFFIC_ID, + _AUTHOR_ID, + _MARKING_ID, + start_time=None, + stop_time=None, + labels=mapping.lifecycle_labels(row), + external_id=mapping.lifecycle_external_id(row), + ) + assert via_row.id == via_parts.id + assert via_row.labels == via_parts.labels + + +def test_dedupe_bundle_objects_collapses_repeated_ids(): + """Two STIX objects sharing an id must collapse to one in dedupe_bundle_objects().""" + a = stix2.IPv4Address(value="10.0.0.1") + a_again = stix2.IPv4Address(value="10.0.0.1") + b = stix2.IPv4Address(value="10.0.0.2") + result = mapping.dedupe_bundle_objects([a, a_again, b]) + assert len(result) == 2 diff --git a/open-cti/connectors/vs-ticket-ingestion/Dockerfile b/open-cti/connectors/vs-ticket-ingestion/Dockerfile new file mode 100644 index 000000000..8a8dbc27b --- /dev/null +++ b/open-cti/connectors/vs-ticket-ingestion/Dockerfile @@ -0,0 +1,30 @@ +FROM python:3.12-slim + +WORKDIR /opt/connector + +# Zscaler TLS-inspects outbound HTTPS on this network -- identical setup to connector D's +# Dockerfile (see its comments/README.md "TLS interception" for the full explanation). Reused +# verbatim rather than rediscovered: this is a property of the *network* connector D's EC2 host +# sits behind, not of connector D specifically -- every connector's build hits the same failure. +# +# Pulled from a named build context (docker-compose.yml's additional_contexts, pointed at +# /opt/open-cti/tls -- runtime state, never this repo checkout), never COPYed from anywhere +# inside this directory -- the cert never touches git. +COPY --from=zscaler_cert zscaler.pem /usr/local/share/ca-certificates/zscaler.crt +RUN apt-get update \ + && apt-get install -y --no-install-recommends ca-certificates \ + && update-ca-certificates \ + && rm -rf /var/lib/apt/lists/* +ENV PIP_CERT=/etc/ssl/certs/ca-certificates.crt \ + REQUESTS_CA_BUNDLE=/etc/ssl/certs/ca-certificates.crt \ + AWS_CA_BUNDLE=/etc/ssl/certs/ca-certificates.crt \ + SSL_CERT_FILE=/etc/ssl/certs/ca-certificates.crt + +COPY requirements.txt . +RUN pip install --no-cache-dir -r requirements.txt + +COPY src/ src/ +COPY tests/ tests/ + +# Relative imports in src/main.py require running as a module, not a script -- see README.md. +ENTRYPOINT ["python3", "-m", "src.main"] diff --git a/open-cti/connectors/vs-ticket-ingestion/README.md b/open-cti/connectors/vs-ticket-ingestion/README.md new file mode 100644 index 000000000..4f05e73d9 --- /dev/null +++ b/open-cti/connectors/vs-ticket-ingestion/README.md @@ -0,0 +1,159 @@ +# VS Ticket Ingestion Connector + +Connector A in the four-connector VS architecture — see +[`../../OpenCTI-connector.md`](../../OpenCTI-connector.md) §7a for the full design, §8 for build +order (this one depends on connector D), and §10 (especially §10i) for the implementation +discipline this code follows. + +Depends on [connector D](../vs-org-bootstrap/) — the organization `Identity` a ticket resolves to +must already exist. Deploy order: D, then this one. + +## What it does + +Reads `Ticket` from `mini_data_lake` and upserts `IPv4-Addr`/`IPv6-Addr` (scanned host) and +`Vulnerability` (CVE, or a named risky-service finding for CVE-less tickets) objects, plus two +kinds of relationship: + +- IP `related-to` Vulnerability, carrying the ticket's open/closed lifecycle as native + `start_time`/`stop_time` — no `Case`/`Incident` container (§7a's reasoning for why not). Also + carries `Ticket.updated_timestamp` as a custom property, `x_opencti_updated_timestamp` — see + "Ticket.updated_timestamp" below. +- Organization `related-to` IP, so the finding is actually reachable from its owning org in the + graph (the org `Identity` itself is *looked up*, never created here — that's connector D's job). + +### `Ticket.updated_timestamp` + +Carried onto the IP↔Vulnerability relationship (not dropped after being used for the watermark, +and deliberately not stamped onto the `Vulnerability` SDO — see below) as +`x_opencti_updated_timestamp`, a plain custom property rather than stix2's native `modified`. + +Why not `modified`: OpenCTI/stix2 give that field real merge/versioning semantics, and this +connector's own bootstrap poll can legitimately hand back tickets whose `updated_timestamp` is +older than what's already stored for a given relationship — driving `modified` from it risks the +platform treating an old bootstrap row as stale and silently skipping the write. A plain custom +property carries the same information with none of that risk. Verified directly against the +installed `stix2` library that a real `datetime.datetime` passed through `custom_properties` +serializes correctly to a proper `Z`-suffixed ISO string, and that custom properties have no +bearing on this relationship's id either way (it's always explicitly pinned via `existing_id`, +never derived from its own content). + +Why not on the `Vulnerability` SDO: that object is deliberately minimal and shared/deduped across +every ticket and org referencing the same CVE (see `mapping.py`'s `map_vulnerability()` docstring) +— stamping one ticket's update time onto it would mean whichever connector run happened to touch +that CVE last overwrites a fact that has nothing to do with the CVE itself. The relationship, +scoped to one specific (ip, vulnerability) pair, is the correct home — the same place +`opened_timestamp`/`closed_timestamp` already live as `start_time`/`stop_time`. + +A ticket that flips to `false_positive=True` on a later poll has its relationship **revoked** +(deleted via the OpenCTI API directly, not through the STIX bundle — bundles only upsert) rather +than left in the graph as a stale "still vulnerable" edge. + +## Structure + +```text +src/ + config.py # env/yaml config loading, fails closed on tlp_marking + org scope + db.py # mini_data_lake access + IS_LOCAL fixture fallback + mapping.py # pure row -> STIX mapping functions (no I/O) -- Loop 1 test target + connector.py # orchestration: fetch -> map -> bundle -> send, state/idempotency, revocation + main.py # entry point +tests/ + fixtures/ # small synthetic sample rows, shaped like real mini_data_lake queries + test_mapping.py # Loop 1: pure unit tests, no DB/OpenCTI/queue at all + test_connector_dry_run.py # Loop 2: full pipeline against fixtures, OpenCTI helper stubbed out +``` + +Structurally identical to [connector D](../vs-org-bootstrap/) on purpose (§10i: apply the same +discipline from the start, don't reinvent it per connector). Run the fast loops constantly: + +```bash +pip install -r requirements.txt +python3 -m pytest tests/ -v +``` + +## Two things this connector refuses to start without + +Same two fail-closed checks as connector D, in `src/config.py`: + +- **`VS_TICKET_INGESTION_TLP_MARKING`** — must match connector D's value once the TLP/marking + policy decision (`OpenCTI-connector.md` §10c) is actually made; it's one shared decision across + all four connectors, not a per-connector default to pick independently. +- **`VS_TICKET_INGESTION_ORG_ACRONYM_ALLOWLIST`** (or explicit + `VS_TICKET_INGESTION_ALLOW_UNSCOPED_RUN=true`) — an empty allowlist means "scoped to nothing," + not "all orgs" (§9c). + +## Keeping a first run fast: `VS_TICKET_INGESTION_LOOKBACK_DAYS` + +A fresh (or reset) connector state has no watermark yet, so the first poll pulls *every* ticket +ever recorded for the scoped orgs — slow to iterate against for any org with real history. Set +`VS_TICKET_INGESTION_LOOKBACK_DAYS` (e.g. `7`) to bound that first poll to the last N days +instead; unset/empty (the default) means no bound, matching the original full-backfill behavior. + +This only ever affects a poll with **no existing watermark** — once one exists, `lookback_days` +is ignored, deliberately. Clamping every ongoing poll to a rolling window would silently create a +permanent gap: a still-open ticket whose last-seen timestamp never changes again would fall +outside the window and then never get picked up, since the watermark only ever advances forward +from here. + +**That gap is real, not theoretical — measured against the live box (2026-08-28):** with a 60-day +lookback, 413,925 real `is_open=true` tickets on `cyhy_mini_data_lake_staging` had a last-touch +older than the cutoff. A plain lookback would have silently, permanently dropped every one of +them, not "eventually" ingested them. The fix: the bootstrap poll (and *only* the bootstrap poll — +`connector.run()` computes `is_first_run = state.get("last_seen_watermark") is None` and passes +it straight through as `include_stale_open`) also pulls in every currently-open ticket regardless +of how stale its last-touch date is, alongside whatever falls inside the lookback window. So +`lookback_days` still controls "how much recent history to also pull on top of that," but no +longer controls "whether currently-actionable findings get missed forever" — those two concerns +used to be conflated in one knob, and aren't anymore. See `db.py`'s `_fetch_tickets_live()` for +the query-level detail and the real-Postgres verification behind it. + +`VS_TICKET_INGESTION_MAX_ROWS_PER_RUN` (also tunable via `docker-compose.yml`) is the +complementary, always-safe lever for both the lookback window and the stale-open backlog — it +only paginates slower across multiple polls, it never skips data. Size it off the real numbers +for your environment (`SELECT count(*) FROM ticket WHERE is_open = true AND +COALESCE(closed_timestamp, updated_timestamp) <= now() - interval 'N days'` via +`crossfeed-staging-bastion`, plus the in-window count) rather than guessing — a large stale-open +count divided by a small row cap is a lot of daily polls before the backlog actually clears. + +## Running it + +Same two modes as connector D — see its README for the fuller explanation of each: + +- **Fixture mode** (`IS_LOCAL=true`) — `db.py` loads `tests/fixtures/tickets.json` instead of + querying Postgres. Still needs a real (or locally stacked) OpenCTI to send to. +- **Against real `mini_data_lake` + OpenCTI** — copy `config.yml.sample` to `config.yml`, fill in + `db_host` and a real `org_acronym_allowlist` scoped to one or two low-volume test orgs, then: + + ```bash + python3 -m src.main + ``` + + (Must run as a module — relative imports. The Dockerfile's `ENTRYPOINT` already does this.) + +## TLS interception (Zscaler) + +Identical setup to connector D — see [its README](../vs-org-bootstrap/README.md#tls-interception-zscaler--required-manual-per-instance-setup-not-git-managed) +for the full explanation. This is a property of the network the EC2 host sits behind, already set +up once for connector D; nothing additional to do per-connector beyond the `additional_contexts` +block in `docker-compose.yml` (already present for this service). + +## Known gaps / things to verify before this is production-ready + +- **Real per-run timings not yet measured.** `max_rows_per_run` starts at the same 5000 default + connector D used for its first scoped runs, but tickets outnumber orgs/CIDRs by a lot (§7a) -- + watch actual poll duration once this runs against real data before assuming that cap is right + at full (unscoped) production scale. +- **`resolve_org_identity_id()`'s coupling to connector D:** this connector never creates or + verifies an org `Identity` — it recomputes the same content-derived id connector D's + `map_organization()` would produce for the same `organization.name`, assuming that name hasn't + drifted between D's last sync and this row's join. Real but narrow; both connectors read the + same source-of-truth column, just at different times. +- **Severity bucketing (`mapping.severity_label`) uses the standard CVSS v3 qualitative scale** + (low/medium/high/critical cutoffs at 4.0/7.0/9.0) — not yet checked against the actual + distribution of real `Ticket.cvss_severity` values in `mini_data_lake`; verify before trusting + the resulting labels for any downstream triage/reporting use. +- Not yet run against a real OpenCTI instance — the fixture-based dry-run pipeline (Loop 2, §9b) + and a throwaway real-Postgres type check (Loop 4, §9b — confirmed `cvss_severity` returns + `decimal.Decimal`, `organization_id`'s join needs no `uuid` cast, and every timestamp column + returns a real `datetime.datetime`) are both done; the actual send-and-verify-in-OpenCTI step is + next, the same order connector D went through. diff --git a/open-cti/connectors/vs-ticket-ingestion/config.yml.sample b/open-cti/connectors/vs-ticket-ingestion/config.yml.sample new file mode 100644 index 000000000..5c75c0ed2 --- /dev/null +++ b/open-cti/connectors/vs-ticket-ingestion/config.yml.sample @@ -0,0 +1,66 @@ +# Connector A — VS Ticket Ingestion +# +# See ../../OpenCTI-connector.md §7a for the design this implements, and §10 (especially §10i, +# lessons from building connector D) for the discipline this config exists to enforce. +# +# Copy to config.yml for local runs (docker-compose.yml drives the containerized version purely +# from env vars instead — see README.md). + +opencti: + url: 'http://localhost:8080' + token: 'ChangeMe' + +connector: + id: 'ChangeMe-generate-a-uuid4' + name: 'VS Ticket Ingestion' + scope: 'ipv4-addr,ipv6-addr,vulnerability' + type: 'EXTERNAL_IMPORT' + confidence_level: 75 + log_level: 'info' + # ISO 8601 duration. Should not poll tighter than VulnScanningSync's own cadence (§5, still + # unresolved) -- P1D matches connector-cisa-known-exploited-vulnerabilities' current setting, + # used as the starting point per §7a. + duration_period: 'P1D' + +vs_ticket_ingestion: + # --- mini_data_lake connectivity (OpenCTI-connector.md §2) --- + db_host: 'ChangeMe' + db_port: 5432 + db_name: 'cyhy_mini_data_lake_staging' + db_user: 'open_cti' + # IAM database authentication, not a static password -- see §2. When true, db_password is + # ignored and a short-lived token is generated per connection via boto3/STS. + db_use_iam_auth: true + db_password: '' + aws_region: 'us-gov-west-1' + + # --- dev/test discipline (§9) --- + # When true, skip mini_data_lake entirely and load tests/fixtures/*.json instead. + is_local: false + local_fixture_dir: './tests/fixtures' + + # Comma-separated org acronym allowlist. REQUIRED to be non-empty outside of a deliberate + # production run -- see §9c. Empty string here means "no orgs," not "all orgs": this connector + # refuses to run unscoped by accident (see src/config.py). + org_acronym_allowlist: '' + # Explicit opt-in to remove the allowlist requirement for a real production run. + allow_unscoped_run: false + # Tickets outnumber orgs/CIDRs by a lot (§7a) -- start at the same cap connector D used for its + # first scoped test runs, tune upward deliberately once real per-run timings are observed. + max_rows_per_run: 5000 + # §9c's "lookback override" lever. Empty/unset (default) = no bound, first poll pulls every + # ticket ever recorded for the scoped orgs. Set to a small number (e.g. 7) during dev to bound + # a fresh/reset connector's *first* poll against an org with a lot of history -- has no effect + # once a real watermark exists. Only skips data outright if a still-open ticket's last-seen + # timestamp genuinely never moves again after falling outside this window on the first poll -- + # a real, if narrow, completeness tradeoff (see connector.py's _effective_since docstring), not + # something to set in production without understanding that risk. + lookback_days: '' + + # --- attribution / marking (§10c) --- + # Same author name as connector D on purpose -- one consistent system Identity across all four + # connectors. + author_name: 'CISA VulnScanning' + # Deliberately left unset -- same undecided policy question as connector D (§10c). Set to a + # marking definition name once the policy call is made; must match connector D's value. + tlp_marking: '' diff --git a/open-cti/connectors/vs-ticket-ingestion/requirements.txt b/open-cti/connectors/vs-ticket-ingestion/requirements.txt new file mode 100644 index 000000000..17d9c9b7f --- /dev/null +++ b/open-cti/connectors/vs-ticket-ingestion/requirements.txt @@ -0,0 +1,5 @@ +boto3==1.43.81 +psycopg2-binary==2.9.9 +pycti==7.260824.0 +python-dateutil==2.9.0.post0 +stix2==3.0.2 diff --git a/open-cti/connectors/vs-ticket-ingestion/src/__init__.py b/open-cti/connectors/vs-ticket-ingestion/src/__init__.py new file mode 100644 index 000000000..a2c6f691d --- /dev/null +++ b/open-cti/connectors/vs-ticket-ingestion/src/__init__.py @@ -0,0 +1 @@ +"""VS Ticket Ingestion connector -- see ../README.md and ../../OpenCTI-connector.md.""" diff --git a/open-cti/connectors/vs-ticket-ingestion/src/config.py b/open-cti/connectors/vs-ticket-ingestion/src/config.py new file mode 100644 index 000000000..3de51534b --- /dev/null +++ b/open-cti/connectors/vs-ticket-ingestion/src/config.py @@ -0,0 +1,195 @@ +"""Config loading for the VS Ticket Ingestion connector. + +Structurally identical to connector D's config.py (OpenCTI-connector.md §10i: apply the same +discipline from the start, don't rediscover it) -- precedence is env var > config.yml > default +via pycti's own get_config_variable(), and the same two things fail closed rather than silently +defaulting to something wrong: + - tlp_marking: still the same undecided policy question (§10c) -- shared across all four + connectors, not re-litigated per connector. + - org_acronym_allowlist: empty means "scoped to nothing," not "all orgs" (§9c). +""" + +# Standard Python Libraries +import os +from typing import Dict, List, Optional + +# Third-Party Libraries +from pycti import get_config_variable +import yaml + + +class ConfigError(RuntimeError): + """Raised when required configuration is missing -- fail closed, not silent default.""" + + +def load_yaml_config() -> Dict: + """Load config.yml next to this connector if present; docker-compose drives env vars only.""" + here = os.path.dirname(os.path.abspath(__file__)) + config_file_path = os.path.join(here, "..", "config.yml") + if os.path.isfile(config_file_path): + with open(config_file_path, encoding="utf-8") as f: + return yaml.safe_load(f) or {} + return {} + + +class Config: + """Typed view over this connector's configuration.""" + + def __init__(self, raw: Optional[Dict] = None): + """Load config from `raw` if given (tests), else config.yml/env vars, then validate.""" + self.raw: Dict = raw if raw is not None else load_yaml_config() + + # --- mini_data_lake connectivity (OpenCTI-connector.md §2) --- + self.db_host = get_config_variable( + "VS_TICKET_INGESTION_DB_HOST", ["vs_ticket_ingestion", "db_host"], self.raw + ) + self.db_port = get_config_variable( + "VS_TICKET_INGESTION_DB_PORT", + ["vs_ticket_ingestion", "db_port"], + self.raw, + isNumber=True, + default=5432, + ) + # cyhy_mini_data_lake_staging is the confirmed real staging DB name (verified 2026-08-27 + # via connector D against the live box); "mini_data_lake" stays the architectural name + # for this data store throughout comments/docs. + self.db_name = get_config_variable( + "VS_TICKET_INGESTION_DB_NAME", + ["vs_ticket_ingestion", "db_name"], + self.raw, + default="cyhy_mini_data_lake_staging", + ) + self.db_user = get_config_variable( + "VS_TICKET_INGESTION_DB_USER", + ["vs_ticket_ingestion", "db_user"], + self.raw, + default="open_cti", + ) + self.db_use_iam_auth = get_config_variable( + "VS_TICKET_INGESTION_DB_USE_IAM_AUTH", + ["vs_ticket_ingestion", "db_use_iam_auth"], + self.raw, + default=True, + ) + self.db_password = get_config_variable( + "VS_TICKET_INGESTION_DB_PASSWORD", + ["vs_ticket_ingestion", "db_password"], + self.raw, + default="", + ) + self.aws_region = get_config_variable( + "VS_TICKET_INGESTION_AWS_REGION", + ["vs_ticket_ingestion", "aws_region"], + self.raw, + ) + + # --- dev/test discipline (§9) --- + self.is_local = get_config_variable( + "IS_LOCAL", ["vs_ticket_ingestion", "is_local"], self.raw, default=False + ) + self.local_fixture_dir = get_config_variable( + "VS_TICKET_INGESTION_LOCAL_FIXTURE_DIR", + ["vs_ticket_ingestion", "local_fixture_dir"], + self.raw, + default="./tests/fixtures", + ) + + org_allowlist_raw = get_config_variable( + "VS_TICKET_INGESTION_ORG_ACRONYM_ALLOWLIST", + ["vs_ticket_ingestion", "org_acronym_allowlist"], + self.raw, + default="", + ) + self.org_acronym_allowlist: List[str] = [ + acronym.strip() + for acronym in (org_allowlist_raw or "").split(",") + if acronym.strip() + ] + self.allow_unscoped_run = get_config_variable( + "VS_TICKET_INGESTION_ALLOW_UNSCOPED_RUN", + ["vs_ticket_ingestion", "allow_unscoped_run"], + self.raw, + default=False, + ) + # Tickets outnumber orgs/CIDRs by a lot (§7a) -- start with the same default row cap as + # connector D for the first scoped test runs, not a bigger number just because there's + # more data; tune upward deliberately once real per-run timings are observed (§9c/§9d). + self.max_rows_per_run = get_config_variable( + "VS_TICKET_INGESTION_MAX_ROWS_PER_RUN", + ["vs_ticket_ingestion", "max_rows_per_run"], + self.raw, + isNumber=True, + default=5000, + ) + # §9c's "Lookback override" scoping lever, planned but not built until now: bounds the + # *first* poll against a fresh/reset connector state to the last N days, instead of every + # ticket ever recorded for the scoped orgs. Read manually (not isNumber=True) because + # get_config_variable() calls int(result) before checking for an empty string -- verified + # directly against the installed pycti source -- so an unset-but-present env var (the + # `${VAR:-}` pattern docker-compose.yml already uses for every other optional lever here) + # would crash with "invalid literal for int()" instead of falling back to the default. + # None (the default) means "no bound, pull the whole history" -- unchanged behavior, + # matching the module docstring's fail-closed philosophy: don't silently start skipping + # data just because a dev convenience knob exists. See connector.py for where this is + # actually applied (only when no watermark exists yet, never overriding one that does -- + # bounding an in-progress incremental poll would create a permanent gap, not just delay). + lookback_days_raw = get_config_variable( + "VS_TICKET_INGESTION_LOOKBACK_DAYS", + ["vs_ticket_ingestion", "lookback_days"], + self.raw, + default="", + ) + self.lookback_days: Optional[int] = ( + int(lookback_days_raw) if lookback_days_raw not in (None, "") else None + ) + + # --- attribution / marking (§10c) --- + # Same author name as connector D on purpose -- all VS-sourced content across all four + # connectors should trace back to one consistent system Identity, not a per-connector one. + self.author_name = get_config_variable( + "VS_TICKET_INGESTION_AUTHOR_NAME", + ["vs_ticket_ingestion", "author_name"], + self.raw, + default="CISA VulnScanning", + ) + self.tlp_marking = get_config_variable( + "VS_TICKET_INGESTION_TLP_MARKING", + ["vs_ticket_ingestion", "tlp_marking"], + self.raw, + ) + + # §7a -- polls on a schedule, should not be tighter than VulnScanningSync's own cadence + # (§5, still unresolved). P1D matches connector-cisa-known-exploited-vulnerabilities' + # current setting, used as the starting point per §7a. + self.duration_period = get_config_variable( + "CONNECTOR_DURATION_PERIOD", + ["connector", "duration_period"], + self.raw, + default="P1D", + ) + + self._validate() + + def _validate(self) -> None: + """Fail closed on the two decisions this doc can't make silently. See module docstring.""" + if not self.tlp_marking: + raise ConfigError( + "vs_ticket_ingestion.tlp_marking is not set. This is a real, undecided policy " + "question (OpenCTI-connector.md §10c) -- refusing to start rather than default " + "to something like TLP:CLEAR for stakeholder vulnerability data. Set " + "VS_TICKET_INGESTION_TLP_MARKING once the data-handling policy call is made." + ) + if not self.org_acronym_allowlist and not self.allow_unscoped_run: + raise ConfigError( + "vs_ticket_ingestion.org_acronym_allowlist is empty and allow_unscoped_run is " + "not set. An empty allowlist means 'scoped to nothing', not 'all orgs' -- see " + "OpenCTI-connector.md §9c. Set VS_TICKET_INGESTION_ORG_ACRONYM_ALLOWLIST for dev, " + "or explicitly set VS_TICKET_INGESTION_ALLOW_UNSCOPED_RUN=true for a deliberate " + "full-scope production run." + ) + if not self.is_local and not self.db_host: + raise ConfigError( + "vs_ticket_ingestion.db_host is not set and is_local is false -- nothing to " + "connect to. Set VS_TICKET_INGESTION_DB_HOST, or IS_LOCAL=true to use fixtures " + "instead." + ) diff --git a/open-cti/connectors/vs-ticket-ingestion/src/connector.py b/open-cti/connectors/vs-ticket-ingestion/src/connector.py new file mode 100644 index 000000000..e3183803a --- /dev/null +++ b/open-cti/connectors/vs-ticket-ingestion/src/connector.py @@ -0,0 +1,265 @@ +"""Orchestration for the VS Ticket Ingestion connector (OpenCTI-connector.md §7a). + +Depends on connector D (§7d/§8) -- the organization Identity a ticket resolves to (§7a's "look +up, don't blindly create") must already exist. Deploy order: D, then this one. + +Builds proactively on every lesson connector D's live debugging surfaced (§10i), rather than +rediscovering the same bug classes: AppLogger call signatures, row-level error isolation from +day one, the `IS NULL OR` unscoped-query guard, and the datetime/JSON watermark gotcha are all +already accounted for below and in db.py -- not patched in after a crash this time. +""" + +# Standard Python Libraries +import datetime +import logging +from typing import Dict, Optional + +# Third-Party Libraries +from pycti import OpenCTIConnectorHelper +import stix2 + +from . import mapping +from .config import Config +from .db import TicketIngestionData, VsTicketIngestionRepository + +LOGGER = logging.getLogger("vs_ticket_ingestion") + +# Same well-known TLP marking-definition ids as connector D -- see its connector.py for why +# TLP:CLEAR/TLP:WHITE share one object (this stix2 version only ships the pre-2.1 name). +_TLP_MARKINGS = { + "TLP:CLEAR": stix2.TLP_WHITE, + "TLP:WHITE": stix2.TLP_WHITE, + "TLP:GREEN": stix2.TLP_GREEN, + "TLP:AMBER": stix2.TLP_AMBER, + "TLP:RED": stix2.TLP_RED, +} + + +def resolve_marking(tlp_marking: str) -> stix2.MarkingDefinition: + """Resolve a TLP string to its well-known marking-definition object. Identical to connector D.""" + marking = _TLP_MARKINGS.get(tlp_marking.strip().upper()) + if marking is None: + raise ValueError( + f"Unrecognized tlp_marking {tlp_marking!r}. Expected one of: " + f"{', '.join(_TLP_MARKINGS)} (see OpenCTI-connector.md §10c)." + ) + return marking + + +class VsTicketIngestionConnector: + """Polls Ticket data from mini_data_lake and upserts it into OpenCTI. + + Produces IPv4-Addr/IPv6-Addr and Vulnerability objects plus the relationship carrying each + ticket's open/closed lifecycle -- no Case/Incident container (§7a). + """ + + def __init__(self, config: Optional[Config] = None, helper=None): + """Build the connector, injecting `config`/`helper` for tests instead of the real thing.""" + self.config = config or Config() + self.helper = helper or OpenCTIConnectorHelper(self.config.raw) + self.repository = VsTicketIngestionRepository(self.config) + + self.author = mapping.build_author(self.config.author_name) + self.marking = resolve_marking(self.config.tlp_marking) + + LOGGER.info( + "VS Ticket Ingestion connector initialized (is_local=%s, org_allowlist=%s)", + self.config.is_local, + self.config.org_acronym_allowlist or "", + ) + + def run(self) -> None: + """Entry point pycti's scheduler calls each duration_period tick.""" + self.helper.connector_logger.info("Starting VS Ticket Ingestion run") + state = self.helper.get_state() or {} + + # is_first_run gates both _effective_since's lookback bound and the stale-open safety + # net below -- both only ever apply to the bootstrap poll, never an in-progress one. + is_first_run = state.get("last_seen_watermark") is None + since_last_seen = self._effective_since(state) + try: + data = self.repository.fetch( + since_last_seen=since_last_seen, include_stale_open=is_first_run + ) + new_watermark = self._process(data, state) + except Exception as e: # pylint: disable=broad-except + # §10e/§10i: don't let one bad run corrupt state -- log and let the next scheduled + # tick retry from the last confirmed watermark. connector_logger is pycti's AppLogger, + # not a stdlib logging.Logger -- (message, meta=None) only, no exc_info kwarg (it + # captures the traceback internally regardless). Got this right from the start here; + # connector D only found it by crashing on a live run. + self.helper.connector_logger.error( + "VS Ticket Ingestion run failed", meta={"error": str(e)} + ) + return + + # §10f: only advance the watermark once the run has actually completed -- new_watermark + # is None (unchanged) whenever there was nothing to process. + if new_watermark is not None: + state["last_seen_watermark"] = new_watermark + self.helper.set_state(state) + + self.helper.connector_logger.info("VS Ticket Ingestion run complete") + + def _effective_since(self, state: Dict) -> Optional[str]: + """Compute the `since_last_seen` to actually poll from -- watermark, lookback bound, or none. + + §9c's "lookback override" lever, finally built: bounds the *first* poll against a + fresh/reset connector state to the last `lookback_days` days, instead of every ticket + ever recorded for the scoped orgs -- the thing that makes an org with years of history + slow to iterate against during dev, and the exact scenario the doc called out ("matters + especially the first time a fresh connector state points at a real environment"). + + Deliberately only kicks in when there is no watermark yet. Once a real incremental poll is + underway, clamping `since` to a rolling lookback window on every run would silently create + a permanent gap -- a ticket whose last-seen timestamp never moves again (e.g. opened long + ago, still open, nothing about it has changed since) would fall outside the window and + then never be picked up again, since the watermark only ever advances forward from here. + That's a real completeness cost, not just a delay, which is why `lookback_days` defaults + to unset/None (§10c's same fail-closed instinct: don't silently start skipping data) and + why this only ever touches the *first* poll, never an in-progress one. + + **This alone is not enough**, discovered against real production numbers (2026-08-28): + 413,925 real `is_open=true` tickets on the live box had a last-touch older than a 60-day + lookback -- not delayed, permanently excluded, since the watermark this method computes + can never move backward. `run()`'s `include_stale_open` flag (passed to + `repository.fetch()` alongside this value) is the actual fix for that gap -- see + `db._fetch_tickets_live()` for the query-level detail. This method's lookback bound still + controls what counts as "recent enough," `include_stale_open` is what keeps + currently-actionable-but-stale tickets from falling through that same crack. + """ + watermark = state.get("last_seen_watermark") + if watermark is not None or self.config.lookback_days is None: + return watermark + since = datetime.datetime.now(datetime.timezone.utc) - datetime.timedelta( + days=self.config.lookback_days + ) + self.helper.connector_logger.info( + f"No watermark yet -- bounding this first poll to the last " + f"{self.config.lookback_days} day(s) (VS_TICKET_INGESTION_LOOKBACK_DAYS)" + ) + return since.isoformat() + + # ------------------------------------------------------------------ + + def _process(self, data: TicketIngestionData, state: Dict) -> Optional[str]: + """Build and send this run's STIX bundle, and revoke any newly-false-positive tickets. + + Returns the new watermark value to persist, or None if there was nothing seen this run + (so run() leaves state untouched). + """ + if not data.tickets: + self.helper.connector_logger.info("No tickets in scope for this run") + return None + + objects = [self.author, self.marking] + rel_map = state.setdefault("ticket_relationship_ids", {}) + org_ip_rel_map = state.setdefault("org_ip_relationship_ids", {}) + + # §10e/§10i: one bad row must not sink the whole run -- built in from the start this + # time, not added after a live crash the way connector D's CIDR loop needed to be. + for ticket in data.tickets: + ticket_id = ticket.get("id") + try: + if ticket.get("false_positive"): + self._revoke_if_known(rel_map, ticket_id) + continue + + # Looked up, not created (§7a) -- connector D owns Identity(class="organization"). + org_id = mapping.resolve_org_identity_id(ticket["organization_name"]) + ip_obj = mapping.map_ip_observable(ticket["ip_string"]) + vuln_obj = mapping.map_vulnerability(ticket) + ticket_relationship = mapping.build_ticket_relationship( + ticket, + ip_obj.id, + vuln_obj.id, + self.author.id, + self.marking.id, + existing_id=rel_map.get(ticket_id), + ) + org_ip_key = f"{org_id}:{ip_obj.id}" + org_ip_relationship = mapping.build_org_owns_ip( + org_id, + ip_obj.id, + self.author.id, + self.marking.id, + existing_id=org_ip_rel_map.get(org_ip_key), + ) + except Exception as e: # pylint: disable=broad-except + self.helper.connector_logger.warning( + f"Skipping ticket {ticket_id!r}: {e}" + ) + continue + + rel_map[ticket_id] = ticket_relationship.id + org_ip_rel_map[org_ip_key] = org_ip_relationship.id + objects.append(ip_obj) + objects.append(vuln_obj) + objects.append(ticket_relationship) + objects.append(org_ip_relationship) + + # §9 queue discipline: a run that only revoked false positives has nothing new to send -- + # don't put an author+marking-only bundle on the queue for no reason. + if len(objects) > 2: + objects = mapping.dedupe_bundle_objects(objects) + bundle = self.helper.stix2_create_bundle(objects) + self.helper.send_stix2_bundle(bundle, update=True) + self.helper.connector_logger.info( + f"Sent {len(objects)} objects ({len(data.tickets)} tickets seen this run)" + ) + else: + self.helper.connector_logger.info( + f"Nothing new to send ({len(data.tickets)} tickets seen this run, " + "all false-positive/skipped)" + ) + + return self._compute_watermark(data.tickets) + + def _revoke_if_known(self, rel_map: Dict, ticket_id: Optional[str]) -> None: + """Revoke a ticket's relationship if we ever sent one, now that it's false_positive. + + §7a: leaving a stale "still vulnerable" edge in the graph is worse than doing nothing, but + a ticket that was *never* ingested (rel_map has no entry) needs no action either way. + Deletion goes through the OpenCTI API directly, not the STIX bundle -- send_stix2_bundle() + upserts by id, it has no delete primitive; this is why revocation needs + `helper.api.stix_core_relationship.delete()` instead (confirmed against the installed + pycti version's stix_core_relationship API entity). + + `ticket_id` is Optional only because `dict.get("id")` types that way -- Ticket.id is a + non-nullable primary key in practice, so a `None` here means a malformed row, not a real + ticket; nothing to revoke either way, same outcome as "never ingested." + """ + if ticket_id is None: + return + rel_id = rel_map.pop(ticket_id, None) + if rel_id is None: + return + self.helper.api.stix_core_relationship.delete(id=rel_id) + self.helper.connector_logger.info( + f"Revoked relationship for ticket {ticket_id!r} (now false_positive)" + ) + + @staticmethod + def _compute_watermark(tickets) -> Optional[str]: + """COALESCE(closed_timestamp, updated_timestamp), computed in Python over every ticket seen. + + Uses every fetched row, not just the ones that produced STIX objects -- skipped and + revoked tickets still need to move the watermark past them, exactly like connector D takes + the max across every organization row regardless of whether it emitted a CIDR. + + Same live-vs-fixture type gap as connector D's watermark (§10i): psycopg2 hands back a + real datetime.datetime for these timestamptz columns, but pycti's set_state() does a plain + json.dumps() with no datetime support. isinstance, not hasattr, for the same mypy + narrowing reason documented in connector D's connector.py. + """ + last_seen_values = [ + mapping.normalize_timestamp(t.get("closed_timestamp")) + or mapping.normalize_timestamp(t.get("updated_timestamp")) + for t in tickets + ] + watermark = max((v for v in last_seen_values if v is not None), default=None) + return ( + watermark.isoformat() + if isinstance(watermark, datetime.datetime) + else watermark + ) diff --git a/open-cti/connectors/vs-ticket-ingestion/src/db.py b/open-cti/connectors/vs-ticket-ingestion/src/db.py new file mode 100644 index 000000000..3a9a7178d --- /dev/null +++ b/open-cti/connectors/vs-ticket-ingestion/src/db.py @@ -0,0 +1,189 @@ +"""mini_data_lake access for the VS Ticket Ingestion connector. + +Same two-implementation shape as connector D's db.py (real Postgres + IS_LOCAL fixtures, see +OpenCTI-connector.md §9a) -- kept structurally identical rather than reinvented, per §10i. + +Table/column names taken from backend/src/xfd_django/xfd_mini_dl/models.py -- see +OpenCTI-connector.md §7a. `Ticket` (L2846) has `db_table = "ticket"` and its `organization` +FK column is `organization_id` (Django's default naming, no explicit db_column override -- +confirmed by reading the model, not guessed). + +Deliberate design choice, applying §10i's uuid-cast lesson before it could bite here too: +`Ticket.organization_id` is a native Postgres `uuid` column, same type as `Location.id` was in +connector D (where an unguarded `= ANY(%(list)s)` needed an explicit `::uuid[]` cast). Rather than +pull ticket rows first and then look up organizations by a Python list of ids the way connector D's +`_fetch_locations_live` did, this joins `ticket` to `organization` directly in one query and scopes +by `organization.acronym` (varchar) instead -- sidesteps the uuid/text[] type-adaptation gotcha +entirely rather than needing a cast to work around it. Verified against a real throwaway +`postgres:17` container before ever touching the live box (OpenCTI-connector.md §9b Loop 4). +""" + +# Standard Python Libraries +from dataclasses import dataclass, field +import json +import logging +import os +from typing import Dict, List, Optional + +# Third-Party Libraries +import boto3 +import psycopg2 +import psycopg2.extras + +from .config import Config + +LOGGER = logging.getLogger("vs_ticket_ingestion.db") + + +@dataclass +class TicketIngestionData: + """Everything one connector run needs, regardless of where it came from.""" + + tickets: List[Dict] = field(default_factory=list) + + +def _generate_iam_auth_token(config: Config) -> str: + """Generate a short-lived RDS IAM auth token. Same pattern as connector D -- see its db.py.""" + client = boto3.client("rds", region_name=config.aws_region) + return client.generate_db_auth_token( + DBHostname=config.db_host, + Port=config.db_port, + DBUsername=config.db_user, + ) + + +def get_connection(config: Config): + """Open a psycopg2 connection to mini_data_lake, IAM-auth or password per config.""" + password = ( + _generate_iam_auth_token(config) + if config.db_use_iam_auth + else config.db_password + ) + return psycopg2.connect( + host=config.db_host, + port=config.db_port, + dbname=config.db_name, + user=config.db_user, + password=password, + sslmode="require", # IAM auth requires TLS regardless of rds.force_ssl -- §2 + cursor_factory=psycopg2.extras.RealDictCursor, + connect_timeout=10, + ) + + +class VsTicketIngestionRepository: + """Fetches ticket data, scoped and capped per config (§9c).""" + + def __init__(self, config: Config): + """Hold the config; connections are opened per-call, not cached (§10d).""" + self.config = config + + def fetch( + self, since_last_seen: Optional[str], include_stale_open: bool = False + ) -> TicketIngestionData: + """Fetch one run's worth of data, from fixtures (IS_LOCAL) or mini_data_lake. + + `include_stale_open` is the bootstrap-only safety net (see connector.py's `run()` for + where it's computed) -- see `_fetch_tickets_live` for why it exists. + """ + if self.config.is_local: + return self._fetch_local() + return self._fetch_live(since_last_seen, include_stale_open) + + # ------------------------------------------------------------------ + # Live path + # ------------------------------------------------------------------ + + def _fetch_live( + self, since_last_seen: Optional[str], include_stale_open: bool + ) -> TicketIngestionData: + conn = get_connection(self.config) + try: + return TicketIngestionData( + tickets=self._fetch_tickets_live( + conn, since_last_seen, include_stale_open + ) + ) + finally: + conn.close() + + def _fetch_tickets_live( + self, conn, since_last_seen: Optional[str], include_stale_open: bool + ) -> List[Dict]: + # Explicit column projection (§10d) -- only what mapping.py actually maps to STIX. + # + # Incremental strategy (§7a): COALESCE(closed_timestamp, updated_timestamp), already + # indexed as ticket_last_seen_idx (models.py ~L3044) -- a ticket flipping open->closed, + # or not-false-positive->false-positive, shows up naturally as an "updated" row on the + # next poll without any separate closure-detection logic. + # + # Deliberately NOT filtering false_positive=True out of the query. §7a: a + # previously-ingested ticket that flips to false_positive=True needs its relationship + # revoked (connector.py), not silently dropped -- filtering it out in SQL would mean the + # connector never sees that row again after the flip and could never detect the reversal. + # + # (%(acronyms)s IS NULL OR ...) guard built in from the start, not added after a live + # "returned zero rows" surprise -- that's what happened building connector D + # (OpenCTI-connector.md §10i) with the exact same acronym = ANY(NULL) semantics. + # + # `include_stale_open` -- real production finding, not a hypothetical (2026-08-28): + # `VS_TICKET_INGESTION_LOOKBACK_DAYS` only bounds the very first query (no watermark + # yet); every query after that filters on `> watermark`, which only ever advances + # forward. A ticket whose last-touch predates the lookback cutoff at the moment of that + # first query is therefore not delayed -- it's permanently excluded, since the watermark + # can never move backward to re-examine it. Measured directly against the real + # cyhy_mini_data_lake_staging box: 413,925 tickets were `is_open = true` with a last-touch + # older than a 60-day lookback -- real, currently-actionable findings a plain lookback + # would have silently dropped forever, not "eventually" picked up. This flag, true only + # on the bootstrap run (state.get("last_seen_watermark") is None -- see connector.py), + # adds every currently-open ticket to that first query's candidate set regardless of how + # stale its last-touch date is, while leaving every later, steady-state poll unchanged. + # `t.is_open IS TRUE` (not `= true`) is deliberate: is_open is a nullable BooleanField, + # and a NULL must not count as "open" here. Verified against a real postgres:17 container + # (§9b Loop 4) before this touched the live box: bootstrap mode correctly includes a + # stale-but-open row and a recent row, correctly excludes a stale-*closed* row (no value + # ingesting a long-dead ticket) and a stale row with is_open IS NULL; with the flag off, + # behavior is byte-for-byte identical to the query before this fix existed. + query = """ + SELECT t.id, t.cve_string, t.vuln_name, t.service_name, t.vuln_source, + t.ip_string, t.opened_timestamp, t.closed_timestamp, t.updated_timestamp, + t.is_open, t.is_kev, t.is_kev_ransomware, t.is_risky, t.false_positive, + t.cvss_severity, t.vuln_port, t.port_protocol, + o.acronym AS organization_acronym, o.name AS organization_name + FROM ticket t + JOIN organization o ON o.id = t.organization_id + WHERE (%(acronyms)s IS NULL OR o.acronym = ANY(%(acronyms)s)) + AND ( + %(since)s IS NULL + OR COALESCE(t.closed_timestamp, t.updated_timestamp) > %(since)s + OR (%(include_stale_open)s AND t.is_open IS TRUE) + ) + ORDER BY COALESCE(t.closed_timestamp, t.updated_timestamp) + LIMIT %(limit)s + """ + with conn.cursor() as cur: + cur.execute( + query, + { + "acronyms": self.config.org_acronym_allowlist or None, + "since": since_last_seen, + "include_stale_open": include_stale_open, + "limit": self.config.max_rows_per_run, + }, + ) + return [dict(row) for row in cur.fetchall()] + + # ------------------------------------------------------------------ + # IS_LOCAL fixture path (§9a) -- same shape as the live path, zero DB/network. + # ------------------------------------------------------------------ + + def _fetch_local(self) -> TicketIngestionData: + directory = self.config.local_fixture_dir + LOGGER.info("IS_LOCAL=true -- loading fixtures from %s", directory) + return TicketIngestionData(tickets=self._load_json(directory, "tickets.json")) + + @staticmethod + def _load_json(directory: str, filename: str): + path = os.path.join(directory, filename) + with open(path, encoding="utf-8") as f: + return json.load(f) diff --git a/open-cti/connectors/vs-ticket-ingestion/src/main.py b/open-cti/connectors/vs-ticket-ingestion/src/main.py new file mode 100644 index 000000000..4282520c9 --- /dev/null +++ b/open-cti/connectors/vs-ticket-ingestion/src/main.py @@ -0,0 +1,27 @@ +"""Entry point. See OpenCTI-connector.md §7a for design, README.md for how to run this locally.""" + +# Standard Python Libraries +import sys +import time + +from .config import ConfigError +from .connector import VsTicketIngestionConnector + +if __name__ == "__main__": + try: + connector = VsTicketIngestionConnector() + except ConfigError as e: + # Fail closed and loud (§10c/§9c) -- not a stack trace to dig through. + print(f"[vs-ticket-ingestion] refusing to start: {e}", file=sys.stderr) + sys.exit(1) + + try: + connector.helper.schedule_iso( + message_callback=connector.run, + duration_period=connector.config.duration_period, + ) + except KeyboardInterrupt: + connector.helper.connector_logger.info("Stopping VS Ticket Ingestion connector") + connector.helper.stop() + time.sleep(1) + sys.exit(0) diff --git a/open-cti/connectors/vs-ticket-ingestion/src/mapping.py b/open-cti/connectors/vs-ticket-ingestion/src/mapping.py new file mode 100644 index 000000000..b83fcf4e5 --- /dev/null +++ b/open-cti/connectors/vs-ticket-ingestion/src/mapping.py @@ -0,0 +1,325 @@ +"""Pure row -> STIX mapping functions for the VS Ticket Ingestion connector. + +Deliberately dependency-free beyond stix2/pycti and the standard library -- no DB, no OpenCTI +API, no network. This is "Loop 1" from OpenCTI-connector.md §9b. See tests/test_mapping.py. + +STIX mapping decisions implemented here are documented in OpenCTI-connector.md §7a; the +idempotency pattern (why the relationship builder takes an `existing_id` param) is §10a, and +§10i (added after connector D) is why the live-vs-fixture type gaps called out below were +checked against a real Postgres container before this ever ran against the live box. +""" + +# Standard Python Libraries +import datetime +from ipaddress import ip_address +from typing import Dict, List, Optional + +# Third-Party Libraries +from dateutil import parser as dateutil_parser +from pycti import Identity as OpenCTIIdentity +from pycti import StixCoreRelationship +from pycti import Vulnerability as OpenCTIVulnerability +import stix2 + +VS_EXTERNAL_SOURCE = "VS" + + +def build_author(name: str) -> stix2.Identity: + """Build the single system Identity every VS-sourced object should cite as createdBy (§10c). + + Identical construction to connector D's build_author() -- same name, same identity_class -- + so both connectors' Identity(class="system") objects resolve to the exact same content-derived + id (OpenCTIIdentity.generate_id() is deterministic on (name, identity_class)) and upsert onto + one shared author Identity in OpenCTI, not two separate ones. + """ + return stix2.Identity( + id=OpenCTIIdentity.generate_id(name, "system"), + name=name, + identity_class="system", + ) + + +def resolve_org_identity_id(organization_name: str) -> str: + """Resolve an organization's existing Identity id -- look up, don't blindly create (§7a). + + Connector D (§7d) is what actually creates/maintains Identity(class="organization") objects. + This connector never emits one of its own; it only needs to know what id to point + relationships at. Identity's standard_id is deterministic, content-derived from + (name, identity_class) via OpenCTIIdentity.generate_id() -- the same call connector D's + mapping.map_organization() makes -- so recomputing it here reaches the identical id without + a GraphQL round-trip per ticket. That's the point: at ticket volume (thousands of rows per + poll, §7a) a live API lookup per row would be real, avoidable load (§10d's discipline + generalizes to the OpenCTI API, not just mini_data_lake). The tradeoff to keep in mind: this + assumes `organization.name` hasn't changed between connector D's last sync and this row's + join (both read the same column, just at different times) -- a real but narrow edge case, + not verified here structurally, only by both connectors reading the same source of truth. + """ + return OpenCTIIdentity.generate_id(organization_name, "organization") + + +def map_ip_observable(ip_string: str): + """IP string -> IPv4Address/IPv6Address SCO. + + Ticket.ip_string is a single scanned host's address (models.py: "IP address of the host that + was vulnerability scanned"), not a CIDR block like connector D's Cidr.network -- ip_address(), + not ip_network(). Raises ValueError on anything malformed; left to propagate so connector.py's + per-row try/except (§10i) can skip just that one ticket rather than the whole run. + """ + parsed = ip_address(ip_string) + if parsed.version == 4: + return stix2.IPv4Address(value=ip_string) + return stix2.IPv6Address(value=ip_string) + + +def severity_label(cvss_severity) -> Optional[str]: + """Bucket a CVSS severity score into the standard low/medium/high/critical qualitative scale. + + Ticket.cvss_severity is a DecimalField, not a string bucket -- psycopg2 hands back a real + decimal.Decimal on a live run (verified against a real postgres:17 container, §9b Loop 4, + before this ever touched the live box), not a plain float the way an IS_LOCAL JSON fixture + would represent it. Decimal supports direct comparison against float literals in Python (also + verified), so no explicit cast is required here -- just documented, since it's exactly the + kind of live-vs-fixture type gap §10i calls out as a recurring bug class. Returns None for + unscored/zero tickets rather than a "none" label -- the bucket only means something once a + ticket actually has a score. + """ + if cvss_severity is None: + return None + try: + score = float(cvss_severity) + except (TypeError, ValueError): + return None + if score <= 0: + return None + if score < 4.0: + return "vs-severity-low" + if score < 7.0: + return "vs-severity-medium" + if score < 9.0: + return "vs-severity-high" + return "vs-severity-critical" + + +def map_vulnerability(ticket: Dict) -> stix2.Vulnerability: + """Ticket row -> minimal Vulnerability SDO. + + Deliberately minimal when cve_string is present -- name only, no CVSS/description of our own + (§7a). connector-cve (docker-compose.yml) already runs continuously against NVD and almost + certainly already has (or will shortly have) a Vulnerability for that CVE; a bare-minimum + upsert here resolves to the same content-derived id rather than forking a second, VS-flavored + copy of the same CVE's data. + + For CVE-less tickets (nmap-sourced risky-service tickets, vuln_source="nmap") there's no + connector-cve equivalent to defer to, so build a real Vulnerability named from vuln_name or + service_name instead of skipping the ticket -- STIX Vulnerability doesn't require a CVE id. + Raises ValueError when neither cve_string, vuln_name, nor service_name is usable; left to + propagate to connector.py's per-row isolation rather than fabricating a placeholder name. + """ + cve_string = (ticket.get("cve_string") or "").strip() + if cve_string: + name = cve_string.upper() + else: + name = (ticket.get("vuln_name") or ticket.get("service_name") or "").strip() + if not name: + raise ValueError( + f"Ticket {ticket.get('id')!r} has no cve_string, vuln_name, or service_name to " + "name a Vulnerability from" + ) + return stix2.Vulnerability( + id=OpenCTIVulnerability.generate_id(name), + name=name, + ) + + +def normalize_timestamp(value): + """Normalize a DateTimeField value to something stix2's TimestampProperty accepts. + + Ticket.opened_timestamp/closed_timestamp/updated_timestamp are all DateTimeField + (timestamptz), unlike connector D's CidrOrgs.first_seen/last_seen (DateField, date-only) -- + so the date-only precision-loss risk mostly doesn't apply here. Still needed because + IS_LOCAL fixtures store these as plain ISO strings, not live psycopg2 datetimes. + + Deliberately more careful than connector D's same-named helper about *which* string formats + it hands back to stix2 unmodified: found by actually running this connector's own test suite + (§9b Loop 1) that stix2's TimestampProperty rejects a `+00:00`-offset ISO string outright + ("must be a datetime object, date object, or timestamp string in a recognizable format") while + accepting the exact same instant written as `...Z` or as a real datetime object. Connector D + never hit this because none of its date-only CidrOrgs fixtures ever exercised that passthrough + branch with an offset suffix -- but any fixture author writing `datetime.isoformat()`-style + strings (which produce `+00:00`, not `Z`) for *this* connector's DateTimeField fixtures would. + Parsing every string through dateutil (already a pinned dependency, §7a) and always returning + a real datetime object -- never a bare string -- closes the gap for every ISO variant at once + instead of special-casing `Z`. + """ + if value is None: + return None + if isinstance(value, datetime.datetime): + return value + if isinstance(value, datetime.date): + return datetime.datetime( + value.year, value.month, value.day, tzinfo=datetime.timezone.utc + ) + if isinstance(value, str): + parsed = dateutil_parser.isoparse(value) + if parsed.tzinfo is None: + parsed = parsed.replace(tzinfo=datetime.timezone.utc) + return parsed + return value + + +def _relationship_id( + relationship_type: str, + source_ref: str, + target_ref: str, + start_time, + stop_time, + existing_id: Optional[str], +) -> str: + """Apply the §10a idempotency pattern -- identical reasoning to connector D's mapping.py. + + Recomputing via generate_id() every time hashes in start_time/stop_time, so it would produce + a *different* id the moment a ticket closes (stop_time going from unset to set). Only the + first sighting of a given Ticket.id computes a fresh id; every later write reuses the id + connector.py already recorded in state (keyed directly by Ticket.id -- simpler than connector + D's composite CIDR key, since Ticket.id is already a durable, unique external key on its own). + """ + if existing_id: + return existing_id + return StixCoreRelationship.generate_id( + relationship_type, source_ref, target_ref, start_time, stop_time + ) + + +def build_org_owns_ip( + org_id: str, + ip_observable_id: str, + author_id: str, + marking_id: str, + existing_id: Optional[str] = None, +) -> stix2.Relationship: + """Org -> IP observable, so a ticket's vulnerability is actually reachable from its org. + + Without this, resolve_org_identity_id()'s result would be computed and never used -- + §7a lists organization resolution as part of this connector's per-ticket mapping, and the + only way that's meaningful is a relationship linking the org into the graph, alongside + connector D's own org->CIDR edges. Not time-scoped (no start/stop) since it isn't tracking a + lifecycle the way the ticket's own relationship does -- just "this org is associated with + this host," which multiple tickets against the same host should collapse onto one shared + edge, not one per ticket. Same idempotency pattern as every relationship builder in this file. + """ + rel_id = _relationship_id( + "related-to", org_id, ip_observable_id, None, None, existing_id + ) + return stix2.Relationship( + id=rel_id, + relationship_type="related-to", + source_ref=org_id, + target_ref=ip_observable_id, + created_by_ref=author_id, + object_marking_refs=[marking_id], + ) + + +def build_ticket_relationship( + ticket: Dict, + ip_observable_id: str, + vulnerability_id: str, + author_id: str, + marking_id: str, + existing_id: Optional[str] = None, +) -> stix2.Relationship: + """Build the IP-observable -> Vulnerability relationship carrying a ticket's lifecycle. + + start_time/stop_time = opened_timestamp/closed_timestamp (stop_time omitted while open), + native SRO fields rather than a separate container object (§7a's Case/Incident decision). + Severity/kev/risky/source become labels; an External Reference records Ticket.id for our own + bookkeeping only -- per §10a it plays no role in OpenCTI's own dedup, which is why the id + itself is pinned via `existing_id` instead. + + `updated_timestamp` (Ticket's own "last updated" column, and the value this connector's + watermark is COALESCE'd from -- see connector.py/db.py) is carried onto this relationship as + `x_opencti_updated_timestamp`, a plain custom property rather than the SRO's native + `modified`. Deliberately not `modified`: OpenCTI/stix2 give that field real merge/versioning + semantics of their own, and a bootstrap run can hand back tickets whose `updated_timestamp` is + genuinely in the past relative to what's already stored -- letting that drive `modified` risks + the platform quietly treating an old bootstrap row as stale and skipping the write. A plain + custom property carries the same information with none of that risk (same reasoning already + proven safe for connector C's `x_opencti_*` fields -- verified there that custom properties + don't feed a STIX object's id the way `start_time`/`stop_time` do, and it's doubly true here + since this relationship's id is always explicitly pinned, never derived from its own content). + Deliberately NOT put on the Vulnerability SDO itself -- that object is intentionally minimal + and shared/deduped across every ticket and org that references the same CVE (see + `map_vulnerability()`'s docstring); stamping one ticket's update time onto it would let + whichever connector run happened last overwrite a fact that has nothing to do with that CVE + itself. This relationship, scoped to one specific (ip, vulnerability) pair, is the right home + for it -- the same place opened_timestamp/closed_timestamp already live. + + Same defensive stop_time<=start_time guard as connector D's build_owns_cidr, for the same + reason: STIX 2.1 requires strictly-later stop_time (verified against the installed stix2 + library -- even equal values raise ValueError), and while Ticket's timestamps are full + DateTimeTime precision (unlike connector D's date-only CidrOrgs columns), an opened-and-closed + -in-the-same-write-batch ticket is a real, if rarer, possibility -- not something to find out + about for the first time at full production scale again (§10i). + """ + start_time = normalize_timestamp(ticket.get("opened_timestamp")) + stop_time = normalize_timestamp(ticket.get("closed_timestamp")) + if start_time is not None and stop_time is not None and stop_time <= start_time: + stop_time = None + + custom_properties: Dict[str, object] = {} + updated_timestamp = normalize_timestamp(ticket.get("updated_timestamp")) + if updated_timestamp is not None: + custom_properties["x_opencti_updated_timestamp"] = updated_timestamp + + labels: List[str] = [] + if ticket.get("vuln_source"): + labels.append(f"vs-source-{ticket['vuln_source']}") + if ticket.get("is_kev"): + labels.append("vs-kev") + if ticket.get("is_kev_ransomware"): + labels.append("vs-kev-ransomware") + if ticket.get("is_risky"): + labels.append("vs-risky") + severity = severity_label(ticket.get("cvss_severity")) + if severity: + labels.append(severity) + + rel_id = _relationship_id( + "related-to", + ip_observable_id, + vulnerability_id, + start_time, + stop_time, + existing_id, + ) + return stix2.Relationship( + id=rel_id, + relationship_type="related-to", + source_ref=ip_observable_id, + target_ref=vulnerability_id, + start_time=start_time, + stop_time=stop_time, + labels=labels or None, + created_by_ref=author_id, + object_marking_refs=[marking_id], + external_references=[ + stix2.ExternalReference( + source_name=VS_EXTERNAL_SOURCE, external_id=ticket["id"] + ) + ], + custom_properties=custom_properties, + ) + + +def dedupe_bundle_objects(objects: List) -> List: + """Collapse a list of STIX objects to one per id, keeping the last occurrence of each. + + Same rationale as connector D: many tickets against the same host/CVE repeat the same + IP/Vulnerability object across a run (e.g. two tickets, same host, two different CVEs still + share the IP observable) -- collapse before sending, both for bundle size (§10b) and because + stix2.Bundle construction is order-sensitive about duplicate ids. + """ + seen = {} + for obj in objects: + seen[obj["id"]] = obj + return list(seen.values()) diff --git a/open-cti/connectors/vs-ticket-ingestion/tests/__init__.py b/open-cti/connectors/vs-ticket-ingestion/tests/__init__.py new file mode 100644 index 000000000..fca1c80bf --- /dev/null +++ b/open-cti/connectors/vs-ticket-ingestion/tests/__init__.py @@ -0,0 +1 @@ +"""Tests for the VS Ticket Ingestion connector -- see README.md for the two loops.""" diff --git a/open-cti/connectors/vs-ticket-ingestion/tests/fixtures/tickets.json b/open-cti/connectors/vs-ticket-ingestion/tests/fixtures/tickets.json new file mode 100644 index 000000000..3cf909adf --- /dev/null +++ b/open-cti/connectors/vs-ticket-ingestion/tests/fixtures/tickets.json @@ -0,0 +1,86 @@ +[ + { + "closed_timestamp": null, + "cve_string": "cve-2024-1234", + "cvss_severity": 9.8, + "false_positive": false, + "id": "cyhy-ticket-0001", + "ip_string": "198.51.100.5", + "is_kev": true, + "is_kev_ransomware": false, + "is_open": true, + "is_risky": false, + "opened_timestamp": "2026-08-01T12:00:00+00:00", + "organization_acronym": "TESTORG1", + "organization_name": "Test Organization One", + "port_protocol": "tcp", + "service_name": null, + "updated_timestamp": "2026-08-20T09:00:00+00:00", + "vuln_name": null, + "vuln_port": 443, + "vuln_source": "nessus" + }, + { + "closed_timestamp": "2026-08-21T09:30:00+00:00", + "cve_string": "CVE-2023-9999", + "cvss_severity": 5.4, + "false_positive": false, + "id": "cyhy-ticket-0002", + "ip_string": "198.51.100.6", + "is_kev": false, + "is_kev_ransomware": false, + "is_open": false, + "is_risky": false, + "opened_timestamp": "2026-06-01T00:00:00+00:00", + "organization_acronym": "TESTORG1", + "organization_name": "Test Organization One", + "port_protocol": "tcp", + "service_name": null, + "updated_timestamp": "2026-08-21T09:30:00+00:00", + "vuln_name": null, + "vuln_port": 8443, + "vuln_source": "nessus" + }, + { + "closed_timestamp": null, + "cve_string": null, + "cvss_severity": null, + "false_positive": false, + "id": "cyhy-ticket-0003", + "ip_string": "2001:db8::5", + "is_kev": false, + "is_kev_ransomware": false, + "is_open": true, + "is_risky": true, + "opened_timestamp": "2026-08-15T00:00:00+00:00", + "organization_acronym": "TESTORG2-CHILD", + "organization_name": "Test Organization Two (Child)", + "port_protocol": "tcp", + "service_name": "telnet", + "updated_timestamp": "2026-08-19T00:00:00+00:00", + "vuln_name": null, + "vuln_port": 23, + "vuln_source": "nmap" + }, + { + "closed_timestamp": null, + "cve_string": "CVE-2022-5555", + "cvss_severity": 4.2, + "false_positive": true, + "id": "cyhy-ticket-0004", + "ip_string": "198.51.100.7", + "is_kev": false, + "is_kev_ransomware": false, + "is_open": true, + "is_risky": false, + "opened_timestamp": "2026-07-01T00:00:00+00:00", + "organization_acronym": "TESTORG1", + "organization_name": "Test Organization One", + "port_protocol": "tcp", + "service_name": null, + "updated_timestamp": "2026-08-22T00:00:00+00:00", + "vuln_name": null, + "vuln_port": 22, + "vuln_source": "nessus" + } +] diff --git a/open-cti/connectors/vs-ticket-ingestion/tests/test_connector_dry_run.py b/open-cti/connectors/vs-ticket-ingestion/tests/test_connector_dry_run.py new file mode 100644 index 000000000..7fc86669a --- /dev/null +++ b/open-cti/connectors/vs-ticket-ingestion/tests/test_connector_dry_run.py @@ -0,0 +1,426 @@ +"""Loop 2 (OpenCTI-connector.md §9b): the real poll->map->bundle pipeline against fixtures. + +Uses a stub in place of OpenCTIConnectorHelper so no network/queue is touched at all. Also proves +the false-positive revocation path (§7a) and the §9 queue-discipline rule (a revocation-only run +sends no bundle) -- both built in from the start per §10i, not discovered by a live crash. +""" + +# Standard Python Libraries +import copy +import datetime +import json +import os +import sys + +sys.path.insert(0, os.path.join(os.path.dirname(__file__), "..")) + +# Third-Party Libraries +from pycti.utils.opencti_logger import logger as pycti_logger + +# First-Party +from src.config import Config # noqa: E402 +from src.connector import VsTicketIngestionConnector # noqa: E402 + + +class _StubRelationshipApi: + """Stands in for helper.api.stix_core_relationship. + + Captures delete() calls instead of actually calling the OpenCTI GraphQL API. + """ + + def __init__(self): + """Start with an empty capture list -- nothing has been "deleted" yet.""" + self.deleted_ids = [] + + def delete(self, **kwargs): + """Capture a revocation the way pycti's real API entity would apply it.""" + self.deleted_ids.append(kwargs.get("id")) + + +class _StubApi: + """Just enough of helper.api's surface for the revocation path (§7a) to run against.""" + + def __init__(self): + """Build the one sub-API this connector actually touches.""" + self.stix_core_relationship = _StubRelationshipApi() + + +class StubHelper: + """Just enough of OpenCTIConnectorHelper's surface for _process() to run against. + + connector_logger deliberately uses pycti's *real* AppLogger, not a plain stdlib + logging.getLogger() -- connector D's dry-run tests only caught its two AppLogger-signature + bugs (§10i) once they switched to this; using it from the start here means those bug classes + can't slip through un-caught in the first place. + """ + + def __init__(self): + """Start with empty capture state -- nothing sent or deleted yet.""" + self.connector_logger = pycti_logger(level=20, json_logging=False)( + "stub_helper" + ) + self.sent_bundles = [] + self.api = _StubApi() + self._state = {} + + def get_state(self): + """Return the current stored state, mirroring the real helper's equivalent method.""" + return self._state + + def set_state(self, state): + """Persist state, mirroring the real helper's equivalent method.""" + self._state = state + + @staticmethod + def stix2_create_bundle(items): + """Delegate to pycti's real bundle serialization rather than reimplementing it.""" + # Third-Party Libraries + from pycti import OpenCTIConnectorHelper + + return OpenCTIConnectorHelper.stix2_create_bundle(items) + + def send_stix2_bundle(self, bundle, **kwargs): + """Capture the bundle instead of sending it -- this *is* the dry-run boundary.""" + self.sent_bundles.append(bundle) + return [bundle] + + +def build_test_connector(): + """Build a VsTicketIngestionConnector wired to fixtures and a StubHelper, for the tests below.""" + config = Config( + raw={ + "vs_ticket_ingestion": { + "org_acronym_allowlist": "TESTORG1,TESTORG2-CHILD", + "is_local": True, + "local_fixture_dir": os.path.join( + os.path.dirname(__file__), "fixtures" + ), + "tlp_marking": "TLP:AMBER", + } + } + ) + return VsTicketIngestionConnector(config=config, helper=StubHelper()) + + +def test_first_run_produces_expected_objects(): + """A single run against the fixtures should produce exactly the objects §7a's design expects.""" + connector = build_test_connector() + data = connector.repository.fetch(since_last_seen=None) + state = {} + + watermark = connector._process(data, state) + + # Max COALESCE(closed_timestamp, updated_timestamp) across ALL fetched tickets, including + # the false-positive one -- it still needs to move the watermark past it (§7a/§10i). + assert watermark == "2026-08-22T00:00:00+00:00" + assert len(connector.helper.sent_bundles) == 1 + + bundle = json.loads(connector.helper.sent_bundles[0]) + types = [obj["type"] for obj in bundle["objects"]] + assert ( + types.count("identity") == 1 + ) # author only -- org Identity is looked up, not created + assert types.count("marking-definition") == 1 + assert types.count("ipv4-addr") == 2 # 198.51.100.5, 198.51.100.6 + assert types.count("ipv6-addr") == 1 # 2001:db8::5 + assert ( + types.count("vulnerability") == 3 + ) # 2 CVEs + 1 nmap/service_name fallback ("telnet") + # 3 ticket relationships (the false-positive ticket contributes none) + 3 org->ip + # relationships (TESTORG1 x2 distinct ips, TESTORG2-CHILD x1) + assert types.count("relationship") == 6 + + # The closed ticket's relationship should carry a stop_time; the two still-open ones shouldn't. + ticket_rels = [ + obj + for obj in bundle["objects"] + if obj["type"] == "relationship" + and obj.get("target_ref", "").startswith("vulnerability--") + ] + assert len(ticket_rels) == 3 + assert sum(1 for rel in ticket_rels if "stop_time" in rel) == 1 + # updated_timestamp must survive fixture -> mapping -> real bundle serialization for every + # ticket relationship, not just in the unit-level mapping.py tests. + assert all("x_opencti_updated_timestamp" in rel for rel in ticket_rels) + + +def test_false_positive_ticket_is_never_sent_and_leaves_no_relationship_recorded(): + """A ticket that's already false_positive=True on first sight is a no-op, not an error.""" + connector = build_test_connector() + data = connector.repository.fetch(since_last_seen=None) + state = {} + + connector._process(data, state) + + assert "cyhy-ticket-0004" not in state["ticket_relationship_ids"] + assert ( + connector.helper.api.stix_core_relationship.deleted_ids == [] + ) # never ingested, nothing to revoke + + +def test_ticket_flipping_to_false_positive_on_a_later_poll_gets_revoked(): + """A previously-ingested ticket that flips false_positive=True must get revoked. + + §7a: the relationship needs to be actually removed, not just silently dropped from future + bundles. + """ + connector = build_test_connector() + data = connector.repository.fetch(since_last_seen=None) + # Make ticket 0004 a normal, real ticket for the first run, so it actually gets ingested. + ticket_4 = next(t for t in data.tickets if t["id"] == "cyhy-ticket-0004") + ticket_4["false_positive"] = False + state = {} + + connector._process(data, state) + assert "cyhy-ticket-0004" in state["ticket_relationship_ids"] + recorded_rel_id = state["ticket_relationship_ids"]["cyhy-ticket-0004"] + + # Now it flips to false_positive=True on the next poll. + ticket_4["false_positive"] = True + connector._process(data, state) + + assert "cyhy-ticket-0004" not in state["ticket_relationship_ids"] + assert connector.helper.api.stix_core_relationship.deleted_ids == [recorded_rel_id] + + +def test_revocation_only_run_sends_no_bundle(): + """A run whose only work is revoking a false positive must send no bundle at all. + + §9 queue discipline: nothing new to add to the graph shouldn't put an author+marking-only + bundle on the queue. + """ + # Third-Party Libraries + from src.db import TicketIngestionData + + connector = build_test_connector() + data = connector.repository.fetch(since_last_seen=None) + ticket_4 = next(t for t in data.tickets if t["id"] == "cyhy-ticket-0004") + ticket_4["false_positive"] = False + state = {} + connector._process(data, state) + sent_before = len(connector.helper.sent_bundles) + + # A later poll where the only ticket in scope this run is 0004, now flipped -- nothing new + # needs to go out, only a revocation via the API. + ticket_4_flipped = copy.deepcopy(ticket_4) + ticket_4_flipped["false_positive"] = True + connector._process(TicketIngestionData(tickets=[ticket_4_flipped]), state) + + assert len(connector.helper.sent_bundles) == sent_before # no new bundle sent + assert "cyhy-ticket-0004" not in state["ticket_relationship_ids"] + + +def test_relationship_ids_are_stable_across_consecutive_runs(): + """Prove the §10a claim executably: re-running against unchanged data must be a no-op.""" + connector = build_test_connector() + data = connector.repository.fetch(since_last_seen=None) + + state = {} + connector._process(data, state) + first_run_ticket_rel_ids = dict(state["ticket_relationship_ids"]) + first_run_org_ip_rel_ids = dict(state["org_ip_relationship_ids"]) + + connector._process(data, state) + + assert state["ticket_relationship_ids"] == first_run_ticket_rel_ids + assert state["org_ip_relationship_ids"] == first_run_org_ip_rel_ids + + +def test_ticket_relationship_id_stays_pinned_when_ticket_closes(): + """Same §10a contract connector D proved for CIDR retirement, here for ticket closure.""" + connector = build_test_connector() + data = connector.repository.fetch(since_last_seen=None) + open_ticket = next(t for t in data.tickets if t["id"] == "cyhy-ticket-0001") + + state = {} + connector._process(data, state) + open_id = state["ticket_relationship_ids"]["cyhy-ticket-0001"] + + # Simulate the ticket closing on the next poll. + open_ticket["closed_timestamp"] = "2026-08-25T00:00:00+00:00" + open_ticket["is_open"] = False + connector._process(data, state) + + closed_id = state["ticket_relationship_ids"]["cyhy-ticket-0001"] + assert closed_id == open_id # pinned, as designed + + bundle = json.loads(connector.helper.sent_bundles[-1]) + rel_obj = next(obj for obj in bundle["objects"] if obj.get("id") == closed_id) + assert ( + "stop_time" in rel_obj + ) # ...and the content that changed rides along on that same id + + +def test_process_watermark_is_json_serializable_for_real_datetime_objects(): + """Prove the watermark fix against what psycopg2 actually returns, not just fixture strings. + + Ticket.updated_timestamp/closed_timestamp are timestamptz columns -- psycopg2 hands back real + datetime.datetime objects on a live run, but every fixture here stores them as already-JSON + -safe ISO strings. Override with a real datetime the way live data actually looks (same class + of bug connector D hit -- "Object of type datetime is not JSON serializable" -- caught here + from day one instead of at full production scale). + """ + connector = build_test_connector() + data = connector.repository.fetch(since_last_seen=None) + for ticket in data.tickets: + ticket["updated_timestamp"] = datetime.datetime( + 2026, 8, 22, 0, 0, tzinfo=datetime.timezone.utc + ) + ticket["closed_timestamp"] = None + + watermark = connector._process(data, {}) + + assert isinstance(watermark, str) + json.dumps({"last_seen_watermark": watermark}) # must not raise + + +def test_one_bad_ticket_does_not_abort_the_whole_run(): + """A malformed row must not sink every other ticket in the same run. + + §10e/§10i row-isolation, built in from the start this time -- proven the same way + connector D's regression was. + """ + connector = build_test_connector() + data = connector.repository.fetch(since_last_seen=None) + data.tickets.append( + { + "id": "cyhy-ticket-bad", + "cve_string": "CVE-2024-0000", + "vuln_name": None, + "service_name": None, + "vuln_source": "nessus", + "ip_string": "not-a-valid-ip", + "opened_timestamp": "2026-08-01T00:00:00+00:00", + "closed_timestamp": None, + "updated_timestamp": "2026-08-23T00:00:00+00:00", + "is_open": True, + "is_kev": False, + "is_kev_ransomware": False, + "is_risky": False, + "false_positive": False, + "cvss_severity": None, + "vuln_port": None, + "port_protocol": None, + "organization_acronym": "TESTORG1", + "organization_name": "Test Organization One", + } + ) + + watermark = connector._process(data, {}) + + assert watermark is not None # the run completed, it didn't raise + bundle = json.loads(connector.helper.sent_bundles[-1]) + ips_sent = [ + obj["value"] + for obj in bundle["objects"] + if obj["type"] in ("ipv4-addr", "ipv6-addr") + ] + assert "not-a-valid-ip" not in ips_sent # the bad one was skipped + assert "198.51.100.5" in ips_sent # the good ones still went out + + +def test_effective_since_is_unbounded_by_default_with_no_watermark(): + """§9c's lookback lever defaults to off -- unchanged behavior from before it existed.""" + connector = build_test_connector() + assert connector.config.lookback_days is None + + assert connector._effective_since({}) is None + + +def test_effective_since_bounds_the_first_poll_when_lookback_days_is_set(): + """With no watermark yet, lookback_days bounds the poll to roughly now minus N days.""" + connector = build_test_connector() + connector.config.lookback_days = 7 + + before = datetime.datetime.now(datetime.timezone.utc) + since = connector._effective_since({}) + after = datetime.datetime.now(datetime.timezone.utc) + + assert since is not None + since_dt = datetime.datetime.fromisoformat(since) + # Should land within [before - 7d, after - 7d] -- a tight window around "right now minus 7 + # days", not some other bound. + assert ( + before - datetime.timedelta(days=7) + <= since_dt + <= after - datetime.timedelta(days=7) + ) + + +def test_effective_since_ignores_lookback_days_once_a_watermark_exists(): + """lookback_days must only ever apply to the first poll, never an in-progress one. + + A rolling lookback bound on every run would silently create a permanent gap -- see + connector.py's _effective_since docstring. + """ + connector = build_test_connector() + connector.config.lookback_days = 7 + + since = connector._effective_since( + {"last_seen_watermark": "2020-01-01T00:00:00+00:00"} + ) + + assert since == "2020-01-01T00:00:00+00:00" + + +def test_config_lookback_days_empty_string_does_not_crash(): + """An empty (but present) env var must resolve to None, not raise. + + The `${VAR:-}` pattern docker-compose.yml uses for every other optional lever here sets the + env var to an empty string, not unset -- verified directly against the installed pycti + source that get_config_variable(isNumber=True) calls int('') before ever checking for an + empty string, which raises. config.py reads this one manually instead specifically to avoid + that crash. + """ + config = Config( + raw={ + "vs_ticket_ingestion": { + "org_acronym_allowlist": "TESTORG1", + "is_local": True, + "tlp_marking": "TLP:AMBER", + "lookback_days": "", + } + } + ) + assert config.lookback_days is None + + +def test_config_lookback_days_parses_a_real_value(): + """A real numeric string must parse to the equivalent int.""" + config = Config( + raw={ + "vs_ticket_ingestion": { + "org_acronym_allowlist": "TESTORG1", + "is_local": True, + "tlp_marking": "TLP:AMBER", + "lookback_days": "14", + } + } + ) + assert config.lookback_days == 14 + + +def test_run_marks_only_the_first_poll_as_bootstrap(): + """run() must only mark the very first poll (no watermark yet) as the bootstrap one. + + This is the actual fix for the 413,925-stale-open-ticket gap (§9c/§10i): the lookback bound + alone only controls what counts as "recent," `include_stale_open` is what keeps + currently-actionable-but-stale tickets from being permanently excluded. Spies on + repository.fetch() rather than db.py's SQL directly -- the query-level behavior itself was + verified against a real postgres:17 container (see connector.py's _effective_since + docstring and db.py's _fetch_tickets_live comments). + """ + connector = build_test_connector() + calls = [] + original_fetch = connector.repository.fetch + + def spying_fetch(since_last_seen, include_stale_open=False): + calls.append(include_stale_open) + return original_fetch(since_last_seen, include_stale_open=include_stale_open) + + connector.repository.fetch = spying_fetch + + connector.run() # no watermark yet -- bootstrap + connector.run() # watermark now set -- steady-state + + assert calls == [True, False] diff --git a/open-cti/connectors/vs-ticket-ingestion/tests/test_mapping.py b/open-cti/connectors/vs-ticket-ingestion/tests/test_mapping.py new file mode 100644 index 000000000..c46649448 --- /dev/null +++ b/open-cti/connectors/vs-ticket-ingestion/tests/test_mapping.py @@ -0,0 +1,348 @@ +"""Loop 1 (OpenCTI-connector.md §9b): pure mapping unit tests, no DB/OpenCTI/queue at all.""" + +# Standard Python Libraries +from decimal import Decimal +import os +import sys + +sys.path.insert(0, os.path.join(os.path.dirname(__file__), "..")) + +# Third-Party Libraries +from pycti import Identity as OpenCTIIdentity # noqa: E402 +from pycti import StixCoreRelationship # noqa: E402 +import pytest # noqa: E402 + +# First-Party +from src import mapping # noqa: E402 +import stix2 # noqa: E402 + +# stix2 validates id shape strictly (--) -- these need to be real, well-formed ids, +# not readable placeholders like "identity--author", for stix2.Relationship() to accept them at +# all (found by running these tests, not by inspection). +_AUTHOR_ID = OpenCTIIdentity.generate_id("Test Author", "system") +_MARKING_ID = stix2.TLP_AMBER.id +_IP_ID = mapping.map_ip_observable("198.51.100.5").id +_VULN_ID = mapping.map_vulnerability({"id": "x", "cve_string": "CVE-2024-0001"}).id + + +def test_severity_label_buckets_match_standard_cvss_scale(): + """Scores must bucket into the standard CVSS v3 low/medium/high/critical cutoffs.""" + assert mapping.severity_label(None) is None + assert mapping.severity_label(0) is None + assert mapping.severity_label(3.9) == "vs-severity-low" + assert mapping.severity_label(4.0) == "vs-severity-medium" + assert mapping.severity_label(6.9) == "vs-severity-medium" + assert mapping.severity_label(7.0) == "vs-severity-high" + assert mapping.severity_label(8.9) == "vs-severity-high" + assert mapping.severity_label(9.0) == "vs-severity-critical" + assert mapping.severity_label(10.0) == "vs-severity-critical" + + +def test_severity_label_handles_real_decimal_from_psycopg2(): + """Ticket.cvss_severity is a DecimalField, not a plain float. + + Verified against a real postgres:17 container (§9b Loop 4) that this comes back as + decimal.Decimal, not float, on a live run. + """ + assert mapping.severity_label(Decimal("7.50")) == "vs-severity-high" + + +def test_severity_label_returns_none_for_garbage_input(): + """A non-numeric value must not crash the mapping, just return no label.""" + assert mapping.severity_label("not-a-number") is None + + +def test_map_ip_observable_v4(): + """An IPv4 host address maps to an IPv4-Addr observable.""" + obj = mapping.map_ip_observable("198.51.100.5") + assert obj.type == "ipv4-addr" + assert obj.value == "198.51.100.5" + + +def test_map_ip_observable_v6(): + """An IPv6 host address maps to an IPv6-Addr observable.""" + obj = mapping.map_ip_observable("2001:db8::5") + assert obj.type == "ipv6-addr" + + +def test_map_ip_observable_raises_on_malformed_input(): + """A malformed ip_string must raise, not silently produce a garbage observable.""" + with pytest.raises(ValueError): + mapping.map_ip_observable("not-an-ip") + + +def test_map_vulnerability_uses_uppercased_cve_string_when_present(): + """A ticket with a cve_string names its Vulnerability from the uppercased CVE id.""" + ticket = { + "id": "t1", + "cve_string": "cve-2024-1234", + "vuln_name": None, + "service_name": None, + } + obj = mapping.map_vulnerability(ticket) + assert obj.name == "CVE-2024-1234" + + +def test_map_vulnerability_falls_back_to_vuln_name_for_cve_less_tickets(): + """A CVE-less ticket names its Vulnerability from vuln_name when present.""" + ticket = { + "id": "t2", + "cve_string": None, + "vuln_name": "Weak Telnet Auth", + "service_name": None, + } + obj = mapping.map_vulnerability(ticket) + assert obj.name == "Weak Telnet Auth" + + +def test_map_vulnerability_falls_back_to_service_name_when_vuln_name_missing(): + """A CVE-less, vuln_name-less ticket falls back to service_name.""" + ticket = { + "id": "t3", + "cve_string": None, + "vuln_name": None, + "service_name": "telnet", + } + obj = mapping.map_vulnerability(ticket) + assert obj.name == "telnet" + + +def test_map_vulnerability_raises_when_nothing_usable_to_name_it_from(): + """A ticket with no cve_string, vuln_name, or service_name must raise, not fabricate a name.""" + ticket = {"id": "t4", "cve_string": None, "vuln_name": None, "service_name": None} + with pytest.raises(ValueError): + mapping.map_vulnerability(ticket) + + +def test_resolve_org_identity_id_matches_connector_ds_own_identity_construction(): + """§7a: 'look up, don't blindly create' -- this must land on connector D's own id. + + Specifically the exact same content-derived id connector D's mapping.map_organization() + computes for the same org name, since that's the only way this resolves to the *same* + Identity connector D already created. + """ + connector_d_id = OpenCTIIdentity.generate_id( + "Test Organization One", "organization" + ) + assert mapping.resolve_org_identity_id("Test Organization One") == connector_d_id + + +def test_build_ticket_relationship_carries_start_and_stop_time_for_a_closed_ticket(): + """A ticket with both opened_timestamp and closed_timestamp gets both on the relationship.""" + ticket = { + "id": "t5", + "opened_timestamp": "2026-06-01T00:00:00+00:00", + "closed_timestamp": "2026-08-21T09:30:00+00:00", + "vuln_source": "nessus", + "is_kev": False, + "is_kev_ransomware": False, + "is_risky": False, + "cvss_severity": None, + } + rel = mapping.build_ticket_relationship( + ticket, _IP_ID, _VULN_ID, _AUTHOR_ID, _MARKING_ID + ) + assert rel.start_time is not None + assert rel.stop_time is not None + + +def test_build_ticket_relationship_carries_updated_timestamp_as_custom_property(): + """Ticket.updated_timestamp must survive onto the relationship, not get dropped. + + Carried as x_opencti_updated_timestamp -- a plain custom property, not stix2's native + `modified` -- see build_ticket_relationship()'s docstring for why. + """ + ticket = { + "id": "t-updated", + "opened_timestamp": "2026-06-01T00:00:00+00:00", + "closed_timestamp": None, + "updated_timestamp": "2026-08-30T14:00:00+00:00", + "vuln_source": "nessus", + "is_kev": False, + "is_kev_ransomware": False, + "is_risky": False, + "cvss_severity": None, + } + rel = mapping.build_ticket_relationship( + ticket, _IP_ID, _VULN_ID, _AUTHOR_ID, _MARKING_ID + ) + assert rel.x_opencti_updated_timestamp == mapping.normalize_timestamp( + ticket["updated_timestamp"] + ) + assert "modified" not in rel or rel.modified != rel.x_opencti_updated_timestamp + + +def test_build_ticket_relationship_omits_updated_timestamp_property_when_absent(): + """A ticket with no updated_timestamp at all must not fabricate the custom property.""" + ticket = { + "id": "t-no-updated", + "opened_timestamp": "2026-06-01T00:00:00+00:00", + "closed_timestamp": None, + "updated_timestamp": None, + "vuln_source": "nessus", + "is_kev": False, + "is_kev_ransomware": False, + "is_risky": False, + "cvss_severity": None, + } + rel = mapping.build_ticket_relationship( + ticket, _IP_ID, _VULN_ID, _AUTHOR_ID, _MARKING_ID + ) + assert "x_opencti_updated_timestamp" not in rel + + +def test_build_ticket_relationship_id_is_stable_when_updated_timestamp_changes(): + """The id must not drift just because updated_timestamp changes on a rescan/re-ticket. + + §10i-style verification, done proactively: custom_properties never feed this relationship's + id (it's always explicitly pinned via `_relationship_id()`, never derived from its own + content) -- but confirmed here directly rather than just asserted in a docstring. + """ + base = { + "id": "t-stable", + "opened_timestamp": "2026-06-01T00:00:00+00:00", + "closed_timestamp": None, + "vuln_source": "nessus", + "is_kev": False, + "is_kev_ransomware": False, + "is_risky": False, + "cvss_severity": None, + } + rel1 = mapping.build_ticket_relationship( + dict(base, updated_timestamp="2026-08-01T00:00:00+00:00"), + _IP_ID, + _VULN_ID, + _AUTHOR_ID, + _MARKING_ID, + ) + rel2 = mapping.build_ticket_relationship( + dict(base, updated_timestamp="2026-08-30T00:00:00+00:00"), + _IP_ID, + _VULN_ID, + _AUTHOR_ID, + _MARKING_ID, + ) + assert rel1.id == rel2.id + + +def test_build_ticket_relationship_omits_stop_time_while_open(): + """A ticket with no closed_timestamp must not carry a stop_time at all.""" + ticket = { + "id": "t6", + "opened_timestamp": "2026-08-01T00:00:00+00:00", + "closed_timestamp": None, + "vuln_source": "nessus", + "is_kev": False, + "is_kev_ransomware": False, + "is_risky": False, + "cvss_severity": None, + } + rel = mapping.build_ticket_relationship( + ticket, _IP_ID, _VULN_ID, _AUTHOR_ID, _MARKING_ID + ) + assert "stop_time" not in rel + + +def test_build_ticket_relationship_drops_stop_time_when_not_strictly_after_start_time(): + """§10i: STIX 2.1 requires strictly-later stop_time. + + Verified against the installed stix2 library that even equal values raise ValueError. An + opened-and-closed-in-the-same-write-batch ticket (same timestamp for both) must not crash + the whole run over one row. + """ + ticket = { + "id": "t7", + "opened_timestamp": "2026-08-01T00:00:00+00:00", + "closed_timestamp": "2026-08-01T00:00:00+00:00", + "vuln_source": "nessus", + "is_kev": False, + "is_kev_ransomware": False, + "is_risky": False, + "cvss_severity": None, + } + rel = mapping.build_ticket_relationship( + ticket, _IP_ID, _VULN_ID, _AUTHOR_ID, _MARKING_ID + ) + assert "stop_time" not in rel + + +def test_build_ticket_relationship_id_changes_with_lifecycle_when_not_pinned(): + """Shows generate_id() alone WOULD change id on closure -- the reason connector.py pins it.""" + open_ticket = { + "id": "t8", + "opened_timestamp": "2026-08-01T00:00:00+00:00", + "closed_timestamp": None, + "vuln_source": "nessus", + "is_kev": False, + "is_kev_ransomware": False, + "is_risky": False, + "cvss_severity": None, + } + closed_ticket = dict(open_ticket, closed_timestamp="2026-08-21T00:00:00+00:00") + + open_rel = mapping.build_ticket_relationship( + open_ticket, _IP_ID, _VULN_ID, _AUTHOR_ID, _MARKING_ID + ) + closed_rel = mapping.build_ticket_relationship( + closed_ticket, _IP_ID, _VULN_ID, _AUTHOR_ID, _MARKING_ID + ) + assert ( + open_rel.id != closed_rel.id + ) # not pinned here -- connector.py is what pins it + + +def test_build_ticket_relationship_reuses_existing_id_when_pinned(): + """existing_id must be returned verbatim, not recomputed from source/target/time.""" + ticket = { + "id": "t9", + "opened_timestamp": "2026-08-01T00:00:00+00:00", + "closed_timestamp": "2026-08-21T00:00:00+00:00", + "vuln_source": "nessus", + "is_kev": False, + "is_kev_ransomware": False, + "is_risky": False, + "cvss_severity": None, + } + # A real, differently-computed relationship id -- proves build_ticket_relationship() returns + # *this* id verbatim rather than recomputing its own from source/target/time, not just that + # some well-formed id round-trips. + pinned_id = StixCoreRelationship.generate_id( + "related-to", _VULN_ID, _IP_ID, None, None + ) + rel = mapping.build_ticket_relationship( + ticket, _IP_ID, _VULN_ID, _AUTHOR_ID, _MARKING_ID, existing_id=pinned_id + ) + assert rel.id == pinned_id + + +def test_build_ticket_relationship_labels(): + """Source/kev/ransomware/risky/severity flags must all become labels.""" + ticket = { + "id": "t10", + "opened_timestamp": "2026-08-01T00:00:00+00:00", + "closed_timestamp": None, + "vuln_source": "nmap", + "is_kev": True, + "is_kev_ransomware": True, + "is_risky": True, + "cvss_severity": 9.5, + } + rel = mapping.build_ticket_relationship( + ticket, _IP_ID, _VULN_ID, _AUTHOR_ID, _MARKING_ID + ) + assert set(rel.labels) == { + "vs-source-nmap", + "vs-kev", + "vs-kev-ransomware", + "vs-risky", + "vs-severity-critical", + } + + +def test_dedupe_bundle_objects_collapses_repeated_ids(): + """Two STIX objects sharing an id must collapse to one in dedupe_bundle_objects().""" + a = stix2.IPv4Address(value="10.0.0.1") + a_again = stix2.IPv4Address(value="10.0.0.1") + b = stix2.IPv4Address(value="10.0.0.2") + result = mapping.dedupe_bundle_objects([a, a_again, b]) + assert len(result) == 2 diff --git a/open-cti/connectors/vs-vulnscan-enrichment/Dockerfile b/open-cti/connectors/vs-vulnscan-enrichment/Dockerfile new file mode 100644 index 000000000..1d26ce56b --- /dev/null +++ b/open-cti/connectors/vs-vulnscan-enrichment/Dockerfile @@ -0,0 +1,30 @@ +FROM python:3.12-slim + +WORKDIR /opt/connector + +# Zscaler TLS-inspects outbound HTTPS on this network -- identical setup to connectors A/D's +# Dockerfiles (see their comments/README.md "TLS interception" for the full explanation). Reused +# verbatim rather than rediscovered: this is a property of the *network*, not of any one +# connector -- every connector's build hits the same failure. +# +# Pulled from a named build context (docker-compose.yml's additional_contexts, pointed at +# /opt/open-cti/tls -- runtime state, never this repo checkout), never COPYed from anywhere +# inside this directory -- the cert never touches git. +COPY --from=zscaler_cert zscaler.pem /usr/local/share/ca-certificates/zscaler.crt +RUN apt-get update \ + && apt-get install -y --no-install-recommends ca-certificates \ + && update-ca-certificates \ + && rm -rf /var/lib/apt/lists/* +ENV PIP_CERT=/etc/ssl/certs/ca-certificates.crt \ + REQUESTS_CA_BUNDLE=/etc/ssl/certs/ca-certificates.crt \ + AWS_CA_BUNDLE=/etc/ssl/certs/ca-certificates.crt \ + SSL_CERT_FILE=/etc/ssl/certs/ca-certificates.crt + +COPY requirements.txt . +RUN pip install --no-cache-dir -r requirements.txt + +COPY src/ src/ +COPY tests/ tests/ + +# Relative imports in src/main.py require running as a module, not a script -- see README.md. +ENTRYPOINT ["python3", "-m", "src.main"] diff --git a/open-cti/connectors/vs-vulnscan-enrichment/README.md b/open-cti/connectors/vs-vulnscan-enrichment/README.md new file mode 100644 index 000000000..f8fe708e9 --- /dev/null +++ b/open-cti/connectors/vs-vulnscan-enrichment/README.md @@ -0,0 +1,117 @@ +# VS VulnScan Enrichment Connector + +Connector B in the four-connector VS architecture — see +[`../../OpenCTI-connector.md`](../../OpenCTI-connector.md) §7b for the full design, §8 for build +order (this one depends on connector A), and §10 for the implementation discipline this code +follows. + +Depends on [connector A](../vs-ticket-ingestion/) (or some other process) having already created +the `Vulnerability`/`IPv4-Addr`/`IPv6-Addr` entity being enriched — this is not a standalone +ingestion path. + +## What it does — and how it's structurally different from connectors A/D + +Given an analyst-triggered "Enrich" click on an IP or Vulnerability entity in OpenCTI, looks up +`VulnScan` scanner-level detail (CVSS v2/v3, plugin name/family, solution, synopsis, risk factor, +exploit availability, see-also references) and attaches it as a `Note` — additive, never +overwriting the `Vulnerability` SDO's own CVSS fields, which `connector-cve`/ +`connector-vulncheck` already own (§7b). + +This is `INTERNAL_ENRICHMENT`, not `EXTERNAL_IMPORT` — one real structural difference from +connectors A/D, not just a different table: + +- Triggered per-entity via `helper.listen()`, not polled on a schedule via `helper.schedule_iso()`. + There's no `run()`/watermark/`duration_period` here — each invocation is already scoped to + exactly one entity by construction, so there's no "accidentally unscoped run" failure mode to + guard against the way A/D's `org_acronym_allowlist`/`allow_unscoped_run` pair does. +- `CONNECTOR_AUTO=false` (manual trigger only) to start, matching every other enrichment + connector already running in this deployment (`connector-censys-enrichment`, etc.) — see + `config.yml.sample` for the reasoning. + +## Structure + +```text +src/ + config.py # env/yaml config loading, fails closed on tlp_marking + db.py # mini_data_lake access + IS_LOCAL fixture fallback, by-ip / by-cve lookups + mapping.py # pure row -> STIX mapping functions (no I/O) -- Loop 1 test target + connector.py # process_message(): lookup -> map -> bundle -> send, Note idempotency + main.py # entry point (helper.listen(), not schedule_iso()) +tests/ + fixtures/ # small synthetic sample rows, shaped like real mini_data_lake queries + test_mapping.py # Loop 1: pure unit tests, no DB/OpenCTI/queue at all + test_connector_dry_run.py # Loop 2: full pipeline against fixtures, OpenCTI helper stubbed out +``` + +Structurally identical to connectors A/D wherever the different lifecycle allows (§10i: apply +the same discipline from the start, don't reinvent it per connector). Run the fast loops +constantly: + +```bash +pip install -r requirements.txt +python3 -m pytest tests/ -v +``` + +## The one thing this connector refuses to start without + +- **`VS_VULNSCAN_ENRICHMENT_TLP_MARKING`** — must match connectors A/D's value once the + TLP/marking policy decision (`OpenCTI-connector.md` §10c) is actually made; it's one shared + decision across all four connectors. + +(No `ORG_ACRONYM_ALLOWLIST`/`ALLOW_UNSCOPED_RUN` pair here — see "What it does" above for why +that lever doesn't apply to a per-entity-triggered connector.) + +## Idempotency: why a Note needs its id pinned too, not just relationships + +`pycti.Note.generate_id()` hashes the Note's own `content` (verified against the installed pycti +source) — recomputing it fresh on every re-enrichment would produce a *different* Note id the +moment scanner data changes even slightly, the same failure mode §10a already documented for +relationships' `start_time`/`stop_time`. `connector.py` pins the first-seen Note id per triggered +entity in connector state (`state["note_ids"][entity_id]`) and reuses it verbatim on every later +enrichment of that same entity — OpenCTI's own "always upserts by standard id" behavior (already +confirmed while building connector D) is what turns that into an update-in-place, not a pile of +near-duplicate Notes accumulating on the same entity every time someone clicks "Enrich" again. + +## Running it + +Same two modes as connectors A/D: + +- **Fixture mode** (`IS_LOCAL=true`) — `db.py` loads `tests/fixtures/vuln_scans.json` instead of + querying Postgres. +- **Against real `mini_data_lake` + OpenCTI** — copy `config.yml.sample` to `config.yml`, fill in + `db_host`, then: + + ```bash + python3 -m src.main + ``` + + Unlike A/D, this blocks on `helper.listen()` waiting for a real "Enrich" trigger from OpenCTI — + there's nothing to observe until you actually click it on a real `IPv4-Addr`/`IPv6-Addr`/ + `Vulnerability` entity that connector A has already created. + +## TLS interception (Zscaler) + +Identical setup to connectors A/D — see +[connector D's README](../vs-org-bootstrap/README.md#tls-interception-zscaler--required-manual-per-instance-setup-not-git-managed). +Already set up once for this network; nothing additional needed beyond the `additional_contexts` +block already present in `docker-compose.yml`. + +## Known gaps / things to verify before this is production-ready + +- **`VulnScan` has no indexes at all** (`backend/src/xfd_django/xfd_mini_dl/models.py`'s + `VulnScan.Meta` defines none, unlike `Ticket`'s `tickets_is_open_idx`/`ticket_last_seen_idx`) + — `ip_string`/`cve_string` lookups may currently be sequential scans over the whole table. + Matters more here than for A/D's background polls, since this runs synchronously while an + analyst is waiting on the "Enrich" button. Confirm the real query plan (`EXPLAIN ANALYZE` via + `crossfeed-staging-bastion`) before treating this as production-ready — not yet done. +- **`see_also`/`xref` format is unconfirmed.** `mapping.build_external_references()` + conservatively only turns a `see_also` value into a clickable External Reference when it + already looks like a URL; everything else is still captured in the Note's own content, just + not double-represented as a reference. Worth revisiting once real values are inspected. +- **A Note that no longer has any matching VulnScan rows on re-enrichment is left as-is, not + revoked.** Low-priority in practice (scanner data essentially only grows/updates), but a real, + documented gap rather than an oversight — connector A's false-positive revocation path (§7a) + doesn't have an equivalent here yet. +- Not yet run against a real OpenCTI instance — the fixture-based dry-run pipeline (Loop 2, §9b) + is done and passing; the actual "Enrich" click against real `mini_data_lake` data is next, the + same order connectors D and A went through. diff --git a/open-cti/connectors/vs-vulnscan-enrichment/config.yml.sample b/open-cti/connectors/vs-vulnscan-enrichment/config.yml.sample new file mode 100644 index 000000000..9572b302c --- /dev/null +++ b/open-cti/connectors/vs-vulnscan-enrichment/config.yml.sample @@ -0,0 +1,51 @@ +# Connector B — VS VulnScan Enrichment +# +# See ../../OpenCTI-connector.md §7b for the design this implements, and §10 for the discipline +# this config exists to enforce. +# +# Copy to config.yml for local runs (docker-compose.yml drives the containerized version purely +# from env vars instead — see README.md). + +opencti: + url: 'http://localhost:8080' + token: 'ChangeMe' + +connector: + id: 'ChangeMe-generate-a-uuid4' + name: 'VS VulnScan Enrichment' + type: 'INTERNAL_ENRICHMENT' + scope: 'IPv4-Addr,IPv6-Addr,Vulnerability' + confidence_level: 75 + log_level: 'info' + # Manual "Enrich" trigger, not automatic on every write (§7b) -- matches every other + # enrichment connector already running in this deployment (connector-censys-enrichment, etc.). + # Auto-triggering would mean one VulnScan lookup per Ticket per Connector A poll, hitting the + # same primary DB Connector A just queried -- revisit only once real data volume is known. + auto: false + +vs_vulnscan_enrichment: + # --- mini_data_lake connectivity (OpenCTI-connector.md §2) --- + db_host: 'ChangeMe' + db_port: 5432 + db_name: 'cyhy_mini_data_lake_staging' + db_user: 'open_cti' + # IAM database authentication, not a static password -- see §2. + db_use_iam_auth: true + db_password: '' + aws_region: 'us-gov-west-1' + + # --- dev/test discipline (§9) --- + is_local: false + local_fixture_dir: './tests/fixtures' + + # Bounds how many VulnScan rows go into one Note per triggered entity (§9c) -- a second, + # independent safety net against an unexpectedly large match set (many ports on one host, or + # many hosts sharing one CVE), the same way max_rows_per_run bounds connectors A/D's polls. + max_vulnscan_rows_per_entity: 50 + + # --- attribution / marking (§10c) --- + # Same author name as connectors A/D on purpose -- one consistent system Identity. + author_name: 'CISA VulnScanning' + # Deliberately left unset -- same undecided policy question as A/D. Must match their value + # once the policy call is made. + tlp_marking: '' diff --git a/open-cti/connectors/vs-vulnscan-enrichment/requirements.txt b/open-cti/connectors/vs-vulnscan-enrichment/requirements.txt new file mode 100644 index 000000000..4550261bf --- /dev/null +++ b/open-cti/connectors/vs-vulnscan-enrichment/requirements.txt @@ -0,0 +1,4 @@ +boto3==1.43.81 +psycopg2-binary==2.9.9 +pycti==7.260824.0 +stix2==3.0.2 diff --git a/open-cti/connectors/vs-vulnscan-enrichment/src/__init__.py b/open-cti/connectors/vs-vulnscan-enrichment/src/__init__.py new file mode 100644 index 000000000..17b862c4d --- /dev/null +++ b/open-cti/connectors/vs-vulnscan-enrichment/src/__init__.py @@ -0,0 +1 @@ +"""VS VulnScan Enrichment connector -- see ../README.md and ../../OpenCTI-connector.md.""" diff --git a/open-cti/connectors/vs-vulnscan-enrichment/src/config.py b/open-cti/connectors/vs-vulnscan-enrichment/src/config.py new file mode 100644 index 000000000..1b2cf6b36 --- /dev/null +++ b/open-cti/connectors/vs-vulnscan-enrichment/src/config.py @@ -0,0 +1,146 @@ +"""Config loading for the VS VulnScan Enrichment connector. + +Same fail-closed shape as connectors A/D's config.py -- precedence is env var > config.yml > +default via pycti's own get_config_variable() (§10i: apply the same discipline, don't reinvent +it per connector). Two things this module still fails closed on: + - tlp_marking: the same shared, still-undecided policy question (§10c) -- one call across all + four connectors, not re-litigated per connector. + +What's deliberately *not* here, unlike A/D: an org_acronym_allowlist / allow_unscoped_run pair. +This connector never polls wholesale (§7b) -- it's INTERNAL_ENRICHMENT, triggered per-entity by +an analyst (or, if CONNECTOR_AUTO is ever flipped on, by the platform) via `helper.listen()`, not +`schedule_iso()`. There's no "accidentally unscoped run" failure mode to guard against here the +way there is for A/C's wholesale polls -- each invocation is already scoped to exactly one +triggered entity by construction. +""" + +# Standard Python Libraries +import os +from typing import Dict, Optional + +# Third-Party Libraries +from pycti import get_config_variable +import yaml + + +class ConfigError(RuntimeError): + """Raised when required configuration is missing -- fail closed, not silent default.""" + + +def load_yaml_config() -> Dict: + """Load config.yml next to this connector if present; docker-compose drives env vars only.""" + here = os.path.dirname(os.path.abspath(__file__)) + config_file_path = os.path.join(here, "..", "config.yml") + if os.path.isfile(config_file_path): + with open(config_file_path, encoding="utf-8") as f: + return yaml.safe_load(f) or {} + return {} + + +class Config: + """Typed view over this connector's configuration.""" + + def __init__(self, raw: Optional[Dict] = None): + """Load config from `raw` if given (tests), else config.yml/env vars, then validate.""" + self.raw: Dict = raw if raw is not None else load_yaml_config() + + # --- mini_data_lake connectivity (OpenCTI-connector.md §2) --- + self.db_host = get_config_variable( + "VS_VULNSCAN_ENRICHMENT_DB_HOST", + ["vs_vulnscan_enrichment", "db_host"], + self.raw, + ) + self.db_port = get_config_variable( + "VS_VULNSCAN_ENRICHMENT_DB_PORT", + ["vs_vulnscan_enrichment", "db_port"], + self.raw, + isNumber=True, + default=5432, + ) + self.db_name = get_config_variable( + "VS_VULNSCAN_ENRICHMENT_DB_NAME", + ["vs_vulnscan_enrichment", "db_name"], + self.raw, + default="cyhy_mini_data_lake_staging", + ) + self.db_user = get_config_variable( + "VS_VULNSCAN_ENRICHMENT_DB_USER", + ["vs_vulnscan_enrichment", "db_user"], + self.raw, + default="open_cti", + ) + self.db_use_iam_auth = get_config_variable( + "VS_VULNSCAN_ENRICHMENT_DB_USE_IAM_AUTH", + ["vs_vulnscan_enrichment", "db_use_iam_auth"], + self.raw, + default=True, + ) + self.db_password = get_config_variable( + "VS_VULNSCAN_ENRICHMENT_DB_PASSWORD", + ["vs_vulnscan_enrichment", "db_password"], + self.raw, + default="", + ) + self.aws_region = get_config_variable( + "VS_VULNSCAN_ENRICHMENT_AWS_REGION", + ["vs_vulnscan_enrichment", "aws_region"], + self.raw, + ) + + # --- dev/test discipline (§9) --- + self.is_local = get_config_variable( + "IS_LOCAL", ["vs_vulnscan_enrichment", "is_local"], self.raw, default=False + ) + self.local_fixture_dir = get_config_variable( + "VS_VULNSCAN_ENRICHMENT_LOCAL_FIXTURE_DIR", + ["vs_vulnscan_enrichment", "local_fixture_dir"], + self.raw, + default="./tests/fixtures", + ) + + # A single "Enrich" click can, in principle, match many VulnScan rows for the same + # host (many ports/plugins) or the same CVE (many hosts) -- bound how many go into one + # Note the same way max_rows_per_run bounds A/D's polls, for the same reason (§9c): a + # hard cap as a second, independent safety net against an unexpectedly large match set, + # not something a per-entity trigger is naturally exempt from just because it's scoped + # to one entity. + self.max_vulnscan_rows_per_entity = get_config_variable( + "VS_VULNSCAN_ENRICHMENT_MAX_ROWS_PER_ENTITY", + ["vs_vulnscan_enrichment", "max_vulnscan_rows_per_entity"], + self.raw, + isNumber=True, + default=50, + ) + + # --- attribution / marking (§10c) --- + # Same author name as connectors A/D on purpose -- one consistent system Identity across + # all four connectors. + self.author_name = get_config_variable( + "VS_VULNSCAN_ENRICHMENT_AUTHOR_NAME", + ["vs_vulnscan_enrichment", "author_name"], + self.raw, + default="CISA VulnScanning", + ) + self.tlp_marking = get_config_variable( + "VS_VULNSCAN_ENRICHMENT_TLP_MARKING", + ["vs_vulnscan_enrichment", "tlp_marking"], + self.raw, + ) + + self._validate() + + def _validate(self) -> None: + """Fail closed on the one decision this doc can't make silently. See module docstring.""" + if not self.tlp_marking: + raise ConfigError( + "vs_vulnscan_enrichment.tlp_marking is not set. This is a real, undecided " + "policy question (OpenCTI-connector.md §10c) -- refusing to start rather than " + "default to something like TLP:CLEAR for stakeholder vulnerability data. Set " + "VS_VULNSCAN_ENRICHMENT_TLP_MARKING once the data-handling policy call is made." + ) + if not self.is_local and not self.db_host: + raise ConfigError( + "vs_vulnscan_enrichment.db_host is not set and is_local is false -- nothing to " + "connect to. Set VS_VULNSCAN_ENRICHMENT_DB_HOST, or IS_LOCAL=true to use " + "fixtures instead." + ) diff --git a/open-cti/connectors/vs-vulnscan-enrichment/src/connector.py b/open-cti/connectors/vs-vulnscan-enrichment/src/connector.py new file mode 100644 index 000000000..034e7dbbc --- /dev/null +++ b/open-cti/connectors/vs-vulnscan-enrichment/src/connector.py @@ -0,0 +1,151 @@ +"""Orchestration for the VS VulnScan Enrichment connector (OpenCTI-connector.md §7b). + +Depends on connector A (or some other process) having already created the Vulnerability/ +IPv4-Addr/IPv6-Addr entity being enriched (§7b) -- this is not a standalone ingestion path. + +Structurally different from connectors A/D in one real way, not just cosmetically: this is +INTERNAL_ENRICHMENT, triggered per-entity via `helper.listen()`, not `schedule_iso()`. There's no +`run()`/watermark/scoping-lever machinery here because there's no wholesale poll to guard -- each +invocation is already scoped to exactly one entity by construction. Everything else (author/ +marking setup, AppLogger call discipline, idempotent-id pinning via connector state) follows the +same §10i discipline as A/D. +""" + +# Standard Python Libraries +import logging +from typing import Dict, Optional + +# Third-Party Libraries +from pycti import OpenCTIConnectorHelper +import stix2 + +from . import mapping +from .config import Config +from .db import VulnScanEnrichmentRepository + +LOGGER = logging.getLogger("vs_vulnscan_enrichment") + +# Same well-known TLP marking-definition ids as connectors A/D. +_TLP_MARKINGS = { + "TLP:CLEAR": stix2.TLP_WHITE, + "TLP:WHITE": stix2.TLP_WHITE, + "TLP:GREEN": stix2.TLP_GREEN, + "TLP:AMBER": stix2.TLP_AMBER, + "TLP:RED": stix2.TLP_RED, +} + +# CONNECTOR_SCOPE (§7b) -- the only entity types this connector ever gets triggered on. +_IP_TYPES = {"IPv4-Addr", "IPv6-Addr"} +_VULNERABILITY_TYPE = "Vulnerability" + + +def resolve_marking(tlp_marking: str) -> stix2.MarkingDefinition: + """Resolve a TLP string to its well-known marking-definition object. Identical to A/D.""" + marking = _TLP_MARKINGS.get(tlp_marking.strip().upper()) + if marking is None: + raise ValueError( + f"Unrecognized tlp_marking {tlp_marking!r}. Expected one of: " + f"{', '.join(_TLP_MARKINGS)} (see OpenCTI-connector.md §10c)." + ) + return marking + + +class VsVulnscanEnrichmentConnector: + """Enriches a triggered IP/Vulnerability entity with VulnScan scanner-level detail. + + Produces one Note per triggered entity (§7b's "additive, not overwriting" design), pinned + per entity in connector state so re-enrichment updates that same Note rather than piling up + duplicates. + """ + + def __init__(self, config: Optional[Config] = None, helper=None): + """Build the connector, injecting `config`/`helper` for tests instead of the real thing.""" + self.config = config or Config() + self.helper = helper or OpenCTIConnectorHelper(self.config.raw) + self.repository = VulnScanEnrichmentRepository(self.config) + + self.author = mapping.build_author(self.config.author_name) + self.marking = resolve_marking(self.config.tlp_marking) + + LOGGER.info( + "VS VulnScan Enrichment connector initialized (is_local=%s)", + self.config.is_local, + ) + + def process_message(self, data: Dict) -> str: + """Entry point pycti's `listen()` calls once per "Enrich" trigger. + + `data` is pycti's own event dict (ListenQueue._data_handler) -- `entity_id`, + `entity_type`, and `stix_entity` (already parsed from JSON) are what this connector + needs; `work_id`/error reporting back to OpenCTI are handled entirely by pycti itself + around this call (it catches whatever this raises and marks the work as errored, so + there's no need to catch-and-report manually here the way A/D's run() does for its own + top-level state-corruption concern -- a single bad entity has no shared state to + protect, unlike a wholesale poll). + """ + entity_id = data["entity_id"] + entity_type = data.get("entity_type") + stix_entity = data.get("stix_entity") or {} + + vuln_scans = self._lookup(entity_type, stix_entity).vuln_scans + + # §9 queue discipline, same instinct as connector A's revocation-only-run: nothing found + # means nothing worth putting on the queue. A permanent "nothing found" Note attached to + # the entity would just be clutter in OpenCTI's UI, not a useful artifact -- the status + # message returned below (visible in the work's own history for this Enrich click) is + # where "we checked and found nothing" belongs, not a persistent object on the entity. + if not vuln_scans: + message = f"No VS scanner-level detail found for {entity_id}" + self.helper.connector_logger.info(message) + return message + + state = self.helper.get_state() or {} + note_map = state.setdefault("note_ids", {}) + note = mapping.build_note( + entity_id, + vuln_scans, + self.author.id, + self.marking.id, + existing_id=note_map.get(entity_id), + ) + + objects = mapping.dedupe_bundle_objects([self.author, self.marking, note]) + bundle = self.helper.stix2_create_bundle(objects) + # work_id defaults to self.helper.work_id, which pycti already set before calling this + # method -- no need to pass it explicitly (confirmed against the installed pycti source). + self.helper.send_stix2_bundle(bundle, update=True) + + # Only pin note_id in connector state once the bundle has actually been accepted by + # OpenCTI, not before -- same discipline as A/D's watermark-after-send (§10f). Persisting + # this any earlier would let a failed send leave state pointing at a Note that was never + # created, and a retry would then reuse that id assuming it already exists. + note_map[entity_id] = note.id + self.helper.set_state(state) + + message = f"Enriched {entity_id} with {len(vuln_scans)} VS scanner finding(s)" + self.helper.connector_logger.info(message) + return message + + def _lookup(self, entity_type: Optional[str], stix_entity: Dict): + """Resolve the triggered entity to a lookup key and query VulnScan for it. + + Raises for anything outside CONNECTOR_SCOPE (§7b) rather than silently returning nothing + -- a scope mismatch is a configuration problem worth surfacing via the work's error + state, not swallowing. + """ + if entity_type in _IP_TYPES: + ip_string = stix_entity.get("value") + if not ip_string: + raise ValueError(f"IP entity has no usable 'value': {stix_entity!r}") + return self.repository.fetch_by_ip(ip_string) + if entity_type == _VULNERABILITY_TYPE: + cve_string = stix_entity.get("name") + if not cve_string: + raise ValueError( + f"Vulnerability entity has no usable 'name': {stix_entity!r}" + ) + return self.repository.fetch_by_cve(cve_string) + raise ValueError( + f"Unexpected entity_type {entity_type!r} -- outside this connector's " + f"CONNECTOR_SCOPE ({', '.join(sorted(_IP_TYPES | {_VULNERABILITY_TYPE}))})" + ) diff --git a/open-cti/connectors/vs-vulnscan-enrichment/src/db.py b/open-cti/connectors/vs-vulnscan-enrichment/src/db.py new file mode 100644 index 000000000..2ebb53046 --- /dev/null +++ b/open-cti/connectors/vs-vulnscan-enrichment/src/db.py @@ -0,0 +1,166 @@ +"""mini_data_lake access for the VS VulnScan Enrichment connector. + +Same two-implementation shape as connectors A/D's db.py (real Postgres + IS_LOCAL fixtures, see +OpenCTI-connector.md §9a) -- kept structurally identical rather than reinvented (§10i), even +though this connector is looked up per-entity rather than polled wholesale (§7b). + +Table/column names taken from backend/src/xfd_django/xfd_mini_dl/models.py -- `VulnScan` +(L1737) has `db_table = "vuln_scan"`. + +Real, open verification item (not yet confirmed against the live box): `VulnScan.Meta` defines +no `indexes` at all (unlike `Ticket`'s `tickets_is_open_idx`/`ticket_last_seen_idx`) -- meaning +`ip_string`/`cve_string` lookups here may currently be sequential scans over the whole table. +That matters more for this connector than A/D's polls: this runs synchronously while an analyst +is waiting on OpenCTI's "Enrich" button, not on a background schedule. Confirm the real query +plan (`EXPLAIN ANALYZE`) against `cyhy_mini_data_lake_staging` via `crossfeed-staging-bastion` +before treating this as production-ready -- flagged here rather than assumed either way. +""" + +# Standard Python Libraries +from dataclasses import dataclass, field +import json +import logging +import os +from typing import Dict, List + +# Third-Party Libraries +import boto3 +import psycopg2 +import psycopg2.extras + +from .config import Config + +LOGGER = logging.getLogger("vs_vulnscan_enrichment.db") + + +@dataclass +class VulnScanEnrichmentData: + """Everything one enrichment lookup needs, regardless of where it came from.""" + + vuln_scans: List[Dict] = field(default_factory=list) + + +def _generate_iam_auth_token(config: Config) -> str: + """Generate a short-lived RDS IAM auth token. Same pattern as connectors A/D -- see their db.py.""" + client = boto3.client("rds", region_name=config.aws_region) + return client.generate_db_auth_token( + DBHostname=config.db_host, + Port=config.db_port, + DBUsername=config.db_user, + ) + + +def get_connection(config: Config): + """Open a psycopg2 connection to mini_data_lake, IAM-auth or password per config.""" + password = ( + _generate_iam_auth_token(config) + if config.db_use_iam_auth + else config.db_password + ) + return psycopg2.connect( + host=config.db_host, + port=config.db_port, + dbname=config.db_name, + user=config.db_user, + password=password, + sslmode="require", # IAM auth requires TLS regardless of rds.force_ssl -- §2 + cursor_factory=psycopg2.extras.RealDictCursor, + connect_timeout=10, + ) + + +# Explicit column projection (§10d) -- only what mapping.py actually maps into a Note. Each +# query below is one complete literal string, matching connectors A/D's db.py convention -- +# no f-string/`.format()` assembly of query text at the call site. That wasn't just a style +# preference: bandit's B608 (hardcoded_sql_expressions) flagged an earlier version of this file +# that built the query via f-string-interpolating a shared column-list constant, even though the +# actual variable data was always going through a proper %(value)s placeholder, never the +# f-string. A real, low-confidence-but-worth-fixing finding -- writing each query as one +# complete literal (some column-list duplication, same tradeoff A/D already made) removes the +# pattern entirely instead of suppressing the warning. +_IP_QUERY = """ + SELECT id, ip_string, cve_string, cvss_base_score, cvss_vector, cvss3_base_score, + cvss3_vector, cvss3_temporal_score, plugin_id, plugin_name, plugin_family, + solution, synopsis, description, risk_factor, exploit_available, + exploitability_ease, see_also, xref, port, port_protocol, service, source, + owner, vuln_detection_timestamp + FROM vuln_scan + WHERE ip_string = %(value)s + LIMIT %(limit)s +""" + +_CVE_QUERY = """ + SELECT id, ip_string, cve_string, cvss_base_score, cvss_vector, cvss3_base_score, + cvss3_vector, cvss3_temporal_score, plugin_id, plugin_name, plugin_family, + solution, synopsis, description, risk_factor, exploit_available, + exploitability_ease, see_also, xref, port, port_protocol, service, source, + owner, vuln_detection_timestamp + FROM vuln_scan + WHERE UPPER(cve_string) = UPPER(%(value)s) + LIMIT %(limit)s +""" + + +class VulnScanEnrichmentRepository: + """Looks up VulnScan rows for one triggered entity, capped per config (§9c).""" + + def __init__(self, config: Config): + """Hold the config; connections are opened per-call, not cached (§10d).""" + self.config = config + + def fetch_by_ip(self, ip_string: str) -> VulnScanEnrichmentData: + """Look up scanner detail for an IPv4-Addr/IPv6-Addr entity, by exact ip_string match.""" + if self.config.is_local: + return self._fetch_local(lambda row: row.get("ip_string") == ip_string) + return self._fetch_live(_IP_QUERY, ip_string) + + def fetch_by_cve(self, cve_string: str) -> VulnScanEnrichmentData: + """Look up scanner detail for a Vulnerability entity, by case-insensitive cve_string match. + + Case-insensitive because connector A uppercases the `Vulnerability.name` it creates + (OpenCTI-connector.md §7a), but `VulnScan.cve_string` is raw scanner data whose casing + isn't independently confirmed here -- safer to match loosely than to silently miss real + rows over a casing mismatch. + """ + if self.config.is_local: + return self._fetch_local( + lambda row: (row.get("cve_string") or "").upper() == cve_string.upper() + ) + return self._fetch_live(_CVE_QUERY, cve_string) + + # ------------------------------------------------------------------ + # Live path + # ------------------------------------------------------------------ + + def _fetch_live(self, query: str, value: str) -> VulnScanEnrichmentData: + conn = get_connection(self.config) + try: + with conn.cursor() as cur: + cur.execute( + query, + {"value": value, "limit": self.config.max_vulnscan_rows_per_entity}, + ) + return VulnScanEnrichmentData( + vuln_scans=[dict(row) for row in cur.fetchall()] + ) + finally: + conn.close() + + # ------------------------------------------------------------------ + # IS_LOCAL fixture path (§9a) -- same shape as the live path, zero DB/network. + # ------------------------------------------------------------------ + + def _fetch_local(self, matches) -> VulnScanEnrichmentData: + directory = self.config.local_fixture_dir + LOGGER.info("IS_LOCAL=true -- loading fixtures from %s", directory) + rows = self._load_json(directory, "vuln_scans.json") + matched = [row for row in rows if matches(row)] + return VulnScanEnrichmentData( + vuln_scans=matched[: self.config.max_vulnscan_rows_per_entity] + ) + + @staticmethod + def _load_json(directory: str, filename: str): + path = os.path.join(directory, filename) + with open(path, encoding="utf-8") as f: + return json.load(f) diff --git a/open-cti/connectors/vs-vulnscan-enrichment/src/main.py b/open-cti/connectors/vs-vulnscan-enrichment/src/main.py new file mode 100644 index 000000000..b2980565b --- /dev/null +++ b/open-cti/connectors/vs-vulnscan-enrichment/src/main.py @@ -0,0 +1,19 @@ +"""Entry point. See OpenCTI-connector.md §7b for design, README.md for how to run this locally.""" + +# Standard Python Libraries +import sys + +from .config import ConfigError +from .connector import VsVulnscanEnrichmentConnector + +if __name__ == "__main__": + try: + connector = VsVulnscanEnrichmentConnector() + except ConfigError as e: + # Fail closed and loud (§10c) -- not a stack trace to dig through. + print(f"[vs-vulnscan-enrichment] refusing to start: {e}", file=sys.stderr) + sys.exit(1) + + # INTERNAL_ENRICHMENT connectors block on listen(), not schedule_iso() -- there's no + # duration_period/watermark here, each invocation is triggered per-entity (§7b). + connector.helper.listen(connector.process_message) diff --git a/open-cti/connectors/vs-vulnscan-enrichment/src/mapping.py b/open-cti/connectors/vs-vulnscan-enrichment/src/mapping.py new file mode 100644 index 000000000..482e16e7e --- /dev/null +++ b/open-cti/connectors/vs-vulnscan-enrichment/src/mapping.py @@ -0,0 +1,181 @@ +"""Pure row -> STIX mapping functions for the VS VulnScan Enrichment connector. + +Deliberately dependency-free beyond stix2/pycti and the standard library -- no DB, no OpenCTI +API, no network. This is "Loop 1" from OpenCTI-connector.md §9b. See tests/test_mapping.py. + +STIX mapping decisions here are documented in OpenCTI-connector.md §7b: enrichment is additive +(a `Note` attached to the triggered entity), never overwriting the `Vulnerability` SDO's own +CVSS fields -- those belong to `connector-cve`/`connector-vulncheck`, not this connector. +""" + +# Standard Python Libraries +from typing import Dict, List, Optional + +# Third-Party Libraries +from pycti import Identity as OpenCTIIdentity +from pycti import Note as OpenCTINote +import stix2 + +VS_EXTERNAL_SOURCE = "VS-VulnScan" + + +def build_author(name: str) -> stix2.Identity: + """Build the single system Identity every VS-sourced object should cite as createdBy (§10c). + + Identical construction to connectors A/D's build_author() -- same name, same identity_class + -- so all three connectors' Identity(class="system") objects resolve to the one shared author. + """ + return stix2.Identity( + id=OpenCTIIdentity.generate_id(name, "system"), + name=name, + identity_class="system", + ) + + +def _format_row(row: Dict) -> str: + """Render one VulnScan row as a short, readable section of the enrichment Note's content. + + `cvss_base_score`/`cvss_vector` (CVSS v2) are plain CharFields; `cvss3_base_score`/ + `cvss3_temporal_score` are DecimalFields (verified against a real postgres:17 container, + §9b Loop 4, before this ever touched the live box) -- both just get stringified for display + here, never used numerically, so the live-vs-fixture type gap that mattered for Ticket's + cvss_severity (§10i) doesn't bite the same way for this connector. + """ + lines = [f"### {row.get('plugin_name') or row.get('plugin_id') or 'Finding'}"] + port = row.get("port") + if port: + lines.append( + f"- Port/service: {port}/{row.get('port_protocol') or '?'} ({row.get('service') or 'unknown'})" + ) + if row.get("cvss_base_score"): + lines.append( + f"- CVSS v2: {row['cvss_base_score']} ({row.get('cvss_vector') or 'no vector'})" + ) + if row.get("cvss3_base_score") is not None: + lines.append( + f"- CVSS v3: {row['cvss3_base_score']} ({row.get('cvss3_vector') or 'no vector'})" + ) + if row.get("risk_factor"): + lines.append(f"- Risk factor: {row['risk_factor']}") + if row.get("exploit_available"): + lines.append( + f"- Exploit available: {row['exploit_available']} " + f"({row.get('exploitability_ease') or 'ease unknown'})" + ) + if row.get("synopsis"): + lines.append(f"- Synopsis: {row['synopsis']}") + if row.get("solution"): + lines.append(f"- Solution: {row['solution']}") + if row.get("source") or row.get("owner"): + lines.append( + f"- Source: {row.get('source') or 'unknown'} (owner: {row.get('owner') or 'unknown'})" + ) + if row.get("vuln_detection_timestamp"): + # Plain display text, not a STIX timestamp property -- no TimestampProperty validation + # involved, so str() is enough regardless of whether this is a real psycopg2 datetime + # (live) or an ISO string (IS_LOCAL fixture). Unlike mapping.normalize_timestamp() in + # connectors A/D, there's nothing here for stix2 to reject. + lines.append(f"- Last detected: {row['vuln_detection_timestamp']}") + return "\n".join(lines) + + +def build_note_content(vuln_scans: List[Dict]) -> str: + """Render every matched VulnScan row into one Note's content, oldest logic first. + + Bounded by db.py's own LIMIT (VS_VULNSCAN_ENRICHMENT_MAX_ROWS_PER_ENTITY, §9c) before this + ever runs -- no further truncation needed here. + """ + if not vuln_scans: + return "No VS scanner-level detail found for this entity." + return "\n\n".join(_format_row(row) for row in vuln_scans) + + +def build_note_abstract(vuln_scans: List[Dict]) -> str: + """One-line summary shown in OpenCTI's Note list view.""" + return f"VS scanner enrichment ({len(vuln_scans)} finding(s))" + + +def build_external_references( + vuln_scans: List[Dict], +) -> Optional[List[stix2.ExternalReference]]: + """Build one External Reference per VulnScan row that has a usable see_also URL. + + `see_also`/`xref` are free-text TextFields with no confirmed format (models.py's own + docstrings don't specify one) -- deliberately conservative here: only rows where `see_also` + looks like an actual URL become a clickable External Reference; everything else is already + captured in the Note's own content via _format_row, not lost, just not double-represented as + a reference. Worth revisiting once real `see_also`/`xref` values from mini_data_lake are + inspected, same as connector A's severity-bucket caveat -- flagged, not guessed past. + """ + refs = [] + for row in vuln_scans: + see_also = (row.get("see_also") or "").strip() + row_id = row.get("id") + if row_id and ( + see_also.startswith("http://") or see_also.startswith("https://") + ): + refs.append( + stix2.ExternalReference( + source_name=VS_EXTERNAL_SOURCE, + external_id=row_id, + url=see_also, + ) + ) + return refs or None + + +def _note_id(content: str, existing_id: Optional[str]) -> str: + """Apply the §10a idempotency pattern to a Note, for a different reason than relationships. + + Same as every relationship builder in connectors A/D, but worth spelling out why it applies + to a Note too. + + pycti.Note.generate_id() hashes `content` (and `created`/`abstract`) directly -- verified + against the installed pycti source -- so recomputing it fresh on every re-enrichment would + produce a *different* id the moment scanner data changes even slightly, the same failure + mode §10a already documented for relationships' start_time/stop_time. Pinning the first-seen + id in connector state (keyed by the triggered entity's id -- see connector.py) and reusing it + verbatim is what turns every re-trigger into an update-in-place on one Note, instead of a + pile of near-duplicate Notes accumulating on the same entity. + """ + if existing_id: + return existing_id + return OpenCTINote.generate_id(created=None, content=content) + + +def build_note( + entity_stix_id: str, + vuln_scans: List[Dict], + author_id: str, + marking_id: str, + existing_id: Optional[str] = None, +) -> stix2.Note: + """Build the enrichment Note attached to the triggered entity. + + Additive, not overwriting (§7b): this never touches the Vulnerability SDO's own CVSS fields + -- connector-cve/connector-vulncheck already own those. object_refs is what actually attaches + this Note to the entity that was enriched. + """ + content = build_note_content(vuln_scans) + return stix2.Note( + id=_note_id(content, existing_id), + abstract=build_note_abstract(vuln_scans), + content=content, + object_refs=[entity_stix_id], + created_by_ref=author_id, + object_marking_refs=[marking_id], + external_references=build_external_references(vuln_scans), + ) + + +def dedupe_bundle_objects(objects: List) -> List: + """Collapse a list of STIX objects to one per id, keeping the last occurrence of each. + + Same rationale as connectors A/D -- the author/marking objects are shared across every + invocation, and this keeps that consistent even as this connector's own object list grows + in the future. + """ + seen = {} + for obj in objects: + seen[obj["id"]] = obj + return list(seen.values()) diff --git a/open-cti/connectors/vs-vulnscan-enrichment/tests/__init__.py b/open-cti/connectors/vs-vulnscan-enrichment/tests/__init__.py new file mode 100644 index 000000000..52a5c5474 --- /dev/null +++ b/open-cti/connectors/vs-vulnscan-enrichment/tests/__init__.py @@ -0,0 +1 @@ +"""Tests for the VS VulnScan Enrichment connector -- see README.md for the two loops.""" diff --git a/open-cti/connectors/vs-vulnscan-enrichment/tests/fixtures/vuln_scans.json b/open-cti/connectors/vs-vulnscan-enrichment/tests/fixtures/vuln_scans.json new file mode 100644 index 000000000..2092cfafe --- /dev/null +++ b/open-cti/connectors/vs-vulnscan-enrichment/tests/fixtures/vuln_scans.json @@ -0,0 +1,56 @@ +[ + { + "cve_string": "CVE-2024-1234", + "cvss3_base_score": 9.8, + "cvss3_temporal_score": 9.1, + "cvss3_vector": "CVSS:3.1/AV:N/AC:L/PR:N/UI:N/S:U/C:H/I:H/A:H", + "cvss_base_score": "7.5", + "cvss_vector": "AV:N/AC:L/Au:N/C:P/I:P/A:P", + "description": "Detailed description of the vulnerability.", + "exploit_available": "true", + "exploitability_ease": "Exploits are available", + "id": "vs-scan-0001", + "ip_string": "198.51.100.5", + "owner": "TESTORG1", + "plugin_family": "General", + "plugin_id": "12345", + "plugin_name": "Example Remote Code Execution", + "port": 443, + "port_protocol": "tcp", + "risk_factor": "Critical", + "see_also": "https://example.com/advisory/1", + "service": "https", + "solution": "Apply vendor patch X.", + "source": "nessus", + "synopsis": "The remote host is affected by a remote code execution vulnerability.", + "vuln_detection_timestamp": "2026-08-20T12:00:00+00:00", + "xref": "CWE:79" + }, + { + "cve_string": null, + "cvss3_base_score": null, + "cvss3_temporal_score": null, + "cvss3_vector": null, + "cvss_base_score": null, + "cvss_vector": null, + "description": "Telnet is enabled and accepts weak credentials.", + "exploit_available": null, + "exploitability_ease": null, + "id": "vs-scan-0002", + "ip_string": "198.51.100.5", + "owner": "TESTORG1", + "plugin_family": "Service detection", + "plugin_id": "54321", + "plugin_name": "Weak Telnet Auth", + "port": 23, + "port_protocol": "tcp", + "risk_factor": "Medium", + "see_also": null, + "service": "telnet", + "solution": "Disable telnet.", + "source": "nmap", + "synopsis": "Telnet service allows weak authentication.", + "vuln_detection_timestamp": "2026-08-18T00:00:00+00:00", + "xref": null + } +] diff --git a/open-cti/connectors/vs-vulnscan-enrichment/tests/test_connector_dry_run.py b/open-cti/connectors/vs-vulnscan-enrichment/tests/test_connector_dry_run.py new file mode 100644 index 000000000..82d04a54d --- /dev/null +++ b/open-cti/connectors/vs-vulnscan-enrichment/tests/test_connector_dry_run.py @@ -0,0 +1,220 @@ +"""Loop 2 (OpenCTI-connector.md §9b): the real lookup->map->bundle pipeline against fixtures. + +Uses a stub in place of OpenCTIConnectorHelper so no network/queue is touched at all. Calls +`connector.process_message()` directly with a synthetic event dict shaped exactly like what +pycti's `ListenQueue._data_handler` hands a real INTERNAL_ENRICHMENT connector (confirmed +against the installed pycti source) -- this *is* the dry-run equivalent of a real "Enrich" click, +since `listen()` itself just blocks on RabbitMQ and can't be exercised here. +""" + +# Standard Python Libraries +import json +import os +import sys + +sys.path.insert(0, os.path.join(os.path.dirname(__file__), "..")) + +# Third-Party Libraries +from pycti import Vulnerability as OpenCTIVulnerability # noqa: E402 +from pycti.utils.opencti_logger import logger as pycti_logger # noqa: E402 +import pytest # noqa: E402 + +# First-Party +from src.config import Config # noqa: E402 +from src.connector import VsVulnscanEnrichmentConnector # noqa: E402 +import stix2 # noqa: E402 + +# stix2 validates id shape strictly (--) -- these need to be real, well-formed ids +# for anything that becomes a Note's object_refs entry (found by running these tests, same +# lesson as connector A's test_mapping.py). +_IP_ENTITY_ID = stix2.IPv4Address(value="198.51.100.5").id +_VULN_ENTITY_ID = stix2.Vulnerability( + id=OpenCTIVulnerability.generate_id("CVE-2024-1234"), name="CVE-2024-1234" +).id + + +class StubHelper: + """Just enough of OpenCTIConnectorHelper's surface for process_message() to run against. + + connector_logger deliberately uses pycti's *real* AppLogger, not a plain stdlib + logging.getLogger() -- same reasoning as connectors A/D's stubs (§10i): a stdlib logger + would silently tolerate call-signature mistakes this dry-run test is specifically meant to + catch. + """ + + def __init__(self): + """Start with empty capture state -- nothing sent yet, no state stored yet.""" + self.connector_logger = pycti_logger(level=20, json_logging=False)( + "stub_helper" + ) + self.sent_bundles = [] + self._state = {} + + def get_state(self): + """Return the current stored state, mirroring the real helper's equivalent method.""" + return self._state + + def set_state(self, state): + """Persist state, mirroring the real helper's equivalent method.""" + self._state = state + + @staticmethod + def stix2_create_bundle(items): + """Delegate to pycti's real bundle serialization rather than reimplementing it.""" + # Third-Party Libraries + from pycti import OpenCTIConnectorHelper + + return OpenCTIConnectorHelper.stix2_create_bundle(items) + + def send_stix2_bundle(self, bundle, **kwargs): + """Capture the bundle instead of sending it -- this *is* the dry-run boundary.""" + self.sent_bundles.append(bundle) + return [bundle] + + +class FailingSendHelper(StubHelper): + """StubHelper whose send_stix2_bundle() raises instead of capturing, on demand. + + Models the failed-send/retry path: `fail_sends` starts True so the first + process_message() call fails after state has already been read but before any bundle + is actually accepted, then flips to False so a subsequent retry goes through normally. + """ + + def __init__(self): + """Start in failing mode -- the first send this test drives must raise.""" + super().__init__() + self.fail_sends = True + + def send_stix2_bundle(self, bundle, **kwargs): + """Raise instead of capturing while `fail_sends` is set, else behave like the base stub.""" + if self.fail_sends: + raise RuntimeError("simulated OpenCTI bundle submission failure") + return super().send_stix2_bundle(bundle, **kwargs) + + +def build_test_connector(helper=None): + """Build a VsVulnscanEnrichmentConnector wired to fixtures and the given (or a fresh) stub.""" + config = Config( + raw={ + "vs_vulnscan_enrichment": { + "is_local": True, + "local_fixture_dir": os.path.join( + os.path.dirname(__file__), "fixtures" + ), + "tlp_marking": "TLP:AMBER", + } + } + ) + return VsVulnscanEnrichmentConnector(config=config, helper=helper or StubHelper()) + + +def _ip_event(entity_id=_IP_ENTITY_ID): + """Build a synthetic enrichment event dict shaped like pycti's real ListenQueue output.""" + return { + "entity_id": entity_id, + "entity_type": "IPv4-Addr", + "stix_entity": {"id": entity_id, "type": "ipv4-addr", "value": "198.51.100.5"}, + } + + +def _vulnerability_event(entity_id=_VULN_ENTITY_ID): + """Build a synthetic enrichment event dict for a Vulnerability-type trigger.""" + return { + "entity_id": entity_id, + "entity_type": "Vulnerability", + "stix_entity": { + "id": entity_id, + "type": "vulnerability", + "name": "CVE-2024-1234", + }, + } + + +def test_ip_entity_enrichment_produces_a_note_with_both_matched_findings(): + """198.51.100.5 matches both fixture rows -- both must show up in one Note.""" + connector = build_test_connector() + + message = connector.process_message(_ip_event()) + + assert "2 VS scanner finding" in message + assert len(connector.helper.sent_bundles) == 1 + bundle = json.loads(connector.helper.sent_bundles[0]) + notes = [obj for obj in bundle["objects"] if obj["type"] == "note"] + assert len(notes) == 1 + assert "Example Remote Code Execution" in notes[0]["content"] + assert "Weak Telnet Auth" in notes[0]["content"] + + +def test_vulnerability_entity_enrichment_matches_only_the_right_cve(): + """CVE-2024-1234 should only pull in the one fixture row for that CVE, not both.""" + connector = build_test_connector() + + message = connector.process_message(_vulnerability_event()) + + assert "1 VS scanner finding" in message + bundle = json.loads(connector.helper.sent_bundles[-1]) + notes = [obj for obj in bundle["objects"] if obj["type"] == "note"] + assert "Example Remote Code Execution" in notes[0]["content"] + assert "Weak Telnet Auth" not in notes[0]["content"] + + +def test_no_matching_vuln_scans_sends_no_bundle(): + """§9 queue discipline: nothing found means nothing worth putting on the queue.""" + connector = build_test_connector() + event = _ip_event() + event["stix_entity"]["value"] = "203.0.113.9" # matches no fixture row + + message = connector.process_message(event) + + assert "No VS scanner-level detail found" in message + assert connector.helper.sent_bundles == [] + + +def test_note_id_stays_pinned_across_repeated_enrichment(): + """§10a: re-enriching the same entity must update the same Note, not create a new one.""" + connector = build_test_connector() + + connector.process_message(_ip_event()) + first_note_id = connector.helper.get_state()["note_ids"][_IP_ENTITY_ID] + + connector.process_message(_ip_event()) + second_note_id = connector.helper.get_state()["note_ids"][_IP_ENTITY_ID] + + assert first_note_id == second_note_id + + +def test_failed_send_does_not_pin_a_note_id_and_retry_still_succeeds(): + """A failed send_stix2_bundle() must not leave state pointing at a Note that was never created. + + Otherwise a retry reuses that id assuming the object already exists (the bug this test + guards against: state was previously persisted before the send was confirmed). + """ + helper = FailingSendHelper() + connector = build_test_connector(helper=helper) + + with pytest.raises(RuntimeError): + connector.process_message(_ip_event()) + + # The failed attempt must not have pinned a note id -- nothing was actually created. + assert helper.get_state().get("note_ids", {}).get(_IP_ENTITY_ID) is None + assert helper.sent_bundles == [] + + # A retry (send now succeeding) must go through cleanly and pin a note id afterwards. + helper.fail_sends = False + message = connector.process_message(_ip_event()) + + assert "2 VS scanner finding" in message + assert len(helper.sent_bundles) == 1 + assert helper.get_state()["note_ids"][_IP_ENTITY_ID] is not None + + +def test_unexpected_entity_type_raises_rather_than_silently_no_ops(): + """A scope mismatch is a configuration problem worth surfacing, not swallowing.""" + connector = build_test_connector() + event = { + "entity_id": "domain-name--x", + "entity_type": "Domain-Name", + "stix_entity": {"value": "example.com"}, + } + with pytest.raises(ValueError): + connector.process_message(event) diff --git a/open-cti/connectors/vs-vulnscan-enrichment/tests/test_mapping.py b/open-cti/connectors/vs-vulnscan-enrichment/tests/test_mapping.py new file mode 100644 index 000000000..b582a81dd --- /dev/null +++ b/open-cti/connectors/vs-vulnscan-enrichment/tests/test_mapping.py @@ -0,0 +1,124 @@ +"""Loop 1 (OpenCTI-connector.md §9b): pure mapping unit tests, no DB/OpenCTI/queue at all.""" + +# Standard Python Libraries +from decimal import Decimal +import os +import sys + +sys.path.insert(0, os.path.join(os.path.dirname(__file__), "..")) + +# Third-Party Libraries +from pycti import Identity as OpenCTIIdentity # noqa: E402 + +# First-Party +from src import mapping # noqa: E402 +import stix2 # noqa: E402 + +_AUTHOR_ID = OpenCTIIdentity.generate_id("Test Author", "system") +_MARKING_ID = stix2.TLP_AMBER.id +_ENTITY_ID = stix2.IPv4Address(value="198.51.100.5").id + + +def test_build_author_is_a_system_identity(): + """The author Identity must be class=system and deterministic by name.""" + obj = mapping.build_author("CISA VulnScanning") + assert obj.identity_class == "system" + assert obj.id == OpenCTIIdentity.generate_id("CISA VulnScanning", "system") + + +def test_build_note_content_lists_every_matched_row(): + """Every VulnScan row must show up in the rendered content, not just the first.""" + rows = [ + {"id": "v1", "plugin_name": "Finding A", "port": 443}, + {"id": "v2", "plugin_name": "Finding B", "port": 22}, + ] + content = mapping.build_note_content(rows) + assert "Finding A" in content + assert "Finding B" in content + + +def test_build_note_content_handles_decimal_cvss3_score(): + """cvss3_base_score is a DecimalField, not a plain float. + + Verified against a real postgres:17 container (§9b Loop 4) that this comes back as + decimal.Decimal on a live run. + """ + rows = [ + {"id": "v1", "plugin_name": "Finding A", "cvss3_base_score": Decimal("9.80")} + ] + content = mapping.build_note_content(rows) + assert "9.80" in content + + +def test_build_note_content_empty_when_no_rows(): + """Zero matched rows still renders something, not an empty string.""" + assert mapping.build_note_content([]) != "" + + +def test_build_note_abstract_reports_the_count(): + """The abstract must include the exact count of matched findings.""" + assert ( + mapping.build_note_abstract([{}, {}, {}]) + == "VS scanner enrichment (3 finding(s))" + ) + + +def test_build_external_references_includes_only_url_looking_see_also(): + """Free-text see_also/xref with no confirmed format -- only real URLs become a reference.""" + rows = [ + {"id": "v1", "see_also": "https://example.com/advisory/1"}, + {"id": "v2", "see_also": "not a url, just scanner notes"}, + {"id": "v3", "see_also": None}, + ] + refs = mapping.build_external_references(rows) + assert len(refs) == 1 + assert refs[0].url == "https://example.com/advisory/1" + + +def test_build_external_references_returns_none_when_nothing_url_shaped(): + """No URL-shaped see_also values means None, not an empty list.""" + rows = [{"id": "v1", "see_also": "just text"}] + assert mapping.build_external_references(rows) is None + + +def test_build_note_reuses_existing_id_when_pinned(): + """A pinned id must be returned verbatim, not recomputed from content. + + §10a idempotency: pycti.Note.generate_id() hashes content, so recomputing on every + re-enrichment would produce a *different* Note id the moment scanner data changes at all. + """ + rows = [{"id": "v1", "plugin_name": "Finding A"}] + # A real, differently-computed note id -- proves build_note() returns *this* id verbatim. + # Third-Party Libraries + from pycti import Note as OpenCTINote + + pinned_id = OpenCTINote.generate_id(created=None, content="something else entirely") + note = mapping.build_note( + _ENTITY_ID, rows, _AUTHOR_ID, _MARKING_ID, existing_id=pinned_id + ) + assert note.id == pinned_id + + +def test_build_note_id_changes_with_content_when_not_pinned(): + """Shows generate_id() alone WOULD change id when content changes -- why connector.py pins it.""" + rows_a = [{"id": "v1", "plugin_name": "Finding A"}] + rows_b = [{"id": "v1", "plugin_name": "Finding A -- rescored"}] + note_a = mapping.build_note(_ENTITY_ID, rows_a, _AUTHOR_ID, _MARKING_ID) + note_b = mapping.build_note(_ENTITY_ID, rows_b, _AUTHOR_ID, _MARKING_ID) + assert note_a.id != note_b.id # not pinned here -- connector.py is what pins it + + +def test_build_note_object_refs_points_at_the_triggered_entity(): + """object_refs is what actually attaches this Note to the enriched entity.""" + rows = [{"id": "v1", "plugin_name": "Finding A"}] + note = mapping.build_note(_ENTITY_ID, rows, _AUTHOR_ID, _MARKING_ID) + assert note.object_refs == [_ENTITY_ID] + + +def test_dedupe_bundle_objects_collapses_repeated_ids(): + """Two STIX objects sharing an id must collapse to one in dedupe_bundle_objects().""" + a = stix2.IPv4Address(value="10.0.0.1") + a_again = stix2.IPv4Address(value="10.0.0.1") + b = stix2.IPv4Address(value="10.0.0.2") + result = mapping.dedupe_bundle_objects([a, a_again, b]) + assert len(result) == 2 diff --git a/open-cti/docker-compose.yml b/open-cti/docker-compose.yml index 257706353..480299369 100644 --- a/open-cti/docker-compose.yml +++ b/open-cti/docker-compose.yml @@ -144,7 +144,17 @@ services: opencti: image: opencti/platform:latest + # Zscaler TLS-inspects outbound HTTPS on this network -- opencti/platform is Node.js + # (confirmed via its own Dockerfile, node:22-alpine), which trusts extra CAs via + # NODE_EXTRA_CA_CERTS. Genuinely additive (unlike Python's REQUESTS_CA_BUNDLE/ + # SSL_CERT_FILE below on the connector services), so the bare zscaler.pem is enough + # here -- no combined bundle needed. See + # connectors/vs-org-bootstrap/README.md "TLS interception" -> + # "Non-custom connectors". + volumes: + - "${ZSCALER_CERT_DIR:-/opt/open-cti/tls}/zscaler.pem:/etc/ssl/certs/zscaler.pem:ro" environment: + - NODE_EXTRA_CA_CERTS=/etc/ssl/certs/zscaler.pem - NODE_OPTIONS=--max-old-space-size=8096 - APP__PORT=8080 - APP__BASE_URL=${OPENCTI_EXTERNAL_SCHEME}://${OPENCTI_HOST}:${OPENCTI_PORT} @@ -312,17 +322,31 @@ services: condition: service_healthy connector-import-external-reference: image: opencti/connector-import-external-reference:latest + # Zscaler TLS-inspects outbound HTTPS on this network -- this connector's whole job + # is fetching arbitrary external URLs, so it needs the same trust every custom VS + # connector already has. Python + REQUESTS_CA_BUNDLE/SSL_CERT_FILE, not additive + # like Node's NODE_EXTRA_CA_CERTS -- see + # connectors/vs-org-bootstrap/README.md "TLS interception" -> "Non-custom + # connectors" for the combined-ca-bundle.pem generation step and why it's needed. + volumes: + - "${ZSCALER_CERT_DIR:-/opt/open-cti/tls}/combined-ca-bundle.pem:/etc/ssl/certs/zscaler-combined-ca-bundle.pem:ro" environment: - OPENCTI_URL=http://opencti:8080 - OPENCTI_TOKEN=${OPENCTI_ADMIN_TOKEN} - CONNECTOR_ID=${CONNECTOR_IMPORT_EXTERNAL_REFERENCE_ID} - CONNECTOR_NAME=ImportExternalReference + - REQUESTS_CA_BUNDLE=/etc/ssl/certs/zscaler-combined-ca-bundle.pem + - SSL_CERT_FILE=/etc/ssl/certs/zscaler-combined-ca-bundle.pem restart: always depends_on: opencti: condition: service_healthy connector-shodan: image: opencti/connector-shodan:latest + # See connector-import-external-reference above -- same Zscaler trust setup, for + # the outbound Shodan API calls. + volumes: + - "${ZSCALER_CERT_DIR:-/opt/open-cti/tls}/combined-ca-bundle.pem:/etc/ssl/certs/zscaler-combined-ca-bundle.pem:ro" environment: - OPENCTI_URL=http://opencti:8080 - OPENCTI_TOKEN=${CONNECTOR_SHODAN_API_KEY} @@ -330,6 +354,8 @@ services: - CONNECTOR_SCOPE=IPv4-Addr,Indicator - CONNECTOR_AUTO=false - CONNECTOR_LOG_LEVEL=error + - REQUESTS_CA_BUNDLE=/etc/ssl/certs/zscaler-combined-ca-bundle.pem + - SSL_CERT_FILE=/etc/ssl/certs/zscaler-combined-ca-bundle.pem restart: always depends_on: opencti: @@ -337,6 +363,10 @@ services: connector-cve: image: opencti/connector-cve:latest + # See connector-import-external-reference above -- same Zscaler trust setup, for + # the outbound NVD (services.nvd.nist.gov) calls. + volumes: + - "${ZSCALER_CERT_DIR:-/opt/open-cti/tls}/combined-ca-bundle.pem:/etc/ssl/certs/zscaler-combined-ca-bundle.pem:ro" environment: - OPENCTI_URL=http://opencti:8080 - OPENCTI_TOKEN=${CONNECTOR_CVE_API_KEY} @@ -350,6 +380,8 @@ services: - CVE_INTERVAL=2 # In hours, minimum 2 recommended by NIST - CVE_BASE_URL=https://services.nvd.nist.gov/rest/json/cves - MAINTAIN_DATA=true + - REQUESTS_CA_BUNDLE=/etc/ssl/certs/zscaler-combined-ca-bundle.pem + - SSL_CERT_FILE=/etc/ssl/certs/zscaler-combined-ca-bundle.pem # - CVE_MAX_DATE_RANGE=120 # - CVE_MAINTAIN_DATA=true # - CVE_PULL_HISTORY=false @@ -361,6 +393,10 @@ services: connector-censys-enrichment: image: opencti/connector-censys-enrichment:latest + # See connector-import-external-reference above -- same Zscaler trust setup, for + # the outbound Censys API calls. + volumes: + - "${ZSCALER_CERT_DIR:-/opt/open-cti/tls}/combined-ca-bundle.pem:/etc/ssl/certs/zscaler-combined-ca-bundle.pem:ro" environment: - OPENCTI_URL=http://opencti:8080 - OPENCTI_TOKEN=${CONNECTOR_CENSYS_API_KEY} @@ -373,10 +409,16 @@ services: - CENSYS_ENRICHMENT_ORGANISATION_ID=${CENSYS_ORG_ID} - CENSYS_ENRICHMENT_TOKEN=${CENSYS_API_KEY} - CENSYS_ENRICHMENT_MAX_TLP=TLP:AMBER + - REQUESTS_CA_BUNDLE=/etc/ssl/certs/zscaler-combined-ca-bundle.pem + - SSL_CERT_FILE=/etc/ssl/certs/zscaler-combined-ca-bundle.pem restart: always connector-vulncheck: image: opencti/connector-vulncheck:latest + # See connector-import-external-reference above -- same Zscaler trust setup, for + # the outbound VulnCheck API calls. + volumes: + - "${ZSCALER_CERT_DIR:-/opt/open-cti/tls}/combined-ca-bundle.pem:/etc/ssl/certs/zscaler-combined-ca-bundle.pem:ro" environment: - OPENCTI_URL=http://opencti:8080 - OPENCTI_TOKEN=${CONNECTOR_VULNCHECK_API_KEY} @@ -387,10 +429,16 @@ services: - CONNECTOR_LOG_LEVEL=error - CONNECTOR_AUTO=false - VULNCHECK_API_KEY=${VULNCHECK_API_KEY} + - REQUESTS_CA_BUNDLE=/etc/ssl/certs/zscaler-combined-ca-bundle.pem + - SSL_CERT_FILE=/etc/ssl/certs/zscaler-combined-ca-bundle.pem restart: always connector-cisa-known-exploited-vulnerabilities: image: opencti/connector-cisa-known-exploited-vulnerabilities:latest + # See connector-import-external-reference above -- same Zscaler trust setup, for + # the outbound cisa.gov feed fetch. + volumes: + - "${ZSCALER_CERT_DIR:-/opt/open-cti/tls}/combined-ca-bundle.pem:/etc/ssl/certs/zscaler-combined-ca-bundle.pem:ro" environment: - OPENCTI_URL=http://opencti:8080 - OPENCTI_TOKEN=${CONNECTOR_CISA_KEV_API_KEY} @@ -403,10 +451,21 @@ services: - CISA_CREATE_INFRASTRUCTURES=false - CISA_KEV_FLAG_ONLY=false - CISA_TLP=TLP:CLEAR + - REQUESTS_CA_BUNDLE=/etc/ssl/certs/zscaler-combined-ca-bundle.pem + - SSL_CERT_FILE=/etc/ssl/certs/zscaler-combined-ca-bundle.pem restart: always connector-qualys-cve-enrichment: image: opencti/connector-qualys-cve-enrichment:latest + # See connector-import-external-reference above -- same Zscaler trust setup, for + # the outbound Qualys API calls. QUALYS_SSL_VERIFY=false (below) was this + # connector's previous workaround for the exact same underlying problem -- + # disabling verification entirely instead of trusting the intercepting CA. Now + # that there's a real trust path, that flag should come back OUT once this is + # confirmed working -- left in place for this change alone so nothing breaks + # before that's verified against a real run. + volumes: + - "${ZSCALER_CERT_DIR:-/opt/open-cti/tls}/combined-ca-bundle.pem:/etc/ssl/certs/zscaler-combined-ca-bundle.pem:ro" environment: - OPENCTI_URL=http://opencti:8080 - OPENCTI_TOKEN=${CONNECTOR_QUALYS_CVE_ENRICHMENT_API_KEY} @@ -420,8 +479,204 @@ services: - CONNECTOR_AUTO=false - QUALYS_CVE_ENRICHMENT_USERNAME=${QUALYS_API_USERNAME} - QUALYS_CVE_ENRICHMENT_PASSWORD=${QUALYS_API_PASSWORD} + - REQUESTS_CA_BUNDLE=/etc/ssl/certs/zscaler-combined-ca-bundle.pem + - SSL_CERT_FILE=/etc/ssl/certs/zscaler-combined-ca-bundle.pem restart: always + ########################### + # VS CUSTOM CONNECTORS # + # See connectors/*/README.md and ../OpenCTI-connector.md for design/build order + # (§8: this one first -- Connectors A/C both depend on it for org Identity + # resolution). + ########################### + + connector-vs-org-bootstrap: + build: + context: ./connectors/vs-org-bootstrap + # Zscaler root CA, referenced directly from runtime state (/opt/open-cti, + # never the replaceable git checkout) via a named build context -- never + # copied into the repo. See connectors/vs-org-bootstrap/README.md "TLS + # interception". Requires BuildKit; there's a known regression for this in + # Compose 2.24.0 (docker/compose#11351) -- confirm this actually resolves + # on `docker compose build` before trusting it blindly. + additional_contexts: + - "zscaler_cert=${ZSCALER_CERT_DIR:-/opt/open-cti/tls}" + environment: + - OPENCTI_URL=http://opencti:8080 + # NOT YET WIRED: needs a real OpenCTI service-account token delivered the same + # way every other connector's *_API_KEY is (bootstrap.sh + an SSM SecureString + + # var.open_cti_secret_keys in infrastructure/vars.tf). See OpenCTI-connector.md + # next steps. + - OPENCTI_TOKEN=${CONNECTOR_VS_ORG_BOOTSTRAP_API_KEY} + - CONNECTOR_ID=d9479ad9-6787-471f-a31c-9e4d0288f289 + - CONNECTOR_NAME=VS Organization & CIDR Bootstrap + - CONNECTOR_TYPE=EXTERNAL_IMPORT + - CONNECTOR_SCOPE=identity,location + - CONNECTOR_LOG_LEVEL=info + - CONNECTOR_DURATION_PERIOD=PT6H + # mini_data_lake connectivity (OpenCTI-connector.md §2) -- IAM auth proven + # end-to-end 2026-08-26 (role, rds_iam grant, table grants, real token + # generation all confirmed). DB_HOST/AWS_REGION flow from Terraform + # (aws_db_instance.db.address/var.aws_region) via env.deploy -- see + # infrastructure/open_cti_user_data_env.sh.tpl and open-cti/bootstrap.sh -- + # deliberately not env.static, since a DB endpoint is per-environment, not + # something that belongs in a file meant to be identical everywhere. + - VS_ORG_BOOTSTRAP_DB_HOST=${VS_ORG_BOOTSTRAP_DB_HOST} + # Confirmed (2026-08-27) via current_database() through + # crossfeed-staging-bastion -- the real deployed name, not the Django + # settings/connector default ("mini_data_lake", kept as the architectural + # name for this data store throughout comments/docs; this is just the + # literal staging Postgres identifier). + - VS_ORG_BOOTSTRAP_DB_NAME=cyhy_mini_data_lake_staging + - VS_ORG_BOOTSTRAP_DB_USER=open_cti + - VS_ORG_BOOTSTRAP_DB_USE_IAM_AUTH=true + - VS_ORG_BOOTSTRAP_AWS_REGION=${VS_ORG_BOOTSTRAP_AWS_REGION} + # §9c -- empty on purpose; a real value (or ALLOW_UNSCOPED_RUN=true) must be set + # explicitly, dev or production, per config.py's fail-closed validation. + - VS_ORG_BOOTSTRAP_ORG_ACRONYM_ALLOWLIST=${VS_ORG_BOOTSTRAP_ORG_ACRONYM_ALLOWLIST:-} + - VS_ORG_BOOTSTRAP_ALLOW_UNSCOPED_RUN=${VS_ORG_BOOTSTRAP_ALLOW_UNSCOPED_RUN:-false} + # §10c -- deliberately empty; real data-handling policy decision still pending. + - VS_ORG_BOOTSTRAP_TLP_MARKING=${VS_ORG_BOOTSTRAP_TLP_MARKING:-} + restart: always + + connector-vs-ticket-ingestion: + build: + context: ./connectors/vs-ticket-ingestion + # Same Zscaler root CA setup as connector-vs-org-bootstrap -- see that service's + # comment above and connectors/vs-ticket-ingestion/README.md "TLS interception". + # Property of the network, not of either connector individually. + additional_contexts: + - "zscaler_cert=${ZSCALER_CERT_DIR:-/opt/open-cti/tls}" + environment: + - OPENCTI_URL=http://opencti:8080 + # NOT YET WIRED: same as connector-vs-org-bootstrap -- needs a real OpenCTI + # service-account token delivered via bootstrap.sh + an SSM SecureString + + # var.open_cti_secret_keys. + - OPENCTI_TOKEN=${CONNECTOR_VS_TICKET_INGESTION_API_KEY} + - CONNECTOR_ID=d110abb6-cb79-4ef7-b34d-76d769276356 + - CONNECTOR_NAME=VS Ticket Ingestion + - CONNECTOR_TYPE=EXTERNAL_IMPORT + - CONNECTOR_SCOPE=ipv4-addr,ipv6-addr,vulnerability + - CONNECTOR_LOG_LEVEL=info + # §7a -- should not poll tighter than VulnScanningSync's own cadence (§5, + # still unresolved). + - CONNECTOR_DURATION_PERIOD=P1D + # mini_data_lake connectivity (OpenCTI-connector.md §2) -- same DB_HOST/AWS_REGION + # already flowing from Terraform for connector-vs-org-bootstrap (env.deploy, not + # env.static -- see infrastructure/open_cti_user_data_env.sh.tpl and + # open-cti/bootstrap.sh). + - VS_TICKET_INGESTION_DB_HOST=${VS_ORG_BOOTSTRAP_DB_HOST} + - VS_TICKET_INGESTION_DB_NAME=cyhy_mini_data_lake_staging + - VS_TICKET_INGESTION_DB_USER=open_cti + - VS_TICKET_INGESTION_DB_USE_IAM_AUTH=true + - VS_TICKET_INGESTION_AWS_REGION=${VS_ORG_BOOTSTRAP_AWS_REGION} + # §9c -- empty on purpose; a real value (or ALLOW_UNSCOPED_RUN=true) must be set + # explicitly, dev or production, per config.py's fail-closed validation. + - VS_TICKET_INGESTION_ORG_ACRONYM_ALLOWLIST=${VS_TICKET_INGESTION_ORG_ACRONYM_ALLOWLIST:-} + - VS_TICKET_INGESTION_ALLOW_UNSCOPED_RUN=${VS_TICKET_INGESTION_ALLOW_UNSCOPED_RUN:-false} + # §9c dev-iteration levers, both empty/default by default (no behavior change + # from an unbounded first pull -- see src/config.py). MAX_ROWS_PER_RUN paginates + # slower but never skips data; LOOKBACK_DAYS bounds a fresh/reset connector's + # *first* poll to the last N days, which is what actually keeps a first run + # against a history-heavy org fast -- set both here (not just in config.yml) + # when iterating against this compose stack. + - VS_TICKET_INGESTION_MAX_ROWS_PER_RUN=${VS_TICKET_INGESTION_MAX_ROWS_PER_RUN:-5000} + - VS_TICKET_INGESTION_LOOKBACK_DAYS=${VS_TICKET_INGESTION_LOOKBACK_DAYS:-} + # §10c -- should match the other VS connectors' TLP marking once the policy call + # is made; deliberately empty by default, same as connector-vs-org-bootstrap. + # Bug fixed 2026-09-02: this used to read ${VS_ORG_BOOTSTRAP_TLP_MARKING:-}, this + # connector's own env var was never actually substituted. + - VS_TICKET_INGESTION_TLP_MARKING=${VS_TICKET_INGESTION_TLP_MARKING:-} + restart: always + + connector-vs-vulnscan-enrichment: + build: + context: ./connectors/vs-vulnscan-enrichment + # Same Zscaler root CA setup as the other VS connectors -- see their service + # comments above and connectors/vs-vulnscan-enrichment/README.md "TLS + # interception". Property of the network, not of any one connector. + additional_contexts: + - "zscaler_cert=${ZSCALER_CERT_DIR:-/opt/open-cti/tls}" + environment: + - OPENCTI_URL=http://opencti:8080 + # NOT YET WIRED: same as the other VS connectors -- needs a real OpenCTI + # service-account token delivered via bootstrap.sh + an SSM SecureString + + # var.open_cti_secret_keys. + - OPENCTI_TOKEN=${CONNECTOR_VS_VULNSCAN_ENRICHMENT_API_KEY} + - CONNECTOR_ID=0e7d7782-0835-40a9-a2e4-29b0b9914efa + - CONNECTOR_NAME=VS VulnScan Enrichment + - CONNECTOR_TYPE=INTERNAL_ENRICHMENT + - CONNECTOR_SCOPE=IPv4-Addr,IPv6-Addr,Vulnerability + - CONNECTOR_LOG_LEVEL=info + # §7b -- manual "Enrich" trigger, matching every other enrichment connector + # already running here (connector-censys-enrichment, etc.). Auto-triggering + # would mean one VulnScan lookup per Ticket per connector-vs-ticket-ingestion + # poll, hitting the same primary DB that connector just queried. + - CONNECTOR_AUTO=false + # mini_data_lake connectivity (OpenCTI-connector.md §2) -- same DB_HOST/AWS_REGION + # already flowing from Terraform for connector-vs-org-bootstrap (env.deploy, not + # env.static -- see infrastructure/open_cti_user_data_env.sh.tpl and + # open-cti/bootstrap.sh). + - VS_VULNSCAN_ENRICHMENT_DB_HOST=${VS_ORG_BOOTSTRAP_DB_HOST} + - VS_VULNSCAN_ENRICHMENT_DB_NAME=cyhy_mini_data_lake_staging + - VS_VULNSCAN_ENRICHMENT_DB_USER=open_cti + - VS_VULNSCAN_ENRICHMENT_DB_USE_IAM_AUTH=true + - VS_VULNSCAN_ENRICHMENT_AWS_REGION=${VS_ORG_BOOTSTRAP_AWS_REGION} + # §9c -- bounds how many VulnScan rows go into one Note per triggered entity; + # tunable here the same way VS_TICKET_INGESTION_MAX_ROWS_PER_RUN is. + - VS_VULNSCAN_ENRICHMENT_MAX_ROWS_PER_ENTITY=${VS_VULNSCAN_ENRICHMENT_MAX_ROWS_PER_ENTITY:-50} + # §10c -- should match the other VS connectors' TLP marking once the policy call + # is made; deliberately empty by default, same as the other VS connectors. Bug + # fixed 2026-09-02: this used to read ${VS_ORG_BOOTSTRAP_TLP_MARKING:-}, this + # connector's own env var was never actually substituted. + - VS_VULNSCAN_ENRICHMENT_TLP_MARKING=${VS_VULNSCAN_ENRICHMENT_TLP_MARKING:-} + restart: always + + connector-vs-port-inventory: + build: + context: ./connectors/vs-port-inventory + # Same Zscaler root CA setup as the other VS connectors -- see their service + # comments above and connectors/vs-port-inventory/README.md "TLS interception". + # Property of the network, not of any one connector. + additional_contexts: + - "zscaler_cert=${ZSCALER_CERT_DIR:-/opt/open-cti/tls}" + environment: + - OPENCTI_URL=http://opencti:8080 + # NOT YET WIRED: same as the other VS connectors -- needs a real OpenCTI + # service-account token delivered via bootstrap.sh + an SSM SecureString + + # var.open_cti_secret_keys. + - OPENCTI_TOKEN=${CONNECTOR_VS_PORT_INVENTORY_API_KEY} + - CONNECTOR_ID=8ca61ddd-f947-40f7-b4e0-505d61dc612b + - CONNECTOR_NAME=VS Port/Service Inventory + - CONNECTOR_TYPE=EXTERNAL_IMPORT + - CONNECTOR_SCOPE=ipv4-addr,ipv6-addr,network-traffic,software + - CONNECTOR_LOG_LEVEL=info + - CONNECTOR_DURATION_PERIOD=P1D + # mini_data_lake connectivity (OpenCTI-connector.md §2) -- same DB_HOST/AWS_REGION + # already flowing from Terraform for connector-vs-org-bootstrap (env.deploy, not + # env.static -- see infrastructure/open_cti_user_data_env.sh.tpl and + # open-cti/bootstrap.sh). + - VS_PORT_INVENTORY_DB_HOST=${VS_ORG_BOOTSTRAP_DB_HOST} + - VS_PORT_INVENTORY_DB_NAME=cyhy_mini_data_lake_staging + - VS_PORT_INVENTORY_DB_USER=open_cti + - VS_PORT_INVENTORY_DB_USE_IAM_AUTH=true + - VS_PORT_INVENTORY_AWS_REGION=${VS_ORG_BOOTSTRAP_AWS_REGION} + # §9c -- empty on purpose; a real value (or ALLOW_UNSCOPED_RUN=true) must be set + # explicitly, dev or production, per config.py's fail-closed validation. + - VS_PORT_INVENTORY_ORG_ACRONYM_ALLOWLIST=${VS_PORT_INVENTORY_ORG_ACRONYM_ALLOWLIST:-} + - VS_PORT_INVENTORY_ALLOW_UNSCOPED_RUN=${VS_PORT_INVENTORY_ALLOW_UNSCOPED_RUN:-false} + # Bounds the bootstrap poll (still possibly large -- every row the source + # considers current, regardless of age) and every steady-state poll after it. + - VS_PORT_INVENTORY_MAX_ROWS_PER_RUN=${VS_PORT_INVENTORY_MAX_ROWS_PER_RUN:-5000} + # This connector's own copy of LATEST_PORT_SCAN_CUTOFF (vs_port_scans.py, + # default 14) -- staleness is computed locally instead of re-polling. See db.py. + - VS_PORT_INVENTORY_CUTOFF_DAYS=${VS_PORT_INVENTORY_CUTOFF_DAYS:-14} + # §10c -- should match the other VS connectors' TLP marking once the policy call + # is made; deliberately empty by default, same as the other VS connectors. Bug + # fixed 2026-09-02: this used to read ${VS_ORG_BOOTSTRAP_TLP_MARKING:-}, this + # connector's own env var was never actually substituted. + - VS_PORT_INVENTORY_TLP_MARKING=${VS_PORT_INVENTORY_TLP_MARKING:-} + restart: always ########################### # OPENCTI DEFAULT DATA # @@ -429,6 +684,10 @@ services: connector-opencti: image: opencti/connector-opencti:latest + # See connector-import-external-reference (VS CUSTOM CONNECTORS section above) -- + # same Zscaler trust setup, for the outbound fetch of Filigran's reference datasets. + volumes: + - "${ZSCALER_CERT_DIR:-/opt/open-cti/tls}/combined-ca-bundle.pem:/etc/ssl/certs/zscaler-combined-ca-bundle.pem:ro" environment: - OPENCTI_URL=http://opencti:8080 - OPENCTI_TOKEN=${OPENCTI_ADMIN_TOKEN} @@ -437,18 +696,26 @@ services: - CONNECTOR_SCOPE=marking-definition,identity,location - CONNECTOR_AUTO_CREATE_SERVICE_ACCOUNT=true - CONNECTOR_AUTO_CREATE_SERVICE_ACCOUNT_CONFIDENCE_LEVEL=100 + - REQUESTS_CA_BUNDLE=/etc/ssl/certs/zscaler-combined-ca-bundle.pem + - SSL_CERT_FILE=/etc/ssl/certs/zscaler-combined-ca-bundle.pem restart: always depends_on: opencti: condition: service_healthy connector-mitre: image: opencti/connector-mitre:latest + # Same Zscaler trust setup, for the outbound MITRE ATT&CK STIX bundle fetch + # (raw.githubusercontent.com). + volumes: + - "${ZSCALER_CERT_DIR:-/opt/open-cti/tls}/combined-ca-bundle.pem:/etc/ssl/certs/zscaler-combined-ca-bundle.pem:ro" environment: - OPENCTI_URL=http://opencti:8080 - OPENCTI_TOKEN=${OPENCTI_ADMIN_TOKEN} - CONNECTOR_ID=${CONNECTOR_MITRE_ID} - CONNECTOR_AUTO_CREATE_SERVICE_ACCOUNT=true - CONNECTOR_AUTO_CREATE_SERVICE_ACCOUNT_CONFIDENCE_LEVEL=75 + - REQUESTS_CA_BUNDLE=/etc/ssl/certs/zscaler-combined-ca-bundle.pem + - SSL_CERT_FILE=/etc/ssl/certs/zscaler-combined-ca-bundle.pem restart: always depends_on: opencti: @@ -460,7 +727,17 @@ services: xtm-one: image: xtmone/platform:latest + # Zscaler TLS-inspects outbound HTTPS on this network. Runtime here is inferred, + # not directly confirmed (no public Dockerfile for xtmone/platform) -- + # DATABASE_URL's postgresql+asyncpg scheme is a strong Python signal, so treated + # the same as the other Python services (combined bundle, not additive). See + # connectors/vs-org-bootstrap/README.md "TLS interception" -> "Non-custom + # connectors" -- worth confirming against real container logs. + volumes: + - "${ZSCALER_CERT_DIR:-/opt/open-cti/tls}/combined-ca-bundle.pem:/etc/ssl/certs/zscaler-combined-ca-bundle.pem:ro" environment: + - REQUESTS_CA_BUNDLE=/etc/ssl/certs/zscaler-combined-ca-bundle.pem + - SSL_CERT_FILE=/etc/ssl/certs/zscaler-combined-ca-bundle.pem - PLATFORM_MODE=xtm_one - PLATFORM_REGISTRATION_TOKEN=${PLATFORM_REGISTRATION_TOKEN} - BASE_URL=${XTM_ONE_EXTERNAL_SCHEME}://${XTM_ONE_HOST}:${XTM_ONE_PORT} @@ -499,7 +776,12 @@ services: xtm-one-worker: image: xtmone/worker:latest + # Same inferred Python/Zscaler treatment as xtm-one above -- see its comment. + volumes: + - "${ZSCALER_CERT_DIR:-/opt/open-cti/tls}/combined-ca-bundle.pem:/etc/ssl/certs/zscaler-combined-ca-bundle.pem:ro" environment: + - REQUESTS_CA_BUNDLE=/etc/ssl/certs/zscaler-combined-ca-bundle.pem + - SSL_CERT_FILE=/etc/ssl/certs/zscaler-combined-ca-bundle.pem - PLATFORM_MODE=xtm_one - PLATFORM_REGISTRATION_TOKEN=${PLATFORM_REGISTRATION_TOKEN} - ADMIN_EMAIL=${XTM_ONE_ADMIN_EMAIL}