From 095c07819690f8490687a0f2524b6dfcff976aad Mon Sep 17 00:00:00 2001 From: Powlinett Date: Wed, 27 May 2026 17:17:34 +0200 Subject: [PATCH 01/13] deps: declare python-dotenv dependency --- connectors-sdk/pyproject.toml | 1 + 1 file changed, 1 insertion(+) diff --git a/connectors-sdk/pyproject.toml b/connectors-sdk/pyproject.toml index a732b259630..d619dfe9b0c 100644 --- a/connectors-sdk/pyproject.toml +++ b/connectors-sdk/pyproject.toml @@ -12,6 +12,7 @@ dependencies = [ "pycti==7.260706.0", "pydantic>=2.8.2,<3", # ISO pycti https://github.com/OpenCTI-Platform/client-python/blob/master/requirements.txt#L8 "pydantic_settings>= 2.9.1,<3", # Actual minimal version used in the connectors + "python-dotenv>=1.2.2,<2", ] [project.optional-dependencies] From 6cab2e60e52e4682f030717e53174660d8e976c7 Mon Sep 17 00:00:00 2001 From: Powlinett Date: Wed, 27 May 2026 17:19:08 +0200 Subject: [PATCH 02/13] feat: create logging module --- connectors-sdk/connectors_sdk/__init__.py | 5 +- .../connectors_sdk/logging/__init__.py | 1 + .../connectors_sdk/logging/_base_logger.py | 229 ++++++++++++++++++ .../connectors_sdk/logging/logger.py | 30 +++ .../connectors_sdk/logging/sdk_logger.py | 39 +++ 5 files changed, 303 insertions(+), 1 deletion(-) create mode 100644 connectors-sdk/connectors_sdk/logging/__init__.py create mode 100644 connectors-sdk/connectors_sdk/logging/_base_logger.py create mode 100644 connectors-sdk/connectors_sdk/logging/logger.py create mode 100644 connectors-sdk/connectors_sdk/logging/sdk_logger.py diff --git a/connectors-sdk/connectors_sdk/__init__.py b/connectors-sdk/connectors_sdk/__init__.py index 262802b55af..fca8ba1b0bc 100644 --- a/connectors-sdk/connectors_sdk/__init__.py +++ b/connectors-sdk/connectors_sdk/__init__.py @@ -3,13 +3,14 @@ __version__ = "0.1.0" from connectors_sdk.connectors.external_import._work_manager import WorkManager +from connectors_sdk.logging.logger import Logger, logger from connectors_sdk.connectors.external_import.base_data_processor import ( BaseDataProcessor, ) from connectors_sdk.connectors.external_import.external_import_connector import ( ExternalImportConnector, ) -from connectors_sdk.connectors.external_import.logger import ConnectorLogger +from connectors_sdk.logging.logger import Logger from connectors_sdk.settings.annotated_types import ( DatetimeFromIsoString, ListFromString, @@ -34,6 +35,8 @@ from connectors_sdk.states.states import ExternalImportConnectorState __all__ = [ + # Logger + "Logger", # mostly for typing purposes # Base Settings "BaseConnectorSettings", # Base Configs diff --git a/connectors-sdk/connectors_sdk/logging/__init__.py b/connectors-sdk/connectors_sdk/logging/__init__.py new file mode 100644 index 00000000000..b916de263c4 --- /dev/null +++ b/connectors-sdk/connectors_sdk/logging/__init__.py @@ -0,0 +1 @@ +"""Module containing a logger for connectors.""" diff --git a/connectors-sdk/connectors_sdk/logging/_base_logger.py b/connectors-sdk/connectors_sdk/logging/_base_logger.py new file mode 100644 index 00000000000..5cf8fccbdce --- /dev/null +++ b/connectors-sdk/connectors_sdk/logging/_base_logger.py @@ -0,0 +1,229 @@ +"""Base logger using the same format as pycti's logger.""" + +from __future__ import annotations + +import logging +import os +import sys +from abc import ABC, abstractmethod +from pathlib import Path +from typing import Any, ClassVar, Literal, Self + +from pycti.utils.opencti_logger import CustomJsonFormatter + +LOG_LEVELS = { + "DEBUG", + "INFO", + "WARN", + "WARNING", + "ERROR", +} + + +def _json_formatter() -> CustomJsonFormatter: + """Get a new instance of the CustomJsonFormatter. + The format is the same as the one used in pycti's `AppLogger`, in order to maintain consistency. + """ + return CustomJsonFormatter("%(timestamp)s %(level)s %(name)s %(message)s") + + +def _prepare_meta(meta: dict[str, Any] | None = None) -> dict[str, Any] | None: + """Prepare metadata for logging. + The format is the same as the one used in pycti's `AppLogger`, in order to maintain consistency. + + Args: + message: The log message. + meta: Optional metadata dict. + """ + return None if meta is None else {"attributes": meta} + + +class BaseLogger(ABC): + """Base class to set up a logger with a default `StreamHandler` using stderr and pycti's `AppLogger` formatter. + Its log level is determined by connector's configuration, with the following precedence: + 1. `CONNECTOR_LOG_LEVEL` environment variable + 2. `connector.log_level` field in `config.yml` + 3. `CONNECTOR_LOG_LEVEL` field in `.env` file + 4. Default to `"ERROR"` if none of the above is found or if the value is invalid. + + Notes: + - This class reproduces the same API as pycti's `AppLogger`, as this last is not importable from pycti. + Once pycti's `AppLogger` will be made public and available for import, this class should be refactored + to inherit from pycti's `AppLogger` instead of maintaining its own implementation. + """ + + _handlers: ClassVar[list[logging.Handler]] = [] + _log_level: ClassVar[Literal["DEBUG", "INFO", "WARN", "WARNING", "ERROR"]] = "ERROR" + + @classmethod + def __init_subclass__(cls, **kwargs: Any) -> None: + """Setup default logging settings: + - add a default `StreamHandler` to stderr with pycti's JSON formatter. + - set log level according to the connector's configuration. + This method ensures that all subclasses share the same default handlers and log level. + """ + super().__init_subclass__(**kwargs) + + stderr_handler = logging.StreamHandler() + stderr_handler.set_name("default_stderr_handler") + stderr_handler.setFormatter(_json_formatter()) # pycti's formatter + cls._handlers = [stderr_handler] + + connector_log_level = cls._get_connector_log_level() + if connector_log_level: + cls._log_level = connector_log_level + + @abstractmethod + def __init__(self, name: str) -> None: + """Instantiate a logger with the given name. + The logger will be set up with the default handlers and log level defined in the class. + + Args: + name: The name of the logger, which will appear in the log output. + + Notes: + - This method is abstract and should be called via `super().__init__(name)` in subclasses. + - The logger is set up at the `BaseLogger` level, so all subclasses share the same default handlers and log level. + - Subclasses can implement additional logic and/or handlers in their `__init__` method, + but they should not set up `_logger` themselves. + """ + self._logger = logging.getLogger(name) + self._logger.propagate = False + + for handler in self._handlers: + self._logger.addHandler(handler) + + self._logger.setLevel(self._log_level) + + @staticmethod + def _get_connector_main_path() -> Path: + """Locate the main module of the running connector. + This method is used to locate configuration files relative to connector's entrypoint. + + Notes: + - This method assumes that the connector is launched using a file-backed entrypoint + (i.e., `python -m ` or `python `). + - At module import time, `__main__.__file__` might not be available yet, + thus this method should be called at runtime only. + """ + main = sys.modules.get("__main__") + if main and getattr(main, "__file__", None): + return Path(main.__file__).resolve() # type: ignore + + raise RuntimeError( + "Cannot determine connector's location: __main__.__file__ is not available. " + "Ensure the connector is launched using `python -m ` or a file-backed entrypoint." + ) + + @staticmethod + def _get_log_level_from_config_yml() -> str: + """Get connector's log level from `config.yml` file.""" + main_path = BaseLogger._get_connector_main_path() + + config_yml_path = None + config_yml_legacy_path = main_path.parent / "config.yml" + if config_yml_legacy_path.is_file(): + config_yml_path = config_yml_legacy_path + else: + config_yml_new_path = main_path.parent.parent / "config.yml" + if config_yml_new_path.is_file(): + config_yml_path = config_yml_new_path + + if config_yml_path: + import yaml # type: ignore[import-untyped] + + with open(config_yml_path, "r") as f: + config = yaml.safe_load(f) + log_level = config.get("connector", {}).get("log_level") + if isinstance(log_level, str): + return log_level.strip().upper() + + return "" # return empty str for convenience + + @staticmethod + def _get_log_level_from_dot_env() -> str: + """Get connector's log level from `.env` file.""" + main_path = BaseLogger._get_connector_main_path() + + dot_env_path = main_path.parent.parent / ".env" + if dot_env_path.is_file(): + import dotenv + + log_level = dotenv.get_key(dot_env_path, "CONNECTOR_LOG_LEVEL") + if isinstance(log_level, str): + return log_level.strip().upper() + + return "" # return empty str for convenience + + @classmethod + def _get_connector_log_level( + cls, + ) -> Literal["DEBUG", "INFO", "WARN", "WARNING", "ERROR"] | None: + """Get the connector log level from environment variables, config files, or defaults. + If the variable is not set or has an invalid value, return `None`. + """ + log_level = os.getenv("CONNECTOR_LOG_LEVEL", "").strip().upper() + if log_level in LOG_LEVELS: + return log_level # type: ignore[return-value] + + log_level = BaseLogger._get_log_level_from_config_yml() + if log_level in LOG_LEVELS: + return log_level # type: ignore[return-value] + + log_level = BaseLogger._get_log_level_from_dot_env() + if log_level in LOG_LEVELS: + return log_level # type: ignore[return-value] + + return None + + def get_child(self, name: str) -> Self: + """Get a child logger of current instance, with the given name. + + Args: + name: The name of the child logger, which will be appended to the parent logger's name. + + Returns: + A new instance of the same logger class, with the name `.`. + """ + cls = self.__class__ + child_name = f"{self._logger.name}.{name}" + + return cls(name=child_name) + + def debug(self, message: str, meta: dict[str, Any] | None = None) -> None: + """Log a DEBUG message. + + Args: + message: The log message. + meta: Optional metadata dict. + """ + self._logger.debug(message, extra=_prepare_meta(meta)) + + def info(self, message: str, meta: dict[str, Any] | None = None) -> None: + """Log an INFO message. + + Args: + message: The log message. + meta: Optional metadata dict. + """ + self._logger.info(message, extra=_prepare_meta(meta)) + + def warning(self, message: str, meta: dict[str, Any] | None = None) -> None: + """Log a WARNING message. + + Args: + message: The log message. + meta: Optional metadata dict. + """ + self._logger.warning(message, extra=_prepare_meta(meta)) + + def error( + self, message: str | Exception, meta: dict[str, Any] | None = None + ) -> None: + """Log an ERROR message. + + Args: + message: The log message. + meta: Optional metadata dict. + """ + self._logger.error(message, exc_info=True, extra=_prepare_meta(meta)) diff --git a/connectors-sdk/connectors_sdk/logging/logger.py b/connectors-sdk/connectors_sdk/logging/logger.py new file mode 100644 index 00000000000..3fc8b813bc8 --- /dev/null +++ b/connectors-sdk/connectors_sdk/logging/logger.py @@ -0,0 +1,30 @@ +"""Logger to be used within a connector.""" + +from connectors_sdk.logging._base_logger import BaseLogger + + +class Logger(BaseLogger): + """Logger to use within a connector. + Logs to stderr via a `StreamHandler` using `CustomJsonFormatter` (same format as pycti's `AppLogger`). + The log level is determined by connector's configuration, with the following precedence: + 1. `CONNECTOR_LOG_LEVEL` environment variable + 2. `connector.log_level` field in `config.yml` + 3. `CONNECTOR_LOG_LEVEL` field in `.env` file + 4. Default to `"ERROR"` if none of the above is found or if the value is invalid. + + Example: + >>> from connectors_sdk import Logger + >>> logger = Logger("my_connector") + >>> logger.info("This is an info message") + ... # Output to stderr (in JSON format): + ... # { + ... # "timestamp": "2026-01-01T00:00:00Z", + ... # "level": "INFO", + ... # "name": "my_connector", + ... # "message": "This is an info message" + ... # } + """ + + def __init__(self, name: str) -> None: + """Set up logger with default handler and formatter.""" + super().__init__(name=name) diff --git a/connectors-sdk/connectors_sdk/logging/sdk_logger.py b/connectors-sdk/connectors_sdk/logging/sdk_logger.py new file mode 100644 index 00000000000..299c055d5a1 --- /dev/null +++ b/connectors-sdk/connectors_sdk/logging/sdk_logger.py @@ -0,0 +1,39 @@ +"""Internal logger used for logging inside the connectors-sdk private modules.""" + +from __future__ import annotations + +from connectors_sdk.logging._base_logger import BaseLogger + + +class SDKLogger(BaseLogger): + r"""Logger for connectors-sdk internal/private modules. + + Logs immediately to stderr via a `StreamHandler` using `CustomJsonFormatter` (same format as pycti's `AppLogger`). + The log level is determined by connector's configuration (see `BaseLogger` for more details). + + /!\\ This logger is intended to log internally within the connectors-sdk, + it shouldn't be used by connectors directly (for that, see `Logger`). + + Example: + >>> from connectors_sdk.logging.sdk_logger import sdk_logger # anywhere in the codebase, at any time + >>> sdk_logger.info("Works even if pycti is not available yet") + ... # Output to stderr (in JSON format): + ... # { + ... # "timestamp": "2026-01-01T00:00:00Z", + ... # "level": "INFO", + ... # "name": "connectors_sdk", + ... # "message": "Works even if pycti is not available yet" + ... # } + """ + + def __init__(self, name: str = "connectors_sdk") -> None: + """Set up logger with the default `StreamHandler` handler.""" + if not name.startswith("connectors_sdk"): + raise ValueError("SDKLogger name must start with 'connectors_sdk'") + + super().__init__(name=name) + + +# For convenience, this module provides a default logger instance, +# but `SDKLogger` can also be instantiated directly if needed (e.g. for testing purpose). +sdk_logger = SDKLogger() From 78957d13fe7218a53120431b7d932cac29429d14 Mon Sep 17 00:00:00 2001 From: Powlinett Date: Wed, 27 May 2026 17:41:38 +0200 Subject: [PATCH 03/13] feat: replace OpenCTIConnectorHelper's logger by BaseLogger subclasses --- connectors-sdk/connectors_sdk/__init__.py | 2 - .../connectors/external_import/__init__.py | 2 +- .../external_import/_work_manager.py | 37 ++++------ .../external_import/base_data_processor.py | 40 ++++++----- .../external_import_connector.py | 36 +++++++--- .../connectors/external_import/logger.py | 72 ------------------- .../connectors_sdk/settings/base_settings.py | 12 ++++ .../connectors_sdk/states/_base_state.py | 15 ++-- connectors-sdk/tests/test_api.py | 5 +- .../tests/test_connectors/test_logger.py | 35 --------- 10 files changed, 89 insertions(+), 167 deletions(-) delete mode 100644 connectors-sdk/connectors_sdk/connectors/external_import/logger.py delete mode 100644 connectors-sdk/tests/test_connectors/test_logger.py diff --git a/connectors-sdk/connectors_sdk/__init__.py b/connectors-sdk/connectors_sdk/__init__.py index fca8ba1b0bc..3d412f3c111 100644 --- a/connectors-sdk/connectors_sdk/__init__.py +++ b/connectors-sdk/connectors_sdk/__init__.py @@ -3,7 +3,6 @@ __version__ = "0.1.0" from connectors_sdk.connectors.external_import._work_manager import WorkManager -from connectors_sdk.logging.logger import Logger, logger from connectors_sdk.connectors.external_import.base_data_processor import ( BaseDataProcessor, ) @@ -59,7 +58,6 @@ "ExternalImportConnectorState", # Connector base classes "ExternalImportConnector", - "ConnectorLogger", "BaseDataProcessor", "WorkManager", ] diff --git a/connectors-sdk/connectors_sdk/connectors/external_import/__init__.py b/connectors-sdk/connectors_sdk/connectors/external_import/__init__.py index 57b34b9b8c7..c1628d3d3ab 100644 --- a/connectors-sdk/connectors_sdk/connectors/external_import/__init__.py +++ b/connectors-sdk/connectors_sdk/connectors/external_import/__init__.py @@ -1,7 +1,7 @@ """Module containing base classes for connector development. This module provides the foundational classes for building OpenCTI connectors: -- ConnectorLogger: Logging wrapper to avoid direct pycti dependency +- Logger: Logging wrapper to avoid direct pycti dependency - WorkManager: Work lifecycle management (initiate, send bundles, complete) - BaseDataProcessor: Abstract base class for data collection/processing - ExternalImportConnector: Full orchestration for external import connectors diff --git a/connectors-sdk/connectors_sdk/connectors/external_import/_work_manager.py b/connectors-sdk/connectors_sdk/connectors/external_import/_work_manager.py index 5424de89a99..53ffe3d4982 100644 --- a/connectors-sdk/connectors_sdk/connectors/external_import/_work_manager.py +++ b/connectors-sdk/connectors_sdk/connectors/external_import/_work_manager.py @@ -19,11 +19,14 @@ from __future__ import annotations -from typing import Any +from typing import TYPE_CHECKING, Any, ClassVar -from connectors_sdk.connectors.external_import.logger import ConnectorLogger +from connectors_sdk.logging.sdk_logger import sdk_logger from pycti import OpenCTIConnectorHelper +if TYPE_CHECKING: + from connectors_sdk.logging._base_logger import BaseLogger + class _Work: """Manage the lifecycle of a single work in OpenCTI. @@ -37,35 +40,29 @@ class _Work: name: The human-readable name of the work, displayed in the OpenCTI UI. """ + logger: ClassVar[BaseLogger] = sdk_logger.get_child("WorkManager._Work") + def __init__( self, + helper: OpenCTIConnectorHelper, work_id: str, work_name: str, - helper: OpenCTIConnectorHelper, - logger: ConnectorLogger, ) -> None: """Initialize the work context. Args: + helper: The ``OpenCTIConnectorHelper`` instance. work_id: The work ID returned by OpenCTI. work_name: The human-readable name of the work, displayed in the OpenCTI UI. - helper: The ``OpenCTIConnectorHelper`` instance. - logger: The ``ConnectorLogger`` instance for logging. """ self.id = work_id self.name = work_name self._helper = helper - self._logger = logger self._closed = False self._has_sent_bundles = False @classmethod - def create( - cls, - helper: OpenCTIConnectorHelper, - logger: ConnectorLogger, - work_name: str, - ) -> _Work: + def create(cls, helper: OpenCTIConnectorHelper, work_name: str) -> _Work: """Create a new work in OpenCTI and return a ``_Work`` instance. This classmethod encapsulates the OpenCTI API call to initiate a work, @@ -73,7 +70,6 @@ def create( Args: helper: The ``OpenCTIConnectorHelper`` instance. - logger: The ``ConnectorLogger`` instance for logging. work_name: The name of the work, displayed in the OpenCTI UI. Returns: @@ -84,7 +80,7 @@ def create( f"Work '{work_id}' initiated", {"work_name": work_name}, ) - return cls(work_id, work_name, helper, logger) + return cls(helper, work_id, work_name) def send_bundle(self, bundle_objects: list[Any], **kwargs: Any) -> None: """Create a STIX bundle from objects and send it to OpenCTI. @@ -168,7 +164,7 @@ class WorkManager: Example:: - work_manager = WorkManager(helper, logger) + work_manager = WorkManager(helper) with work_manager: work_manager.send(stix_objects, "Import indicators") work_manager.send(more_objects, "Import indicators") # same work @@ -176,18 +172,15 @@ class WorkManager: # work auto-closed """ - def __init__( - self, - helper: OpenCTIConnectorHelper, - ) -> None: + logger: ClassVar[BaseLogger] = sdk_logger.get_child("WorkManager") + + def __init__(self, helper: OpenCTIConnectorHelper) -> None: """Initialize the work manager. Args: helper: The ``OpenCTIConnectorHelper`` instance. - logger: The ``ConnectorLogger`` instance. """ self._helper = helper - self._logger = ConnectorLogger(helper) self._current_work: _Work | None = None self._active = False diff --git a/connectors-sdk/connectors_sdk/connectors/external_import/base_data_processor.py b/connectors-sdk/connectors_sdk/connectors/external_import/base_data_processor.py index d8b07e7c223..5f47e926701 100644 --- a/connectors-sdk/connectors_sdk/connectors/external_import/base_data_processor.py +++ b/connectors-sdk/connectors_sdk/connectors/external_import/base_data_processor.py @@ -21,12 +21,14 @@ from abc import ABC, abstractmethod from collections.abc import Generator -from typing import TYPE_CHECKING, Any +from typing import TYPE_CHECKING, Any, ClassVar from connectors_sdk.connectors.external_import._work_manager import WorkManager -from connectors_sdk.connectors.external_import.logger import ConnectorLogger +from connectors_sdk.logging.logger import Logger +from connectors_sdk.logging.sdk_logger import sdk_logger if TYPE_CHECKING: + from connectors_sdk.logging._base_logger import BaseLogger from connectors_sdk.settings.base_settings import BaseConnectorSettings from connectors_sdk.states.states import ExternalImportConnectorState from pycti import OpenCTIConnectorHelper @@ -44,7 +46,7 @@ class BaseDataProcessor(ABC): ``transform()`` can either return a list (single bundle) or yield multiple lists (streaming). ``send()`` detects this and acts accordingly. - The ``work_manager``, ``logger`` and ``state`` attributes are injected by + The ``settings``, ``state`` and ``work_manager`` attributes are injected by ``ExternalImportConnector`` via ``inject_dependencies()``. The processor can read and write state fields (e.g. cursors, checkpoints) @@ -61,43 +63,49 @@ class BaseDataProcessor(ABC): 4. ``process()`` — called by the base connector (runs the pipeline) Attributes: + _logger: A child of ``Logger``, named after the processor class. + settings: The connector settings, injected via ``inject_dependencies()``. + state: The ``ExternalImportConnectorState`` instance, injected via ``inject_dependencies()``. + work_manager: The ``WorkManager`` instance, created in ``inject_dependencies()``. work_name: A human-readable name for the work created by this processor. Changing ``work_name`` between calls to ``send()`` (or between iterations in a generator-based ``transform()``) will close the current work and open a new one with the updated name. - settings: The connector settings, injected via ``inject_dependencies()``. - work_manager: The ``WorkManager`` instance, created in ``inject_dependencies()``. - logger: The ``ConnectorLogger`` instance, injected via ``inject_dependencies()``. - state: The ``ExternalImportConnectorState`` instance, injected via ``inject_dependencies()``. """ - work_name: str + logger: ClassVar[BaseLogger] = sdk_logger.get_child("BaseDataProcessor") + settings: BaseConnectorSettings - work_manager: WorkManager - logger: ConnectorLogger state: ExternalImportConnectorState + work_manager: WorkManager + work_name: str + + @classmethod + def __init_subclass__(cls, **kwargs: Any) -> None: + """Attach a logger child named after the concrete `BaseDataProcessor` subclass.""" + super().__init_subclass__(**kwargs) + package_name = cls.__module__.split(".")[0] + cls.logger = Logger(f"{package_name}.{cls.__name__}") def inject_dependencies( self, settings: BaseConnectorSettings, - helper: OpenCTIConnectorHelper, state: ExternalImportConnectorState, + helper: OpenCTIConnectorHelper, ) -> None: """Inject dependencies from the base connector and create the WorkManager. Called by ``ExternalImportConnector`` after helper initialization. - Sets ``settings``, ``logger`` and ``state``, and creates the ``WorkManager`` - for this processor. + Sets ``settings``, ``state``, and creates the ``WorkManager`` for this processor. Args: settings: The connector configuration settings. - helper: The ``OpenCTIConnectorHelper`` instance. state: The ``ExternalImportConnectorState`` instance. + helper: The ``OpenCTIConnectorHelper`` instance. """ self.settings = settings - self.work_manager = WorkManager(helper) - self.logger = ConnectorLogger(helper) self.state = state + self.work_manager = WorkManager(helper) def post_init(self) -> None: # noqa: B027 """Hook called after ``inject_dependencies()`` wires up dependencies. diff --git a/connectors-sdk/connectors_sdk/connectors/external_import/external_import_connector.py b/connectors-sdk/connectors_sdk/connectors/external_import/external_import_connector.py index d137ca21508..eb807ee27b6 100644 --- a/connectors-sdk/connectors_sdk/connectors/external_import/external_import_connector.py +++ b/connectors-sdk/connectors_sdk/connectors/external_import/external_import_connector.py @@ -7,24 +7,31 @@ Architecture:: ExternalImportConnector - ├── OpenCTIConnectorHelper → pycti bridge (created in _init_dependencies) - ├── ConnectorLogger → Logging (wraps helper's AppLogger) + ├── OpenCTIConnectorHelper → pycti bridge (created in _init_dependencies) + ├── Logger → Logging (wraps helper's AppLogger) ├── ExternalImportConnectorState → State persistence (last_run, custom fields) - └── BaseDataProcessor[] → process(): with work_manager: send(transform(collect())) - └── WorkManager → context manager: open work → send → close work + └── BaseDataProcessor[] → process(): with work_manager: send(transform(collect())) + └── WorkManager → context manager: open work → send → close work """ +from __future__ import annotations + import sys from datetime import datetime, timezone +from typing import TYPE_CHECKING, Any, ClassVar from connectors_sdk.connectors.external_import.base_data_processor import ( BaseDataProcessor, ) -from connectors_sdk.connectors.external_import.logger import ConnectorLogger +from connectors_sdk.logging.logger import Logger +from connectors_sdk.logging.sdk_logger import sdk_logger from connectors_sdk.settings.base_settings import BaseConnectorSettings from connectors_sdk.states.states import ExternalImportConnectorState from pycti import OpenCTIConnectorHelper +if TYPE_CHECKING: + from connectors_sdk.logging._base_logger import BaseLogger + class ExternalImportConnector: """Base class for external import connectors. @@ -44,8 +51,8 @@ class ExternalImportConnector: (e.g. one for indicators, one for reports, one for vulnerabilities). Attributes: + logger(ClassVar): A ``Logger``'s child for logging, named after the connector class. settings: The connector configuration (subclass of ``BaseConnectorSettings``). - logger: The ``ConnectorLogger`` for logging without direct pycti dependency. state: The ``ExternalImportConnectorState`` for persisting connector state. data_processors: The list of ``BaseDataProcessor`` instances. @@ -66,6 +73,15 @@ class ExternalImportConnector: >>> connector.start() """ + logger: ClassVar[BaseLogger] = sdk_logger.get_child("ExternalImportConnector") + + @classmethod + def __init_subclass__(cls, **kwargs: Any) -> None: + """Attach a logger child named after the concrete `ExternalImportConnector` subclass.""" + super().__init_subclass__(**kwargs) + package_name = cls.__module__.split(".")[0] + cls.logger = Logger(f"{package_name}.{cls.__name__}") + def __init__( self, settings: BaseConnectorSettings, @@ -85,8 +101,8 @@ def __init__( """ if not data_processors: raise ValueError("At least one BaseDataProcessor must be provided.") - self.settings = settings self.data_processors = data_processors + self.settings = settings self.state = state if state is not None else ExternalImportConnectorState() def _init_dependencies(self) -> None: @@ -94,12 +110,10 @@ def _init_dependencies(self) -> None: This method: 1. Creates the ``OpenCTIConnectorHelper`` from the config - 2. Creates the ``ConnectorLogger`` - 3. Initializes the state and injects dependencies - 4. Calls ``inject_dependencies()`` on each data processor + 2. Initializes the state and injects dependencies + 3. Calls ``inject_dependencies()`` on each data processor """ self._helper = OpenCTIConnectorHelper(config=self.settings.to_helper_config()) - self.logger = ConnectorLogger(self._helper) self.state.inject_dependencies(self._helper) for processor in self.data_processors: processor.inject_dependencies( diff --git a/connectors-sdk/connectors_sdk/connectors/external_import/logger.py b/connectors-sdk/connectors_sdk/connectors/external_import/logger.py deleted file mode 100644 index 66ce6b6d763..00000000000 --- a/connectors-sdk/connectors_sdk/connectors/external_import/logger.py +++ /dev/null @@ -1,72 +0,0 @@ -"""Connector logger module. - -This module provides a logger wrapper that delegates to the OpenCTI connector helper's logger. -It allows connectors and their subcomponents (clients, converters) to use logging -without directly depending on pycti. -""" - -from typing import Any - -from pycti import OpenCTIConnectorHelper - - -class ConnectorLogger: - """Logger wrapper that delegates to the OpenCTI connector helper's logger. - - This class provides the same interface as pycti's ``AppLogger`` - (``info``, ``debug``, ``warning``, ``error``) while decoupling connector code from pycti. - - Benefits: - - Cleaner API: ``self.logger.info(...)`` instead of ``self.helper.connector_logger.info(...)`` - - Subcomponents (clients, converters) can receive a ``ConnectorLogger`` instead of the full helper - - If pycti's logging interface changes, only this wrapper needs updating - - Easier to mock in tests - - Example: - >>> logger = ConnectorLogger(helper) - >>> logger.info("Processing entity", {"entity_id": "abc-123"}) - """ - - def __init__(self, helper: OpenCTIConnectorHelper) -> None: - """Initialize the logger from a connector helper. - - Args: - helper: The ``OpenCTIConnectorHelper`` instance to delegate logging to. - """ - self._logger: Any = helper.connector_logger - - def info(self, message: str, meta: dict[str, Any] | None = None) -> None: - """Log an info message. - - Args: - message: The message to log. - meta: Optional metadata dictionary for structured logging. - """ - self._logger.info(message, meta) - - def debug(self, message: str, meta: dict[str, Any] | None = None) -> None: - """Log a debug message. - - Args: - message: The message to log. - meta: Optional metadata dictionary for structured logging. - """ - self._logger.debug(message, meta) - - def warning(self, message: str, meta: dict[str, Any] | None = None) -> None: - """Log a warning message. - - Args: - message: The message to log. - meta: Optional metadata dictionary for structured logging. - """ - self._logger.warning(message, meta) - - def error(self, message: str, meta: dict[str, Any] | None = None) -> None: - """Log an error message. - - Args: - message: The message to log. - meta: Optional metadata dictionary for structured logging. - """ - self._logger.error(message, meta) diff --git a/connectors-sdk/connectors_sdk/settings/base_settings.py b/connectors-sdk/connectors_sdk/settings/base_settings.py index 961c3c2e2af..aac77a771f0 100644 --- a/connectors-sdk/connectors_sdk/settings/base_settings.py +++ b/connectors-sdk/connectors_sdk/settings/base_settings.py @@ -13,6 +13,9 @@ from pathlib import Path from typing import Any, ClassVar, Literal, Self +from connectors_sdk.logging.logger import Logger +from connectors_sdk.logging.sdk_logger import sdk_logger +from connectors_sdk.settings._settings_loader import _SettingsLoader from connectors_sdk.settings.annotated_types import ListFromString from connectors_sdk.settings.deprecations import ( Deprecate, @@ -296,6 +299,15 @@ class BaseConnectorSettings(BaseConfigModel, ABC): description="Connector configurations.", ) + logger: ClassVar[BaseLogger] = sdk_logger.get_child("BaseConnectorSettings") + + @classmethod + def __init_subclass__(cls, **kwargs: Any) -> None: + """Attach a logger child named after the concrete `BaseConnectorSettings` subclass.""" + super().__init_subclass__(**kwargs) + package_name = cls.__module__.split(".")[0] + cls.logger = Logger(f"{package_name}.{cls.__name__}") + def __init__(self) -> None: """Initialize the configuration model and handle validation errors.""" try: diff --git a/connectors-sdk/connectors_sdk/states/_base_state.py b/connectors-sdk/connectors_sdk/states/_base_state.py index 9719d1be885..acff9f31965 100644 --- a/connectors-sdk/connectors_sdk/states/_base_state.py +++ b/connectors-sdk/connectors_sdk/states/_base_state.py @@ -19,6 +19,8 @@ from datetime import datetime from typing import TYPE_CHECKING, Any +from connectors_sdk.logging.logger import Logger +from connectors_sdk.logging.sdk_logger import sdk_logger from pydantic import BaseModel, ConfigDict, PrivateAttr, field_serializer if TYPE_CHECKING: @@ -77,15 +79,16 @@ class BaseConnectorState(BaseModel, ABC): validate_assignment=True, ) + logger: ClassVar[BaseLogger] = sdk_logger.get_child("BaseConnectorState") + _client: _StateClient | None = PrivateAttr(default=None) _can_be_loaded: bool = PrivateAttr(default=False) - @field_serializer("*", mode="wrap", when_used="json") - def _serialize_datetimes(self, value: Any, handler: Any) -> Any: - """Replace the default JSON serializer, in order to use +00:00 offset instead of Z prefix. - This is convenient so `assert self.model_dump(mode="json")["last_run"] == self.last_run.isoformat()` - is `True` across both codebase and tests (using the same serializer). - Consistent with `DatetimeFromIsoString` in `connectors_sdk.settings.annotated_types` module too. + def __init_subclass__(cls, **kwargs: Any) -> None: + """Attach a logger child named after the concrete `BaseConnectorState` subclass.""" + super().__init_subclass__(**kwargs) + package_name = cls.__module__.split(".")[0] + cls.logger = Logger(f"{package_name}.{cls.__name__}") Arguments: value: The value to serialize. diff --git a/connectors-sdk/tests/test_api.py b/connectors-sdk/tests/test_api.py index e6bb5165758..0302b5040f1 100644 --- a/connectors-sdk/tests/test_api.py +++ b/connectors-sdk/tests/test_api.py @@ -18,15 +18,16 @@ def test_root_public_api_is_valid(): "BaseStreamConnectorConfig", "ConfigError", "ConfigValidationError", - "ExternalImportConnectorState", "DatetimeFromIsoString", "Deprecate", "DeprecatedField", "ExternalImportConnector", - "ConnectorLogger", + "ExternalImportConnectorState", "BaseDataProcessor", "WorkManager", "ListFromString", + "Logger", + "logger", } missing = imports - set(root_api.__all__) extra = set(root_api.__all__) - imports diff --git a/connectors-sdk/tests/test_connectors/test_logger.py b/connectors-sdk/tests/test_connectors/test_logger.py deleted file mode 100644 index 24a767ec906..00000000000 --- a/connectors-sdk/tests/test_connectors/test_logger.py +++ /dev/null @@ -1,35 +0,0 @@ -# pragma: no cover -# type: ignore -from unittest.mock import MagicMock - -from connectors_sdk.connectors.external_import.logger import ConnectorLogger - - -class TestConnectorLogger: - def test_init_stores_connector_logger(self, mock_helper: MagicMock): - logger = ConnectorLogger(mock_helper) - assert logger._logger is mock_helper.connector_logger - - def test_info(self, mock_logger: ConnectorLogger, mock_helper: MagicMock): - mock_logger.info("hello", {"key": "val"}) - mock_helper.connector_logger.info.assert_called_once_with( - "hello", {"key": "val"} - ) - - def test_info_no_meta(self, mock_logger: ConnectorLogger, mock_helper: MagicMock): - mock_logger.info("hello") - mock_helper.connector_logger.info.assert_called_once_with("hello", None) - - def test_debug(self, mock_logger: ConnectorLogger, mock_helper: MagicMock): - mock_logger.debug("dbg", {"k": "v"}) - mock_helper.connector_logger.debug.assert_called_once_with("dbg", {"k": "v"}) - - def test_warning(self, mock_logger: ConnectorLogger, mock_helper: MagicMock): - mock_logger.warning("warn") - mock_helper.connector_logger.warning.assert_called_once_with("warn", None) - - def test_error(self, mock_logger: ConnectorLogger, mock_helper: MagicMock): - mock_logger.error("err", {"detail": "x"}) - mock_helper.connector_logger.error.assert_called_once_with( - "err", {"detail": "x"} - ) From a43084db60d2965f062b9dfb1fa0ff606cb9c573 Mon Sep 17 00:00:00 2001 From: Powlinett Date: Wed, 27 May 2026 17:43:32 +0200 Subject: [PATCH 04/13] feat: add / update logs --- .../external_import/_work_manager.py | 105 ++++++- .../external_import/base_data_processor.py | 8 + .../external_import_connector.py | 44 +-- .../settings/_settings_loader.py | 165 +++++++++++ .../connectors_sdk/settings/base_settings.py | 280 +++++------------- .../connectors_sdk/states/_base_state.py | 72 +++-- 6 files changed, 414 insertions(+), 260 deletions(-) create mode 100644 connectors-sdk/connectors_sdk/settings/_settings_loader.py diff --git a/connectors-sdk/connectors_sdk/connectors/external_import/_work_manager.py b/connectors-sdk/connectors_sdk/connectors/external_import/_work_manager.py index 53ffe3d4982..4668b51be1f 100644 --- a/connectors-sdk/connectors_sdk/connectors/external_import/_work_manager.py +++ b/connectors-sdk/connectors_sdk/connectors/external_import/_work_manager.py @@ -61,6 +61,8 @@ def __init__( self._closed = False self._has_sent_bundles = False + self.logger.debug(f"{self.__class__.__name__} instantiated succesfully") + @classmethod def create(cls, helper: OpenCTIConnectorHelper, work_name: str) -> _Work: """Create a new work in OpenCTI and return a ``_Work`` instance. @@ -76,9 +78,9 @@ def create(cls, helper: OpenCTIConnectorHelper, work_name: str) -> _Work: A new ``_Work`` instance wrapping the created work. """ work_id: str = helper.api.work.initiate_work(helper.connect_id, work_name) - logger.info( - f"Work '{work_id}' initiated", - {"work_name": work_name}, + cls.logger.debug( + "Work created", + {"work_name": work_name, "work_id": work_id}, ) return cls(helper, work_id, work_name) @@ -92,11 +94,19 @@ def send_bundle(self, bundle_objects: list[Any], **kwargs: Any) -> None: """ stix_objects = self._to_stix(bundle_objects) bundle = self._helper.stix2_create_bundle(stix_objects) - bundles_sent = self._helper.send_stix2_bundle(bundle, work_id=self.id, **kwargs) + bundles_sent = self._helper.send_stix2_bundle( + bundle, + work_id=self.id, + **kwargs, + ) self._has_sent_bundles = True - self._logger.info( + self.logger.info( "Sent STIX objects to OpenCTI", - {"bundles_sent": str(len(bundles_sent))}, + { + "bundles_sent": len(bundles_sent), + "work_name": self.name, + "work_id": self.id, + }, ) def success(self, message: str) -> None: @@ -106,7 +116,10 @@ def success(self, message: str) -> None: message: A completion message stored alongside the work. """ self._helper.api.work.to_processed(self.id, message) - self._logger.info(message) + self.logger.debug( + "Work marked as completed on OpenCTI", + {"work_name": self.name, "work_id": self.id, "message": message}, + ) self._closed = True def fail(self, message: str) -> None: @@ -116,7 +129,10 @@ def fail(self, message: str) -> None: message: An error message stored alongside the work. """ self._helper.api.work.to_processed(self.id, message, in_error=True) - self._logger.error(message) + self.logger.debug( + "Work marked as failed on OpenCTI", + {"work_name": self.name, "work_id": self.id, "message": message}, + ) self._closed = True def _delete(self) -> None: @@ -127,9 +143,9 @@ def _delete(self) -> None: the ``WorkManager`` to clean up orphaned or invalid works. """ self._helper.api.work.delete(id=self.id) - self._logger.info( - "Work deleted", - {"work_id": self.id}, + self.logger.debug( + "Work deleted on OpenCTI", + {"work_name": self.name, "work_id": self.id}, ) self._closed = True @@ -184,6 +200,8 @@ def __init__(self, helper: OpenCTIConnectorHelper) -> None: self._current_work: _Work | None = None self._active = False + self.logger.debug(f"{self.__class__.__name__} instantiated succesfully") + def __enter__(self) -> WorkManager: """Enter the context manager.""" self._current_work = None @@ -204,11 +222,34 @@ def __exit__( """ if self._current_work is not None and not self._current_work._closed: if not self._current_work._has_sent_bundles: + self.logger.info( + "Zero bundles were sent, deleting work", + { + "work_name": self._current_work.name, + "work_id": self._current_work.id, + }, + ) self._current_work._delete() elif exc_type is not None: - self._current_work.fail(f"Work failed with error: {exc_val}") + message = f"Work failed with error: {exc_val}" + self.logger.error( + message, + { + "work_name": self._current_work.name, + "work_id": self._current_work.id, + }, + ) + self._current_work.fail(message) else: - self._current_work.success("Work completed successfully") + message = "Work completed successfully" + self.logger.info( + message, + { + "work_name": self._current_work.name, + "work_id": self._current_work.id, + }, + ) + self._current_work.success(message) self._current_work = None self._active = False @@ -230,19 +271,53 @@ def send( (e.g. ``cleanup_inconsistent_bundle``, ``update``, ``entities_types``). """ if not bundle_objects: + self.logger.info( + "No objects to send", + {"work_name": work_name}, + ) return + if not self._active: msg = "WorkManager.send() must be called inside a 'with' block." raise RuntimeError(msg) + if self._current_work is None or self._current_work.name != work_name: self._close_current_work() - self._current_work = _Work.create(self._helper, self._logger, work_name) + + self.logger.info( + "Creating a new work", + { + "new_work_name": work_name, + "previous_work_name": ( + self._current_work.name + if self._current_work is not None + else None + ), + }, + ) + self._current_work = _Work.create(self._helper, work_name) + self._current_work.send_bundle(bundle_objects, **kwargs) def _close_current_work(self) -> None: """Close the current work if it exists and is not already closed.""" if self._current_work is not None and not self._current_work._closed: if not self._current_work._has_sent_bundles: + self.logger.info( + "Zero bundles were sent, deleting work", + { + "work_name": self._current_work.name, + "work_id": self._current_work.id, + }, + ) self._current_work._delete() else: - self._current_work.success("Work completed successfully") + message = "Work completed successfully" + self.logger.info( + message, + { + "work_name": self._current_work.name, + "work_id": self._current_work.id, + }, + ) + self._current_work.success(message) diff --git a/connectors-sdk/connectors_sdk/connectors/external_import/base_data_processor.py b/connectors-sdk/connectors_sdk/connectors/external_import/base_data_processor.py index 5f47e926701..654b3d14d06 100644 --- a/connectors-sdk/connectors_sdk/connectors/external_import/base_data_processor.py +++ b/connectors-sdk/connectors_sdk/connectors/external_import/base_data_processor.py @@ -87,6 +87,10 @@ def __init_subclass__(cls, **kwargs: Any) -> None: package_name = cls.__module__.split(".")[0] cls.logger = Logger(f"{package_name}.{cls.__name__}") + def __init__(self) -> None: + """Initialize the processor.""" + self.logger.debug(f"{self.__class__.__name__} instantiated succesfully") + def inject_dependencies( self, settings: BaseConnectorSettings, @@ -107,6 +111,10 @@ def inject_dependencies( self.state = state self.work_manager = WorkManager(helper) + self.logger.debug( + f"Dependencies injected into `{self.__class__.__name__}` instance. `WorkManager` set up." + ) + def post_init(self) -> None: # noqa: B027 """Hook called after ``inject_dependencies()`` wires up dependencies. diff --git a/connectors-sdk/connectors_sdk/connectors/external_import/external_import_connector.py b/connectors-sdk/connectors_sdk/connectors/external_import/external_import_connector.py index eb807ee27b6..914a459ed11 100644 --- a/connectors-sdk/connectors_sdk/connectors/external_import/external_import_connector.py +++ b/connectors-sdk/connectors_sdk/connectors/external_import/external_import_connector.py @@ -105,6 +105,11 @@ def __init__( self.settings = settings self.state = state if state is not None else ExternalImportConnectorState() + self.logger.debug( + f"{self.__class__.__name__} instantiated succesfully with {len(data_processors)} processor(s)", + {"data_processors": [p.__class__.__name__ for p in data_processors]}, + ) + def _init_dependencies(self) -> None: """Create the OpenCTI connector helper and wire up all components. @@ -134,47 +139,44 @@ def callback(self) -> None: Override this method for fully custom processing logic. """ - connector_name = self.settings.connector.name - self.logger.info( - "[CONNECTOR] Starting connector...", - {"connector_name": connector_name}, - ) + self.logger.info("Connector's run starting") try: self.state.load(force=True) + self.logger.info( + "Connector's state loaded from OpenCTI", + {"state": self.state.to_json()}, + ) if self.state.last_run: self.logger.info( - "[CONNECTOR] Connector last run", - {"last_run_datetime": str(self.state.last_run)}, + "Connector's 'last_run' datetime found in state", + {"last_run": self.state.last_run.isoformat()}, ) else: - self.logger.info("[CONNECTOR] Connector has never run...") - - self.logger.info( - "[CONNECTOR] Running connector...", - {"connector_name": connector_name}, - ) + self.logger.info("Connector has never run before") + self.logger.info("Running connector's data processors") for processor in self.data_processors: processor.process() self.state.last_run = datetime.now(tz=timezone.utc) self.state.save() - self.logger.info( - f"{connector_name} connector successfully run, " - f"storing last_run as {self.state.last_run}" + "Connector's state saved on OpenCTI", + {"state": self.state.to_json()}, ) - except (KeyboardInterrupt, SystemExit): self.logger.info( - "[CONNECTOR] Connector stopped...", - {"connector_name": connector_name}, + "Connector's run completed, 'last_run' datetime stored in state", + {"last_run": self.state.last_run.isoformat()}, ) + + except (KeyboardInterrupt, SystemExit): + self.logger.info("Connector stopped by user or system") sys.exit(0) except Exception as err: - self.logger.error(str(err)) + self.logger.error(f"Unexpected error: {err}") def start(self) -> None: """Start the connector with scheduled execution. @@ -191,6 +193,8 @@ def start(self) -> None: The ``settings.connector`` must be a ``BaseExternalImportConnectorConfig`` (or subclass) with a ``duration_period`` field. """ + self.logger.info("Connector's starting") + self._init_dependencies() self._helper.schedule_process( message_callback=self.callback, diff --git a/connectors-sdk/connectors_sdk/settings/_settings_loader.py b/connectors-sdk/connectors_sdk/settings/_settings_loader.py new file mode 100644 index 00000000000..b23d2cfc6aa --- /dev/null +++ b/connectors-sdk/connectors_sdk/settings/_settings_loader.py @@ -0,0 +1,165 @@ +from __future__ import annotations + +import sys +from copy import deepcopy +from pathlib import Path +from typing import TYPE_CHECKING, Any, ClassVar + +from connectors_sdk.logging.sdk_logger import sdk_logger +from pydantic import BaseModel, create_model +from pydantic_settings import ( + BaseSettings, + DotEnvSettingsSource, + PydanticBaseSettingsSource, + SettingsConfigDict, + YamlConfigSettingsSource, +) + +if TYPE_CHECKING: + from connectors_sdk.logging._base_logger import BaseLogger + from connectors_sdk.settings.base_settings import BaseConnectorSettings + + +class _SettingsLoader(BaseSettings): + model_config = SettingsConfigDict( + frozen=True, + extra="allow", + env_nested_delimiter="_", + env_nested_max_split=1, + enable_decoding=False, + ) + + logger: ClassVar[BaseLogger] = sdk_logger.get_child("_SettingsLoader") + + @classmethod + def _get_connector_main_path(cls) -> Path: + """Locate the main module of the running connector. + This method is used to locate configuration files relative to connector's entrypoint. + + Notes: + - This method assumes that the connector is launched using a file-backed entrypoint + (i.e., `python -m ` or `python `). + - At module import time, `__main__.__file__` might not be available yet, + thus this method should be called at runtime only. + """ + main = sys.modules.get("__main__") + if main and getattr(main, "__file__", None): + return Path(main.__file__).resolve() # type: ignore + + raise RuntimeError( + "Cannot determine connector's location: __main__.__file__ is not available. " + "Ensure the connector is launched using `python -m ` or a file-backed entrypoint." + ) + + @classmethod + def _get_config_yml_file_path(cls) -> Path | None: + """Locate the `config.yml` file of the running connector.""" + main_path = cls._get_connector_main_path() + config_yml_legacy_file_path = main_path.parent / "config.yml" + config_yml_file_path = main_path.parent.parent / "config.yml" + + if config_yml_legacy_file_path.is_file(): + return config_yml_legacy_file_path + elif config_yml_file_path.is_file(): + return config_yml_file_path + return None + + @classmethod + def _get_dot_env_file_path(cls) -> Path | None: + """Locate the `.env` file of the running connector.""" + main_path = cls._get_connector_main_path() + dot_env_file_path = main_path.parent.parent / ".env" + + return dot_env_file_path if dot_env_file_path.is_file() else None + + @classmethod + def settings_customise_sources( + cls, + settings_cls: type[BaseSettings], + init_settings: PydanticBaseSettingsSource, + env_settings: PydanticBaseSettingsSource, + dotenv_settings: PydanticBaseSettingsSource, + file_secret_settings: PydanticBaseSettingsSource, + ) -> tuple[PydanticBaseSettingsSource, ...]: + """Customise the sources of settings for the connector. + + This method is called by the Pydantic BaseSettings class to determine the order of sources. + The configuration come in this order either from: + 1. Environment variables + 2. YAML file + 3. .env file + 4. Default values + + The variables loading order will remain the same as in `pycti.get_config_variable()`: + 1. If a config.yml file is found, the order will be: `ENV VAR` → config.yml → default value + 2. If a .env file is found, the order will be: `ENV VAR` → .env → default value + """ + config_yml_file_path = cls._get_config_yml_file_path() + if config_yml_file_path: + cls.logger.debug( + "Parsing connector's settings from config.yml file", + {"config_yml_file_path": str(config_yml_file_path)}, + ) + return ( + env_settings, + YamlConfigSettingsSource(settings_cls, yaml_file=config_yml_file_path), + ) + + dot_env_file_path = cls._get_dot_env_file_path() + if dot_env_file_path: + cls.logger.debug( + "Parsing connector's settings from .env file", + {"dot_env_file_path": str(dot_env_file_path)}, + ) + return ( + env_settings, + DotEnvSettingsSource(settings_cls, env_file=dot_env_file_path), + ) + + cls.logger.debug("Parsing connector's settings from environment variables") + + return (env_settings,) + + @classmethod + def build_loader_from_model( + cls, connector_settings: type[BaseConnectorSettings] + ) -> type[_SettingsLoader]: + """Build an untyped `_SettingsLoader` subclass for a connector's settings. + + This method dynamically creates a subclass of `_SettingsLoader` that mirrors the + structure of the provided `BaseConnectorSettings` implementation. It disables all + Pydantic decoding, type coercion and validation so fields accept raw, unprocessed values. + + The resulting model: + * Preserves values as-is from configuration sources + * Keeps YAML values as native Python types + * Keeps environment variables as plain strings + * Allows any field type (`Any`) without validation + + Args: + connector_settings (type[BaseConnectorSettings]): The typed connector settings class to mirror. + + Returns: + type[_SettingsLoader]: A dynamically generated subclass of `_SettingsLoader` + where all fields accept raw, unvalidated input. + """ + + class SettingsLoader(_SettingsLoader): ... + + model_fields = deepcopy(connector_settings.model_fields) + for field_info in model_fields.values(): + annotation = field_info.annotation + if annotation and issubclass(annotation, BaseModel): + fields: dict[str, Any] = dict.fromkeys( + annotation.model_fields.keys(), Any + ) + untyped_model = create_model( + f"{annotation.__name__}Untyped", + __base__=annotation, + **fields, + ) + field_info.annotation = untyped_model + field_info.default_factory = untyped_model + + SettingsLoader.model_fields = model_fields # type: ignore + return SettingsLoader diff --git a/connectors-sdk/connectors_sdk/settings/base_settings.py b/connectors-sdk/connectors_sdk/settings/base_settings.py index aac77a771f0..d348f7e76e6 100644 --- a/connectors-sdk/connectors_sdk/settings/base_settings.py +++ b/connectors-sdk/connectors_sdk/settings/base_settings.py @@ -6,12 +6,11 @@ These models can be extended to create specific configurations for different types of connectors. """ -import sys +from __future__ import annotations + from abc import ABC -from copy import deepcopy from datetime import timedelta -from pathlib import Path -from typing import Any, ClassVar, Literal, Self +from typing import TYPE_CHECKING, Any, ClassVar, Literal, Self from connectors_sdk.logging.logger import Logger from connectors_sdk.logging.sdk_logger import sdk_logger @@ -33,17 +32,12 @@ HttpUrl, ModelWrapValidatorHandler, ValidationError, - create_model, model_validator, ) from pydantic.fields import FieldInfo -from pydantic_settings import ( - BaseSettings, - DotEnvSettingsSource, - PydanticBaseSettingsSource, - SettingsConfigDict, - YamlConfigSettingsSource, -) + +if TYPE_CHECKING: + from connectors_sdk.logging._base_logger import BaseLogger class BaseConfigModel(BaseModel, ABC): @@ -124,137 +118,82 @@ class _BaseConnectorConfig(BaseConfigModel, ABC): ) -class _SettingsLoader(BaseSettings): - model_config = SettingsConfigDict( - frozen=True, - extra="allow", - env_nested_delimiter="_", - env_nested_max_split=1, - enable_decoding=False, - ) +class BaseExternalImportConnectorConfig(_BaseConnectorConfig): + """Settings class for external import connectors. - @classmethod - def _get_connector_main_path(cls) -> Path: - """Locate the main module of the running connector. - This method is used to locate configuration files relative to connector's entrypoint. + Attributes: + type (str): The type of the connector, set to "EXTERNAL_IMPORT" for external import connectors. + duration_period (timedelta): The period of time to await between two runs of the connector. + """ - Notes: - - This method assumes that the connector is launched using a file-backed entrypoint - (i.e., `python -m ` or `python `). - - At module import time, `__main__.__file__` might not be available yet, - thus this method should be called at runtime only. - """ - main = sys.modules.get("__main__") - if main and getattr(main, "__file__", None): - return Path(main.__file__).resolve() # type: ignore + type: Literal["EXTERNAL_IMPORT"] = "EXTERNAL_IMPORT" + duration_period: timedelta = Field( + description="The period of time to await between two runs of the connector." + ) - raise RuntimeError( - "Cannot determine connector's location: __main__.__file__ is not available. " - "Ensure the connector is launched using `python -m ` or a file-backed entrypoint." - ) - @classmethod - def _get_config_yml_file_path(cls) -> Path | None: - """Locate the `config.yml` file of the running connector.""" - main_path = cls._get_connector_main_path() - config_yml_legacy_file_path = main_path.parent / "config.yml" - config_yml_file_path = main_path.parent.parent / "config.yml" - - if config_yml_legacy_file_path.is_file(): - return config_yml_legacy_file_path - elif config_yml_file_path.is_file(): - return config_yml_file_path - return None +class BaseInternalEnrichmentConnectorConfig(_BaseConnectorConfig): + """Settings class for internal enrichment connectors. - @classmethod - def _get_dot_env_file_path(cls) -> Path | None: - """Locate the `.env` file of the running connector.""" - main_path = cls._get_connector_main_path() - dot_env_file_path = main_path.parent.parent / ".env" + Attributes: + type (str): The type of the connector, set to "INTERNAL_ENRICHMENT" for internal enrichment connectors. + auto (bool): Whether the connector should run automatically when an entity is created or updated. + """ + + type: Literal["INTERNAL_ENRICHMENT"] = "INTERNAL_ENRICHMENT" + auto: bool = Field( + default=False, + description="Whether the connector should run automatically when an entity is created or updated.", + ) - return dot_env_file_path if dot_env_file_path.is_file() else None - @classmethod - def settings_customise_sources( - cls, - settings_cls: type[BaseSettings], - init_settings: PydanticBaseSettingsSource, - env_settings: PydanticBaseSettingsSource, - dotenv_settings: PydanticBaseSettingsSource, - file_secret_settings: PydanticBaseSettingsSource, - ) -> tuple[PydanticBaseSettingsSource, ...]: - """Customise the sources of settings for the connector. - - This method is called by the Pydantic BaseSettings class to determine the order of sources. - The configuration come in this order either from: - 1. Environment variables - 2. YAML file - 3. .env file - 4. Default values - - The variables loading order will remain the same as in `pycti.get_config_variable()`: - 1. If a config.yml file is found, the order will be: `ENV VAR` → config.yml → default value - 2. If a .env file is found, the order will be: `ENV VAR` → .env → default value - """ - config_yml_file_path = cls._get_config_yml_file_path() - if config_yml_file_path: - return ( - env_settings, - YamlConfigSettingsSource(settings_cls, yaml_file=config_yml_file_path), - ) +class BaseStreamConnectorConfig(_BaseConnectorConfig): + """Settings class for stream connectors. - dot_env_file_path = cls._get_dot_env_file_path() - if dot_env_file_path: - return ( - env_settings, - DotEnvSettingsSource(settings_cls, env_file=dot_env_file_path), - ) + Attributes: + type (str): The type of the connector, set to "STREAM" for stream connectors + live_stream_id (str): The ID of the live stream to connect to. + live_stream_listen_delete (bool): Whether to listen for delete events on the live stream. + live_stream_no_dependencies (bool): Whether to ignore dependencies when processing events from the live stream. + """ - return (env_settings,) + type: Literal["STREAM"] = "STREAM" + live_stream_id: str = Field( + description="The ID of the live stream to connect to.", + ) + live_stream_listen_delete: bool = Field( + default=True, + description="Whether to listen for delete events on the live stream.", + ) + live_stream_no_dependencies: bool = Field( + default=True, + description="Whether to ignore dependencies when processing events from the live stream.", + ) - @classmethod - def build_loader_from_model( - cls, connector_settings: type["BaseConnectorSettings"] - ) -> type["_SettingsLoader"]: - """Build an untyped `_SettingsLoader` subclass for a connector's settings. - This method dynamically creates a subclass of `_SettingsLoader` that mirrors the - structure of the provided `BaseConnectorSettings` implementation. It disables all - Pydantic decoding, type coercion and validation so fields accept raw, unprocessed values. +class BaseInternalExportFileConnectorConfig(_BaseConnectorConfig): + """Settings class for internal export file connectors. - The resulting model: - * Preserves values as-is from configuration sources - * Keeps YAML values as native Python types - * Keeps environment variables as plain strings - * Allows any field type (`Any`) without validation + Attributes: + type (str): The type of the connector, set to "INTERNAL_EXPORT_FILE" for internal export file connectors. + """ - Args: - connector_settings (type[BaseConnectorSettings]): The typed connector settings class to mirror. + type: Literal["INTERNAL_EXPORT_FILE"] = "INTERNAL_EXPORT_FILE" - Returns: - type[_SettingsLoader]: A dynamically generated subclass of `_SettingsLoader` - where all fields accept raw, unvalidated input. - """ - class SettingsLoader(_SettingsLoader): ... +class BaseInternalImportFileConnectorConfig(_BaseConnectorConfig): + """Settings class for internal import file connectors. - model_fields = deepcopy(connector_settings.model_fields) - for field_info in model_fields.values(): - annotation = field_info.annotation - if annotation and issubclass(annotation, BaseModel): - fields: dict[str, Any] = dict.fromkeys( - annotation.model_fields.keys(), Any - ) - untyped_model = create_model( - f"{annotation.__name__}Untyped", - __base__=annotation, - **fields, - ) - field_info.annotation = untyped_model - field_info.default_factory = untyped_model + Attributes: + type (str): The type of the connector, set to "INTERNAL_IMPORT_FILE" for internal import file connectors. + auto (bool): Whether the connector should run automatically when an entity is created or updated. + """ - SettingsLoader.model_fields = model_fields # type: ignore - return SettingsLoader + type: Literal["INTERNAL_IMPORT_FILE"] = "INTERNAL_IMPORT_FILE" + auto: bool = Field( + default=False, + description="Whether the connector should run automatically when an entity is created or updated.", + ) class BaseConnectorSettings(BaseConfigModel, ABC): @@ -315,6 +254,11 @@ def __init__(self) -> None: except ValidationError as e: raise ConfigValidationError("Error validating configuration.") from e + self.logger.debug( + f"{self.__class__.__name__} instantiated succesfully", + {"settings": self.model_dump(mode="json")}, + ) + @classmethod def config_json_schema( cls, @@ -477,6 +421,12 @@ def _load_config_dict( # Get config/env vars as dict to send for validation config_dict: dict[str, Any] = settings_loader().model_dump() + + cls.logger.debug( + "Raw configuration data loaded for validation", + {"config_data": config_dict}, + ) + return handler(config_dict) def to_helper_config(self) -> dict[str, Any]: @@ -489,81 +439,3 @@ def to_helper_config(self) -> dict[str, Any]: # OpenCTIConnectorHelper handles missing fields with default values or internal logic. exclude_none=True, ) - - -class BaseExternalImportConnectorConfig(_BaseConnectorConfig): - """Settings class for external import connectors. - - Attributes: - type (str): The type of the connector, set to "EXTERNAL_IMPORT" for external import connectors. - duration_period (timedelta): The period of time to await between two runs of the connector. - """ - - type: Literal["EXTERNAL_IMPORT"] = "EXTERNAL_IMPORT" - duration_period: timedelta = Field( - description="The period of time to await between two runs of the connector." - ) - - -class BaseInternalEnrichmentConnectorConfig(_BaseConnectorConfig): - """Settings class for internal enrichment connectors. - - Attributes: - type (str): The type of the connector, set to "INTERNAL_ENRICHMENT" for internal enrichment connectors. - auto (bool): Whether the connector should run automatically when an entity is created or updated. - """ - - type: Literal["INTERNAL_ENRICHMENT"] = "INTERNAL_ENRICHMENT" - auto: bool = Field( - default=False, - description="Whether the connector should run automatically when an entity is created or updated.", - ) - - -class BaseStreamConnectorConfig(_BaseConnectorConfig): - """Settings class for stream connectors. - - Attributes: - type (str): The type of the connector, set to "STREAM" for stream connectors - live_stream_id (str): The ID of the live stream to connect to. - live_stream_listen_delete (bool): Whether to listen for delete events on the live stream. - live_stream_no_dependencies (bool): Whether to ignore dependencies when processing events from the live stream. - """ - - type: Literal["STREAM"] = "STREAM" - live_stream_id: str = Field( - description="The ID of the live stream to connect to.", - ) - live_stream_listen_delete: bool = Field( - default=True, - description="Whether to listen for delete events on the live stream.", - ) - live_stream_no_dependencies: bool = Field( - default=True, - description="Whether to ignore dependencies when processing events from the live stream.", - ) - - -class BaseInternalExportFileConnectorConfig(_BaseConnectorConfig): - """Settings class for internal export file connectors. - - Attributes: - type (str): The type of the connector, set to "INTERNAL_EXPORT_FILE" for internal export file connectors. - """ - - type: Literal["INTERNAL_EXPORT_FILE"] = "INTERNAL_EXPORT_FILE" - - -class BaseInternalImportFileConnectorConfig(_BaseConnectorConfig): - """Settings class for internal import file connectors. - - Attributes: - type (str): The type of the connector, set to "INTERNAL_IMPORT_FILE" for internal import file connectors. - auto (bool): Whether the connector should run automatically when an entity is created or updated. - """ - - type: Literal["INTERNAL_IMPORT_FILE"] = "INTERNAL_IMPORT_FILE" - auto: bool = Field( - default=False, - description="Whether the connector should run automatically when an entity is created or updated.", - ) diff --git a/connectors-sdk/connectors_sdk/states/_base_state.py b/connectors-sdk/connectors_sdk/states/_base_state.py index acff9f31965..a76ca8e04ce 100644 --- a/connectors-sdk/connectors_sdk/states/_base_state.py +++ b/connectors-sdk/connectors_sdk/states/_base_state.py @@ -17,13 +17,14 @@ import warnings from abc import ABC from datetime import datetime -from typing import TYPE_CHECKING, Any +from typing import TYPE_CHECKING, Any, ClassVar from connectors_sdk.logging.logger import Logger from connectors_sdk.logging.sdk_logger import sdk_logger from pydantic import BaseModel, ConfigDict, PrivateAttr, field_serializer if TYPE_CHECKING: + from connectors_sdk.logging._base_logger import BaseLogger from pycti import OpenCTIConnectorHelper @@ -33,6 +34,8 @@ class _StateClient: Connector developers should not interact with this class directly. """ + logger: ClassVar[BaseLogger] = sdk_logger.get_child("_StateClient") + _helper: OpenCTIConnectorHelper def __init__(self, helper: OpenCTIConnectorHelper) -> None: @@ -45,23 +48,26 @@ def __init__(self, helper: OpenCTIConnectorHelper) -> None: """ self._helper = helper + self.logger.debug(f"{self.__class__.__name__} instantiated succesfully") + def load_state(self) -> dict[str, Any]: """Get connector's state stored on OpenCTI.""" - return self._helper.get_state() or {} + state = self._helper.get_state() or {} - def save_state(self, state: BaseConnectorState) -> None: - """Save state's fields as connector's state on OpenCTI.""" - declared_fields = set(type(state).model_fields) + self.logger.debug("Raw state loaded from OpenCTI", {"state": state}) - state_dump = state.model_dump(mode="json", include=declared_fields) - # Send both declared _and_ extra fields to not delete any connector state's attributes on OpenCTI - if state.model_extra: - state_dump.update(state.model_extra) + return state - self._helper.set_state(state_dump) + def save_state(self, state: BaseConnectorState) -> None: + """Save state's fields as connector's state on OpenCTI.""" + # Send both declared and extra fields (but not private attributes) + state_dict = state.to_json() + self._helper.set_state(state_dict) # Ensure the state is updated immediately on OpenCTI (instead of waiting for the next ping) self._helper.force_ping() + self.logger.debug("State saved to OpenCTI", {"state": state_dict}) + class BaseConnectorState(BaseModel, ABC): """Base class for connectors states. @@ -90,17 +96,6 @@ def __init_subclass__(cls, **kwargs: Any) -> None: package_name = cls.__module__.split(".")[0] cls.logger = Logger(f"{package_name}.{cls.__name__}") - Arguments: - value: The value to serialize. - handler: The default JSON serializer to use for non-datetime values. - - Returns: - The serialized value. - """ - if isinstance(value, datetime): - return value.isoformat() # Override default JSON serializer - return handler(value) - def model_post_init(self, context: Any) -> None: """Enable loading the state from OpenCTI after the model is initialized. @@ -121,6 +116,23 @@ def inject_dependencies(self, helper: OpenCTIConnectorHelper) -> None: # Wrap the helper to prepare state API calls properly self._client = _StateClient(helper) + self.logger.debug( + f"Dependencies injected into `{self.__class__.__name__}` instance. `_StateClient` set up." + ) + + def to_json(self) -> dict[str, Any]: + """Get the state as a JSON-serializable dict. + Used before saving the state to OpenCTI, or for logging purposes. + """ + # Include declared and extra fields (but not private attributes) + declared_fields = set(type(self).model_fields) + + state_dump = self.model_dump(mode="json", include=declared_fields) + if self.model_extra: + state_dump.update(self.model_extra) + + return state_dump + def load(self, force: bool = False) -> None: """Load the state from OpenCTI. This will overwrite the instance's fields with the connector's state stored on OpenCTI. @@ -167,3 +179,21 @@ def save(self) -> None: self._client.save_state(self) self._can_be_loaded = True + + @field_serializer("*", mode="wrap", when_used="json") + def _serialize_datetimes(self, value: Any, handler: Any) -> Any: + """Replace the default JSON serializer, in order to use +00:00 offset instead of Z prefix. + This is convenient so `assert self.model_dump(mode="json")["last_run"] == self.last_run.isoformat()` + is `True` across both codebase and tests (using the same serializer). + Consistent with `DatetimeFromIsoString` in `connectors_sdk.settings.annotated_types` module too. + + Arguments: + value: The value to serialize. + handler: The default JSON serializer to use for non-datetime values. + + Returns: + The serialized value. + """ + if isinstance(value, datetime): + return value.isoformat() # Override default JSON serializer + return handler(value) From ef11f498e31797a2d351172d8d2d991dde6bee17 Mon Sep 17 00:00:00 2001 From: Powlinett Date: Thu, 28 May 2026 12:37:54 +0200 Subject: [PATCH 05/13] chore!: remove WorkManager from public API (private module) --- connectors-sdk/connectors_sdk/__init__.py | 8 +++----- connectors-sdk/tests/test_api.py | 3 +-- 2 files changed, 4 insertions(+), 7 deletions(-) diff --git a/connectors-sdk/connectors_sdk/__init__.py b/connectors-sdk/connectors_sdk/__init__.py index 3d412f3c111..c459bcbf627 100644 --- a/connectors-sdk/connectors_sdk/__init__.py +++ b/connectors-sdk/connectors_sdk/__init__.py @@ -2,7 +2,6 @@ __version__ = "0.1.0" -from connectors_sdk.connectors.external_import._work_manager import WorkManager from connectors_sdk.connectors.external_import.base_data_processor import ( BaseDataProcessor, ) @@ -34,6 +33,9 @@ from connectors_sdk.states.states import ExternalImportConnectorState __all__ = [ + # Base Connectors + "BaseDataProcessor", + "ExternalImportConnector", # Logger "Logger", # mostly for typing purposes # Base Settings @@ -56,8 +58,4 @@ "DeprecatedField", # Connector States "ExternalImportConnectorState", - # Connector base classes - "ExternalImportConnector", - "BaseDataProcessor", - "WorkManager", ] diff --git a/connectors-sdk/tests/test_api.py b/connectors-sdk/tests/test_api.py index 0302b5040f1..ee08648f531 100644 --- a/connectors-sdk/tests/test_api.py +++ b/connectors-sdk/tests/test_api.py @@ -11,6 +11,7 @@ def test_root_public_api_is_valid(): imports = { "BaseConnectorSettings", "BaseConfigModel", + "BaseDataProcessor", "BaseExternalImportConnectorConfig", "BaseInternalEnrichmentConnectorConfig", "BaseInternalExportFileConnectorConfig", @@ -23,8 +24,6 @@ def test_root_public_api_is_valid(): "DeprecatedField", "ExternalImportConnector", "ExternalImportConnectorState", - "BaseDataProcessor", - "WorkManager", "ListFromString", "Logger", "logger", From f0a2e648fc3e18c147889fdefd7d3e8c3e8377d0 Mon Sep 17 00:00:00 2001 From: Powlinett Date: Thu, 28 May 2026 19:09:37 +0200 Subject: [PATCH 06/13] test: add / update unit tests replace ConnectorLogger with new Logger when necessary --- connectors-sdk/tests/conftest.py | 12 + .../tests/{test_settings => }/data/.env.test | 2 +- .../{test_settings => }/data/config.test.yml | 2 +- connectors-sdk/tests/test_api.py | 1 - .../tests/test_connectors/conftest.py | 7 - .../test_connectors/test_data_processor.py | 42 +- .../test_external_import_connector.py | 21 +- .../test_connectors/test_work_manager.py | 102 ++--- connectors-sdk/tests/test_logging/conftest.py | 85 ++++ .../tests/test_logging/test_base_logger.py | 425 ++++++++++++++++++ .../tests/test_logging/test_logger.py | 79 ++++ .../tests/test_logging/test_sdk_logger.py | 74 +++ .../tests/test_settings/conftest.py | 17 +- .../tests/test_settings/test_base_settings.py | 250 +---------- .../test_settings/test_settings_loader.py | 247 ++++++++++ .../tests/tests_states/test_base_state.py | 9 + 16 files changed, 1019 insertions(+), 356 deletions(-) rename connectors-sdk/tests/{test_settings => }/data/.env.test (86%) rename connectors-sdk/tests/{test_settings => }/data/config.test.yml (89%) create mode 100644 connectors-sdk/tests/test_logging/conftest.py create mode 100644 connectors-sdk/tests/test_logging/test_base_logger.py create mode 100644 connectors-sdk/tests/test_logging/test_logger.py create mode 100644 connectors-sdk/tests/test_logging/test_sdk_logger.py create mode 100644 connectors-sdk/tests/test_settings/test_settings_loader.py diff --git a/connectors-sdk/tests/conftest.py b/connectors-sdk/tests/conftest.py index aa69dd1511b..a0e105d0f27 100644 --- a/connectors-sdk/tests/conftest.py +++ b/connectors-sdk/tests/conftest.py @@ -7,6 +7,7 @@ import subprocess import sys from pathlib import Path +from types import SimpleNamespace import pytest @@ -19,6 +20,17 @@ ) +@pytest.fixture +def mock_main_path(monkeypatch): + """Mock the path of `__main__.__file__` for `_SettingsLoader._get_connector_main_path` and + `BaseLogger._get_connector_main_path` calls. + """ + + monkeypatch.setitem( + sys.modules, "__main__", SimpleNamespace(__file__="/app/src/main.py") + ) + + @pytest.fixture def fake_valid_organization_author() -> OrganizationAuthor: """Fixture to create a fake valid OrganizationAuthor.""" diff --git a/connectors-sdk/tests/test_settings/data/.env.test b/connectors-sdk/tests/data/.env.test similarity index 86% rename from connectors-sdk/tests/test_settings/data/.env.test rename to connectors-sdk/tests/data/.env.test index bf5927c7dbe..f4b59c4df05 100644 --- a/connectors-sdk/tests/test_settings/data/.env.test +++ b/connectors-sdk/tests/data/.env.test @@ -3,5 +3,5 @@ OPENCTI_TOKEN=changeme CONNECTOR_ID=connector-poc--uid CONNECTOR_NAME=Test Connector CONNECTOR_SCOPE=test -CONNECTOR_LOG_LEVEL=error +CONNECTOR_LOG_LEVEL=debug CONNECTOR_DURATION_PERIOD=PT5M \ No newline at end of file diff --git a/connectors-sdk/tests/test_settings/data/config.test.yml b/connectors-sdk/tests/data/config.test.yml similarity index 89% rename from connectors-sdk/tests/test_settings/data/config.test.yml rename to connectors-sdk/tests/data/config.test.yml index 730f7b2787c..89ad916a168 100644 --- a/connectors-sdk/tests/test_settings/data/config.test.yml +++ b/connectors-sdk/tests/data/config.test.yml @@ -6,6 +6,6 @@ connector: id: connector-poc--uid name: Test Connector scope: test - log_level: error + log_level: debug duration_period: PT5M \ No newline at end of file diff --git a/connectors-sdk/tests/test_api.py b/connectors-sdk/tests/test_api.py index ee08648f531..6844a217a74 100644 --- a/connectors-sdk/tests/test_api.py +++ b/connectors-sdk/tests/test_api.py @@ -26,7 +26,6 @@ def test_root_public_api_is_valid(): "ExternalImportConnectorState", "ListFromString", "Logger", - "logger", } missing = imports - set(root_api.__all__) extra = set(root_api.__all__) - imports diff --git a/connectors-sdk/tests/test_connectors/conftest.py b/connectors-sdk/tests/test_connectors/conftest.py index 269502cbe28..57b10380719 100644 --- a/connectors-sdk/tests/test_connectors/conftest.py +++ b/connectors-sdk/tests/test_connectors/conftest.py @@ -4,7 +4,6 @@ from unittest.mock import MagicMock import pytest -from connectors_sdk.connectors.external_import.logger import ConnectorLogger @pytest.fixture @@ -26,12 +25,6 @@ def mock_helper() -> MagicMock: return helper -@pytest.fixture -def mock_logger(mock_helper: MagicMock) -> ConnectorLogger: - """ConnectorLogger backed by mock helper.""" - return ConnectorLogger(mock_helper) - - @pytest.fixture def mock_settings() -> MagicMock: """Mock BaseConnectorSettings with required attributes.""" diff --git a/connectors-sdk/tests/test_connectors/test_data_processor.py b/connectors-sdk/tests/test_connectors/test_data_processor.py index 334bfec3f3d..3743bb512ed 100644 --- a/connectors-sdk/tests/test_connectors/test_data_processor.py +++ b/connectors-sdk/tests/test_connectors/test_data_processor.py @@ -7,7 +7,7 @@ from connectors_sdk.connectors.external_import.base_data_processor import ( BaseDataProcessor, ) -from connectors_sdk.connectors.external_import.logger import ConnectorLogger +from connectors_sdk.logging.logger import Logger class ListProcessor(BaseDataProcessor): @@ -64,11 +64,11 @@ def transform(self, data: Any) -> Generator[list[Any], None, None]: class TestBaseDataProcessor: + def _attach_deps( self, processor: BaseDataProcessor, mock_helper: MagicMock, - mock_logger: ConnectorLogger, ) -> None: processor.inject_dependencies( settings=MagicMock(), @@ -77,46 +77,42 @@ def _attach_deps( ) processor.post_init() - def test_process_list(self, mock_helper: MagicMock, mock_logger: ConnectorLogger): + def test_init_subclass(self): + assert isinstance(ListProcessor.logger, Logger) + assert ListProcessor.logger._logger.name.endswith(".ListProcessor") + + def test_process_list(self, mock_helper: MagicMock): proc = ListProcessor() - self._attach_deps(proc, mock_helper, mock_logger) + self._attach_deps(proc, mock_helper) proc.process() mock_helper.api.work.initiate_work.assert_called_once() mock_helper.send_stix2_bundle.assert_called_once() - def test_process_generator( - self, mock_helper: MagicMock, mock_logger: ConnectorLogger - ): + def test_process_generator(self, mock_helper: MagicMock): proc = GeneratorProcessor() - self._attach_deps(proc, mock_helper, mock_logger) + self._attach_deps(proc, mock_helper) proc.process() mock_helper.api.work.initiate_work.assert_called_once() # 3 chunks → 3 send calls assert mock_helper.send_stix2_bundle.call_count == 3 - def test_process_empty_list( - self, mock_helper: MagicMock, mock_logger: ConnectorLogger - ): + def test_process_empty_list(self, mock_helper: MagicMock): proc = EmptyListProcessor() - self._attach_deps(proc, mock_helper, mock_logger) + self._attach_deps(proc, mock_helper) proc.process() # Empty list → no work created mock_helper.api.work.initiate_work.assert_not_called() - def test_process_generator_skips_empty_chunks( - self, mock_helper: MagicMock, mock_logger: ConnectorLogger - ): + def test_process_generator_skips_empty_chunks(self, mock_helper: MagicMock): proc = EmptyChunkGeneratorProcessor() - self._attach_deps(proc, mock_helper, mock_logger) + self._attach_deps(proc, mock_helper) proc.process() # Only 2 non-empty chunks sent assert mock_helper.send_stix2_bundle.call_count == 2 - def test_send_passes_work_name( - self, mock_helper: MagicMock, mock_logger: ConnectorLogger - ): + def test_send_passes_work_name(self, mock_helper: MagicMock): proc = ListProcessor() - self._attach_deps(proc, mock_helper, mock_logger) + self._attach_deps(proc, mock_helper) # Call send directly to verify work_name is passed with proc.work_manager: proc.send(["obj1", "obj2"]) @@ -124,9 +120,7 @@ def test_send_passes_work_name( "test-connector-id", "List Import" ) - def test_post_init_called( - self, mock_helper: MagicMock, mock_logger: ConnectorLogger - ): + def test_post_init_called(self, mock_helper: MagicMock): """post_init is called by _init_infrastructure after inject_dependencies.""" class TrackedProcessor(BaseDataProcessor): @@ -144,5 +138,5 @@ def transform(self, data): proc = TrackedProcessor() assert not proc.post_init_called - self._attach_deps(proc, mock_helper, mock_logger) + self._attach_deps(proc, mock_helper) assert proc.post_init_called diff --git a/connectors-sdk/tests/test_connectors/test_external_import_connector.py b/connectors-sdk/tests/test_connectors/test_external_import_connector.py index 55c56c18fbe..b23fa3684c4 100644 --- a/connectors-sdk/tests/test_connectors/test_external_import_connector.py +++ b/connectors-sdk/tests/test_connectors/test_external_import_connector.py @@ -1,5 +1,6 @@ # pragma: no cover # type: ignore +from datetime import datetime, timezone from typing import Any from unittest.mock import MagicMock, patch @@ -10,6 +11,7 @@ from connectors_sdk.connectors.external_import.external_import_connector import ( ExternalImportConnector, ) +from connectors_sdk.logging.logger import Logger PATCH_HELPER = "connectors_sdk.connectors.external_import.external_import_connector.OpenCTIConnectorHelper" @@ -56,7 +58,20 @@ def _make_state_mock() -> MagicMock: return state +def _make_logger_mock() -> MagicMock: + logger = MagicMock() + logger.error.return_value = None + return logger + + class TestExternalImportConnector: + def test_init_subclass(self): + class MyConnector(ExternalImportConnector): + pass + + assert isinstance(MyConnector.logger, Logger) + assert MyConnector.logger._logger.name.endswith(".MyConnector") + def test_init(self, mock_settings: MagicMock): proc = DummyProcessor() connector = ExternalImportConnector( @@ -130,7 +145,7 @@ def test_callback_with_last_run( helper = _make_helper_mock() mock_helper_cls.return_value = helper state = _make_state_mock() - state.last_run = "2025-01-01T00:00:00+00:00" + state.last_run = datetime(2025, 1, 1, tzinfo=timezone.utc) proc = DummyProcessor() connector = ExternalImportConnector( @@ -153,10 +168,12 @@ def test_callback_exception_is_logged( connector = ExternalImportConnector( settings=mock_settings, data_processors=[proc], state=state ) + connector.logger = _make_logger_mock() + connector._init_dependencies() connector.callback() - helper.connector_logger.error.assert_called() + connector.logger.error.assert_called() @patch(PATCH_HELPER) def test_callback_keyboard_interrupt( diff --git a/connectors-sdk/tests/test_connectors/test_work_manager.py b/connectors-sdk/tests/test_connectors/test_work_manager.py index 54f6b8e2db6..8f5d6615031 100644 --- a/connectors-sdk/tests/test_connectors/test_work_manager.py +++ b/connectors-sdk/tests/test_connectors/test_work_manager.py @@ -7,19 +7,23 @@ WorkManager, _Work, ) -from connectors_sdk.connectors.external_import.logger import ConnectorLogger +from connectors_sdk.logging.sdk_logger import SDKLogger class TestWork: - def test_init(self, mock_helper: MagicMock, mock_logger: ConnectorLogger): - work = _Work("w-1", "test-work", mock_helper, mock_logger) + def test_init_subclass(self): + assert isinstance(WorkManager.logger, SDKLogger) + assert WorkManager.logger._logger.name == "connectors_sdk.WorkManager" + + def test_init(self, mock_helper: MagicMock): + work = _Work(mock_helper, "w-1", "test-work") assert work.id == "w-1" assert work.name == "test-work" assert work._closed is False assert work._has_sent_bundles is False - def test_create(self, mock_helper: MagicMock, mock_logger: ConnectorLogger): - work = _Work.create(mock_helper, mock_logger, "test-work") + def test_create(self, mock_helper: MagicMock): + work = _Work.create(mock_helper, "test-work") mock_helper.api.work.initiate_work.assert_called_once_with( "test-connector-id", "test-work" ) @@ -27,17 +31,15 @@ def test_create(self, mock_helper: MagicMock, mock_logger: ConnectorLogger): assert work.name == "test-work" assert work._closed is False - def test_send_bundle(self, mock_helper: MagicMock, mock_logger: ConnectorLogger): - work = _Work("w-1", "test-work", mock_helper, mock_logger) + def test_send_bundle(self, mock_helper: MagicMock): + work = _Work(mock_helper, "w-1", "test-work") work.send_bundle(["obj1", "obj2"]) mock_helper.stix2_create_bundle.assert_called_once_with(["obj1", "obj2"]) mock_helper.send_stix2_bundle.assert_called_once() assert work._has_sent_bundles is True - def test_send_bundle_forwards_kwargs( - self, mock_helper: MagicMock, mock_logger: ConnectorLogger - ): - work = _Work("w-1", "test-work", mock_helper, mock_logger) + def test_send_bundle_forwards_kwargs(self, mock_helper: MagicMock): + work = _Work(mock_helper, "w-1", "test-work") work.send_bundle(["obj"], update=True) mock_helper.send_stix2_bundle.assert_called_once_with( mock_helper.stix2_create_bundle.return_value, @@ -45,22 +47,22 @@ def test_send_bundle_forwards_kwargs( update=True, ) - def test_success(self, mock_helper: MagicMock, mock_logger: ConnectorLogger): - work = _Work("w-1", "test-work", mock_helper, mock_logger) + def test_success(self, mock_helper: MagicMock): + work = _Work(mock_helper, "w-1", "test-work") work.success("Done") mock_helper.api.work.to_processed.assert_called_once_with("w-1", "Done") assert work._closed is True - def test_fail(self, mock_helper: MagicMock, mock_logger: ConnectorLogger): - work = _Work("w-1", "test-work", mock_helper, mock_logger) + def test_fail(self, mock_helper: MagicMock): + work = _Work(mock_helper, "w-1", "test-work") work.fail("Error occurred") mock_helper.api.work.to_processed.assert_called_once_with( "w-1", "Error occurred", in_error=True ) assert work._closed is True - def test_delete(self, mock_helper: MagicMock, mock_logger: ConnectorLogger): - work = _Work("w-1", "test-work", mock_helper, mock_logger) + def test_delete(self, mock_helper: MagicMock): + work = _Work(mock_helper, "w-1", "test-work") work._delete() mock_helper.api.work.delete.assert_called_once_with(id="w-1") assert work._closed is True @@ -75,23 +77,19 @@ def test_to_stix_converts_sdk_objects(self): class TestWorkManager: - def test_init(self, mock_helper: MagicMock, mock_logger: ConnectorLogger): + def test_init(self, mock_helper: MagicMock): wm = WorkManager(mock_helper) assert wm._current_work is None assert wm._active is False - def test_enter_exit_no_work( - self, mock_helper: MagicMock, mock_logger: ConnectorLogger - ): + def test_enter_exit_no_work(self, mock_helper: MagicMock): wm = WorkManager(mock_helper) with wm: assert wm._active is True assert wm._active is False assert wm._current_work is None - def test_send_creates_work_and_sends( - self, mock_helper: MagicMock, mock_logger: ConnectorLogger - ): + def test_send_creates_work_and_sends(self, mock_helper: MagicMock): wm = WorkManager(mock_helper) with wm: wm.send(["obj1"], "default") @@ -103,24 +101,18 @@ def test_send_creates_work_and_sends( "work-123", "Work completed successfully" ) - def test_send_empty_list_is_noop( - self, mock_helper: MagicMock, mock_logger: ConnectorLogger - ): + def test_send_empty_list_is_noop(self, mock_helper: MagicMock): wm = WorkManager(mock_helper) with wm: wm.send([], "default") mock_helper.api.work.initiate_work.assert_not_called() - def test_send_outside_context_raises( - self, mock_helper: MagicMock, mock_logger: ConnectorLogger - ): + def test_send_outside_context_raises(self, mock_helper: MagicMock): wm = WorkManager(mock_helper) with pytest.raises(RuntimeError, match="inside a 'with' block"): wm.send(["obj"], "default") - def test_send_same_name_reuses_work( - self, mock_helper: MagicMock, mock_logger: ConnectorLogger - ): + def test_send_same_name_reuses_work(self, mock_helper: MagicMock): wm = WorkManager(mock_helper) with wm: wm.send(["obj1"], "default") @@ -130,9 +122,7 @@ def test_send_same_name_reuses_work( # Two bundles sent assert mock_helper.send_stix2_bundle.call_count == 2 - def test_send_different_name_closes_previous( - self, mock_helper: MagicMock, mock_logger: ConnectorLogger - ): + def test_send_different_name_closes_previous(self, mock_helper: MagicMock): wm = WorkManager(mock_helper) with wm: wm.send(["obj1"], "default") # creates "default" work @@ -142,9 +132,7 @@ def test_send_different_name_closes_previous( # to_processed called: once for closing "default", once for closing "other" on exit assert mock_helper.api.work.to_processed.call_count == 2 - def test_exit_deletes_work_with_no_bundles( - self, mock_helper: MagicMock, mock_logger: ConnectorLogger - ): + def test_exit_deletes_work_with_no_bundles(self, mock_helper: MagicMock): """Work created but no bundles sent → deleted on exit.""" mock_helper.send_stix2_bundle.return_value = [] wm = WorkManager(mock_helper) @@ -157,15 +145,11 @@ def test_exit_deletes_work_with_no_bundles( wm2 = WorkManager(mock_helper) # Manually test: enter, init work, but don't send anything wm2.__enter__() - wm2._current_work = _Work( - "w-empty", "default", mock_helper, ConnectorLogger(mock_helper) - ) + wm2._current_work = _Work(mock_helper, "w-empty", "default") wm2.__exit__(None, None, None) mock_helper.api.work.delete.assert_called_once_with(id="w-empty") - def test_exit_on_exception_marks_failed( - self, mock_helper: MagicMock, mock_logger: ConnectorLogger - ): + def test_exit_on_exception_marks_failed(self, mock_helper: MagicMock): wm = WorkManager(mock_helper) try: with wm: @@ -177,33 +161,25 @@ def test_exit_on_exception_marks_failed( "work-123", "Work failed with error: boom", in_error=True ) - def test_exit_already_closed_work_is_noop( - self, mock_helper: MagicMock, mock_logger: ConnectorLogger - ): + def test_exit_already_closed_work_is_noop(self, mock_helper: MagicMock): wm = WorkManager(mock_helper) wm.__enter__() - work = _Work("w-1", "default", mock_helper, ConnectorLogger(mock_helper)) + work = _Work(mock_helper, "w-1", "default") work._closed = True wm._current_work = work wm.__exit__(None, None, None) mock_helper.api.work.to_processed.assert_not_called() mock_helper.api.work.delete.assert_not_called() - def test_close_current_work_no_bundles_deletes( - self, mock_helper: MagicMock, mock_logger: ConnectorLogger - ): + def test_close_current_work_no_bundles_deletes(self, mock_helper: MagicMock): wm = WorkManager(mock_helper) - wm._current_work = _Work( - "w-1", "default", mock_helper, ConnectorLogger(mock_helper) - ) + wm._current_work = _Work(mock_helper, "w-1", "default") wm._close_current_work() mock_helper.api.work.delete.assert_called_once_with(id="w-1") - def test_close_current_work_with_bundles_succeeds( - self, mock_helper: MagicMock, mock_logger: ConnectorLogger - ): + def test_close_current_work_with_bundles_succeeds(self, mock_helper: MagicMock): wm = WorkManager(mock_helper) - work = _Work("w-1", "default", mock_helper, ConnectorLogger(mock_helper)) + work = _Work(mock_helper, "w-1", "default") work._has_sent_bundles = True wm._current_work = work wm._close_current_work() @@ -211,15 +187,11 @@ def test_close_current_work_with_bundles_succeeds( "w-1", "Work completed successfully" ) - def test_close_current_work_none_is_noop( - self, mock_helper: MagicMock, mock_logger: ConnectorLogger - ): + def test_close_current_work_none_is_noop(self, mock_helper: MagicMock): wm = WorkManager(mock_helper) wm._close_current_work() # no exception - def test_send_forwards_kwargs_to_send_bundle( - self, mock_helper: MagicMock, mock_logger: ConnectorLogger - ): + def test_send_forwards_kwargs_to_send_bundle(self, mock_helper: MagicMock): wm = WorkManager(mock_helper) with wm: wm.send(["obj"], "default", update=True, entities_types=["Indicator"]) diff --git a/connectors-sdk/tests/test_logging/conftest.py b/connectors-sdk/tests/test_logging/conftest.py new file mode 100644 index 00000000000..b57a4dac186 --- /dev/null +++ b/connectors-sdk/tests/test_logging/conftest.py @@ -0,0 +1,85 @@ +import builtins +from pathlib import Path + +import pytest + +TESTS_DATA_DIRECTORY = Path(__file__).resolve().parents[1] / "data" + + +@pytest.fixture +def mock_connector_log_level_in_legacy_config_yml(mock_main_path, monkeypatch): + """Fake presence of legacy config.yml using real data file content.""" + main_path = Path("/app/src/main.py") + legacy_config_path = main_path.parent / "config.yml" + + # Fake the presence of legacy config.yml file + original_is_file = Path.is_file + monkeypatch.setattr( + Path, + "is_file", + lambda self: (self == legacy_config_path) or original_is_file(self), + ) + + # Patch open to return our test config.test.yml content instead of a real config.yml file + original_open = open + + def _open(path, mode="r", *args, **kwargs): + if Path(path) == legacy_config_path: + return original_open( + TESTS_DATA_DIRECTORY / "config.test.yml", mode, *args, **kwargs + ) + return original_open(path, mode, *args, **kwargs) + + monkeypatch.setattr(builtins, "open", _open) + + +@pytest.fixture +def mock_connector_log_level_in_config_yml(mock_main_path, monkeypatch): + """Fake presence of new config.yml using real data file content.""" + main_path = Path("/app/src/main.py") + new_config_path = main_path.parent.parent / "config.yml" + + # Fake the absence of legacy config.yml file and presence of new config.yml file + original_is_file = Path.is_file + monkeypatch.setattr( + Path, + "is_file", + lambda self: (self == new_config_path) or original_is_file(self), + ) + + # Patch open to return our test config.test.yml content instead of a real config.yml file + original_open = open + + def _open(path, mode="r", *args, **kwargs): + if Path(path) == new_config_path: + return original_open( + TESTS_DATA_DIRECTORY / "config.test.yml", mode, *args, **kwargs + ) + return original_open(path, mode, *args, **kwargs) + + monkeypatch.setattr(builtins, "open", _open) + + +@pytest.fixture +def mock_connector_log_level_in_dot_env(mock_main_path, monkeypatch): + """Fake presence of .env file using real data file content.""" + main_path = Path("/app/src/main.py") + dot_env_path = main_path.parent.parent / ".env" + + # Fake the absence fo config.yml files and presence of .env file + original_is_file = Path.is_file + monkeypatch.setattr( + Path, + "is_file", + lambda self: (self == dot_env_path) or original_is_file(self), + ) + + # Patch dotenv.get_key to read from our test .env.test file instead of a real .env file + from dotenv import dotenv_values + + def _get_key(path: Path, key: str): + if Path(path) == dot_env_path: + return dotenv_values(TESTS_DATA_DIRECTORY / ".env.test").get(key) + return None + + monkeypatch.setattr("dotenv.get_key", _get_key) diff --git a/connectors-sdk/tests/test_logging/test_base_logger.py b/connectors-sdk/tests/test_logging/test_base_logger.py new file mode 100644 index 00000000000..f2eb2a10005 --- /dev/null +++ b/connectors-sdk/tests/test_logging/test_base_logger.py @@ -0,0 +1,425 @@ +"""Tests for BaseLogger.""" + +import inspect +import logging +import logging.handlers +import sys +from pathlib import Path +from unittest.mock import patch + +import pytest +from connectors_sdk.logging._base_logger import BaseLogger, _prepare_meta + + +class DummyLogger(BaseLogger): + """Dummy concrete subclass of `BaseLogger` for testing purposes.""" + + def __init__(self, name: str) -> None: + super().__init__(name=name) + + +@pytest.fixture +def dummy_logger() -> BaseLogger: + """Provides a `DummyLogger` instance for testing.""" + + return DummyLogger(name="dummy_logger") + + +@pytest.fixture +def logger_factory(monkeypatch): + """Return a factory that creates `BaseLogger` subclasses with a given log level.""" + + def _make(level: str) -> BaseLogger: + monkeypatch.setenv("CONNECTOR_LOG_LEVEL", level) + + class DynamicLogger(BaseLogger): + # Define new subclass to trigger log level loading at class initialization + def __init__(self, name: str) -> None: + super().__init__(name=name) + + return DynamicLogger(name=f"{level}_logger") + + return _make + + +@pytest.fixture(autouse=True) +def cleanup_loggers(): + """Remove any test loggers created during a test.""" + yield + for name in list(logging.Logger.manager.loggerDict): + if name in ( + "dummy_logger", + "debug_logger", + "info_logger", + "warning_logger", + "error_logger", + ): + logger = logging.getLogger(name) + logger.handlers.clear() + del logging.Logger.manager.loggerDict[name] + + +class TestPrepareMeta: + def test_none_returns_none(self) -> None: + """Should return None when meta is None.""" + # Given/When: meta is None + result = _prepare_meta(None) + + # Then: returns None + assert result is None + + def test_dict_wrapped_in_attributes(self) -> None: + """Should wrap dict in 'attributes' key.""" + # Given: meta is a dict + meta = {"key": "value"} + + # When: _prepare_meta is called + result = _prepare_meta(meta) + + # Then: returns dict wrapped in 'attributes' + assert result == {"attributes": {"key": "value"}} + + def test_empty_dict_wrapped(self) -> None: + """Should wrap empty dict in 'attributes' key.""" + # Given: meta is an empty dict + meta = {} + + # When: _prepare_meta is called + result = _prepare_meta(meta) + + # Then: returns empty dict wrapped in 'attributes' + assert result == {"attributes": {}} + + +class TestBaseLoggerAbstract: + def test_base_logger_is_abstract(self) -> None: + """Should be an abstract class.""" + # Then: BaseLogger is abstract + assert inspect.isabstract(BaseLogger) + + def test_base_logger_cannot_be_instantited_directly(self) -> None: + """Should not allow direct instantiation.""" + # When/Then: Instantiating raises TypeError + with pytest.raises(TypeError): + BaseLogger(name="base") # type: ignore[abstract] + + def test_base_logger_has_default_log_level_class_var(self) -> None: + """Should have default log level class var 'ERROR'.""" + # Then: _log_level is 'ERROR' + assert BaseLogger._log_level == "ERROR" + + def test_base_logger_has__empty_handlers_class_var(self) -> None: + """Should have empty handlers class var by default.""" + # Then: _handlers is [] + assert BaseLogger._handlers == [] + + +class TestBaseLoggerConcreteClass: + def test_subclass_has_log_level(self) -> None: + """Should inherit log level from BaseLogger.""" + # Then: DummyLogger._log_level is 'ERROR' + assert DummyLogger._log_level == "ERROR" + + def test_subclass_sets_log_level_from_env_var(self, logger_factory) -> None: + """Should set log level from env var.""" + # Given: CONNECTOR_LOG_LEVEL=info + logger = logger_factory("info") + + # Then: logger._log_level is 'INFO' + assert logger._log_level == "INFO" + + def test_subclass_has_default_handlers(self) -> None: + """Should have one default StreamHandler.""" + # Then: DummyLogger._handlers contains one StreamHandler + assert len(DummyLogger._handlers) == 1 + assert isinstance(DummyLogger._handlers[0], logging.StreamHandler) + + +class TestBaseLoggerConcreteClassInit: + def test_subclass_creates_stdlib_logger_with_given_name( + self, dummy_logger: DummyLogger + ) -> None: + """Should create stdlib logger with the given name.""" + # Then: logger name matches + assert dummy_logger._logger.name == "dummy_logger" + + def test_subclass_sets_stdlib_logger_level_correctly( + self, dummy_logger: DummyLogger, logger_factory + ) -> None: + """Should set stdlib logger level according to log level.""" + # Given: dummy_logger and info_logger + info_logger = logger_factory("info") + + # Then: levels match + assert dummy_logger._log_level == "ERROR" + assert dummy_logger._logger.level == logging.ERROR + assert info_logger._log_level == "INFO" + assert info_logger._logger.level == logging.INFO + + def test_subclass_disables_stdlib_logger_propagate( + self, dummy_logger: DummyLogger + ) -> None: + """Should disable stdlib logger propagation.""" + # Then: propagate is False + assert dummy_logger._logger.propagate is False + + def test_subclass_adds_default_stream_handler_to_stdlib_logger( + self, dummy_logger: DummyLogger + ) -> None: + """Should add a default StreamHandler to stdlib logger.""" + # Then: at least one handler is a StreamHandler + assert any( + isinstance(h, logging.StreamHandler) for h in dummy_logger._logger.handlers + ) + + def test_subclass_adds_default_stream_handler_to_stdlib_logger_only_once( + self, + ) -> None: + """Should not duplicate handlers for same logger name.""" + # Given: two DummyLogger instances with same name + name = "dummy_logger_dedup" + DummyLogger(name=name) + DummyLogger(name=name) + + # When: get logger's handlers + logger = logging.getLogger(name) + default_handlers = [ + h for h in logger.handlers if h.get_name() == "default_stderr_handler" + ] + + # Then: only one default_stderr_handler + assert len(default_handlers) == 1 + + def test_subclass_default_stream_handler_has_json_formatter( + self, dummy_logger: DummyLogger + ) -> None: + """Should use CustomJsonFormatter for default handler.""" + from pycti.utils.opencti_logger import CustomJsonFormatter + + # When: get StreamHandler + handler = next( + h + for h in dummy_logger._logger.handlers + if isinstance(h, logging.StreamHandler) + ) + + # Then: formatter is CustomJsonFormatter + assert isinstance(handler.formatter, CustomJsonFormatter) + + +class TestBaseLoggerClassMethods: + def test_base_logger_get_connector_main_path_returns_main_file_path( + self, mock_main_path + ): + """Should locate connector's main.py via _get_connector_main_path.""" + # When: _get_connector_main_path is called + main_path = BaseLogger._get_connector_main_path() + + # Then: returns expected path + assert main_path == Path("/app/src/main.py").resolve() + + def test_base_logger_should_raise_when_main_module_misses_file_attribute( + self, mock_main_path + ): + """Should raise RuntimeError if __main__.__file__ is missing.""" + # Given: __main__.__file__ is missing + sys.modules["__main__"].__file__ = None + + # When/Then: calling _get_connector_main_path raises + with pytest.raises(RuntimeError): + BaseLogger._get_connector_main_path() + + def test_base_logger_get_log_level_from_legacy_config_yml( + self, mock_connector_log_level_in_legacy_config_yml + ): + """Should read log level from legacy config.yml.""" + # When: _get_log_level_from_config_yml is called + log_level = BaseLogger._get_log_level_from_config_yml() + + # Then: returns DEBUG + assert log_level == "DEBUG" + + def test_base_logger_get_log_level_from_config_yml( + self, mock_connector_log_level_in_config_yml + ): + """Should read log level from new config.yml path.""" + # When: _get_log_level_from_config_yml is called + log_level = BaseLogger._get_log_level_from_config_yml() + + # Then: returns DEBUG + assert log_level == "DEBUG" + + def test_base_logger_get_log_level_from_dot_env( + self, mock_connector_log_level_in_dot_env + ): + """Should read log level from .env file.""" + # When: _get_log_level_from_dot_env is called + log_level = BaseLogger._get_log_level_from_dot_env() + + # Then: returns DEBUG + assert log_level == "DEBUG" + + def test_base_logger_get_log_level_from_env_var(self, monkeypatch): + """Should use CONNECTOR_LOG_LEVEL env var if set.""" + # Given: env var set + monkeypatch.setenv("CONNECTOR_LOG_LEVEL", "debug") + + # When: _get_connector_log_level is called + log_level = BaseLogger._get_connector_log_level() + + # Then: returns DEBUG + assert log_level == "DEBUG" + + def test_get_connector_log_level_falls_back_to_config_yml(self, monkeypatch): + """Should fall back to config.yml if env var is not set.""" + # Given: config_yml returns DEBUG + monkeypatch.setattr( + BaseLogger, "_get_log_level_from_config_yml", staticmethod(lambda: "DEBUG") + ) + + # When: _get_connector_log_level is called + log_level = BaseLogger._get_connector_log_level() + + # Then: returns DEBUG + assert log_level == "DEBUG" + + def test_get_connector_log_level_falls_back_to_dot_env(self, monkeypatch): + """Should fall back to .env if env var and config.yml are not set.""" + # Given: config_yml returns empty, dot_env returns DEBUG + monkeypatch.setattr( + BaseLogger, "_get_log_level_from_config_yml", staticmethod(lambda: "") + ) + monkeypatch.setattr( + BaseLogger, "_get_log_level_from_dot_env", staticmethod(lambda: "DEBUG") + ) + + # When: _get_connector_log_level is called + log_level = BaseLogger._get_connector_log_level() + + # Then: returns DEBUG + assert log_level == "DEBUG" + + +class TestGetChild: + def test_subclass_get_child_returns_subclass_instance( + self, dummy_logger: DummyLogger + ) -> None: + """Should return a child logger instance of the same class.""" + # Given: a dummy_logger + # When: get_child is called + child = dummy_logger.get_child("child") + + # Then: child is DummyLogger + assert isinstance(child, DummyLogger) + + def test_subclass_logger_child_name_is_dotted( + self, dummy_logger: DummyLogger + ) -> None: + """Should append child name to logger name with dot.""" + # Given: a dummy_logger + # When: get_child is called + child = dummy_logger.get_child("child") + + # Then: child logger name is dotted + assert child._logger.name == "dummy_logger.child" + + +class TestBaseLoggerLogMethods: + def _capture_records_for( + self, logger: BaseLogger + ) -> logging.handlers.MemoryHandler: + """Attach a MemoryHandler to the logger for capturing log records.""" + mem = logging.handlers.MemoryHandler( + capacity=100, flushLevel=logging.CRITICAL + 1 + ) + logger._logger.addHandler(mem) + return mem + + @pytest.mark.parametrize("level", ["debug", "info", "warning", "error"]) + def test_log_methods_emit_records(self, logger_factory, level: str) -> None: + """Should emit a log record for each log method.""" + # Given: a logger at the right level + logger = logger_factory(level) + capture = self._capture_records_for(logger) + + # When: log method is called + log_method = getattr(logger, level) + log_method(f"{level} message") + + # Then: record is captured + assert any(r.getMessage() == f"{level} message" for r in capture.buffer) + + def test_error_emits_record_for_exception(self, logger_factory) -> None: + """Should emit a log record when error is called with an exception.""" + # Given: error-level logger + logger = logger_factory("error") + capture = self._capture_records_for(logger) + + # When: error is called with exception + logger.error(ValueError("An error occured")) + + # Then: record contains exception message + assert any("An error occured" in r.getMessage() for r in capture.buffer) + + def test_error_sets_exc_info(self, logger_factory) -> None: + """Should set exc_info on error log records.""" + # Given: error-level logger + logger = logger_factory("error") + capture = self._capture_records_for(logger) + + # When: error is called + logger.error("Message with exc_info") + + # Then: exc_info is set on the record + record = next( + r for r in capture.buffer if r.getMessage() == "Message with exc_info" + ) + assert record.exc_info is not None + + @pytest.mark.parametrize( + "meta,has_attributes", [({"key": "val"}, True), (None, False)] + ) + def test_meta_is_mapped_to_attributes( + self, logger_factory, meta: dict[str, str] | None, has_attributes: bool + ) -> None: + """Should map meta argument to log record attributes.""" + # Given: error-level logger + logger = logger_factory("error") + capture = self._capture_records_for(logger) + + # When: error is called with meta + logger.error("Message with meta", meta) + + # Then: attributes are present or not as expected + record = next( + r for r in capture.buffer if r.getMessage() == "Message with meta" + ) + if has_attributes: + assert record.attributes == {"key": "val"} # type: ignore[attr-defined] + else: + assert not hasattr(record, "attributes") + + @pytest.mark.parametrize("level", ["debug", "info", "warning", "error"]) + def test_methods_forward_expected_stdlib_arguments( + self, logger_factory, level: str + ) -> None: + """Should forward correct arguments to stdlib logger methods.""" + # Given: a logger + logger = logger_factory(level) + + # When: log method is called with meta + with patch.object(logger._logger, level) as spy: + log_method = getattr(logger, level) + log_method(f"{level} message", meta={"key": "value"}) + + # Then: correct arguments are forwarded + if level == "error": + spy.assert_called_once_with( + f"{level} message", + extra={"attributes": {"key": "value"}}, + exc_info=True, + ) + else: + spy.assert_called_once_with( + f"{level} message", extra={"attributes": {"key": "value"}} + ) diff --git a/connectors-sdk/tests/test_logging/test_logger.py b/connectors-sdk/tests/test_logging/test_logger.py new file mode 100644 index 00000000000..70756d1a935 --- /dev/null +++ b/connectors-sdk/tests/test_logging/test_logger.py @@ -0,0 +1,79 @@ +"""Tests for Logger.""" + +import logging + +import pytest +from connectors_sdk.logging._base_logger import BaseLogger +from connectors_sdk.logging.logger import Logger + + +@pytest.fixture(autouse=True) +def cleanup_loggers(): + yield + for name in list(logging.Logger.manager.loggerDict): + if name.startswith("connector"): + connector_logger = logging.getLogger(name) + connector_logger.handlers.clear() + del logging.Logger.manager.loggerDict[name] + + +class TestLoggerInit: + def test_inherits_from_base_logger(self) -> None: + """Should inherit from BaseLogger.""" + # Given/When: Logger is instantiated + # Then: It is an instance of BaseLogger + assert isinstance(Logger(name="test"), BaseLogger) + + def test_custom_name(self) -> None: + """Should allow custom logger name.""" + # When: Logger is instantiated with a custom name + connector_logger = Logger(name="my_connector") + + # Then: Name is set correctly + assert connector_logger._logger.name == "my_connector" + + def test_default_level_is_error(self) -> None: + """Should default log level to ERROR.""" + # When: Logger is instantiated + connector_logger = Logger(name="test") + + # Then: Level is ERROR + assert Logger._log_level == "ERROR" + assert connector_logger._logger.level == logging.ERROR + + def test_custom_level(self, monkeypatch) -> None: + """Should set log level from CONNECTOR_LOG_LEVEL env var.""" + # Given: CONNECTOR_LOG_LEVEL is set + monkeypatch.setenv("CONNECTOR_LOG_LEVEL", "info") + + # When: Logger subclass is instantiated + class InfoLogger( + Logger + ): # Recreate a new subclass to trigger log level reading + pass + + info_logger = InfoLogger(name="info_logger") + + # Then: Level is INFO + assert InfoLogger._log_level == "INFO" + assert info_logger._logger.level == logging.INFO + + def test_console_handler_present(self) -> None: + """Should attach a StreamHandler to the logger.""" + # When: Logger is instantiated + connector_logger = Logger(name="test") + + # Then: At least one handler is a StreamHandler + assert any( + isinstance(h, logging.StreamHandler) + for h in connector_logger._logger.handlers + ) + + +def test_import_connectors_sdk_should_not_configure_connector_logger() -> None: + """Importing `connectors_sdk` should not instantiate nor configure the `connector` logger.""" + import connectors_sdk # noqa: F401 + + connector_logger = logging.getLogger("connector") + assert connector_logger.handlers == [] + assert connector_logger.propagate is True diff --git a/connectors-sdk/tests/test_logging/test_sdk_logger.py b/connectors-sdk/tests/test_logging/test_sdk_logger.py new file mode 100644 index 00000000000..72ef1332d30 --- /dev/null +++ b/connectors-sdk/tests/test_logging/test_sdk_logger.py @@ -0,0 +1,74 @@ +"""Tests for SDKLogger singleton.""" + +import logging + +import pytest +from connectors_sdk.logging._base_logger import BaseLogger +from connectors_sdk.logging.sdk_logger import SDKLogger, sdk_logger + +_SDK_LOGGER_NAME = "connectors_sdk" + + +@pytest.fixture(autouse=True) +def cleanup_loggers(): + """Remove any test loggers created during a test.""" + yield + for name in list(logging.Logger.manager.loggerDict): + if name.startswith("test_base_logger"): + logger = logging.getLogger(name) + logger.handlers.clear() + del logging.Logger.manager.loggerDict[name] + + +@pytest.fixture +def stub_sdk_logger() -> SDKLogger: + """Return a SDKLogger instance.""" + return SDKLogger() + + +class TestSDKLoggerInit: + def test_inherits_from_base_logger(self) -> None: + """Should inherit from BaseLogger.""" + # When/Then: SDKLogger is a BaseLogger + assert isinstance(SDKLogger(), BaseLogger) + + def test_logger_name_must_start_with_connectors_sdk(self) -> None: + """Should raise if logger name does not start with 'connectors_sdk'.""" + # When/Then: Instantiating with invalid name raises ValueError + with pytest.raises( + ValueError, match="SDKLogger name must start with 'connectors_sdk'" + ): + SDKLogger(name="invalid_logger_name") + + def test_logger_default_name_is_connectors_sdk( + self, stub_sdk_logger: "SDKLogger" + ) -> None: + """Should default logger name to 'connectors_sdk'.""" + # When/Then: stub_sdk_logger name is correct + assert stub_sdk_logger._logger.name == _SDK_LOGGER_NAME + + def test_default_level_is_error(self, stub_sdk_logger: "SDKLogger") -> None: + """Should default log level to ERROR.""" + # When/Then: log level is ERROR + assert SDKLogger._log_level == "ERROR" + assert stub_sdk_logger._logger.level == logging.ERROR + + def test_console_handler_present(self, stub_sdk_logger: "SDKLogger") -> None: + """Should attach a StreamHandler to the SDKLogger.""" + # When/Then: At least one handler is a StreamHandler + assert any( + isinstance(h, logging.StreamHandler) + for h in stub_sdk_logger._logger.handlers + ) + + +class TestModuleLevelSDKLogger: + def test_sdk_logger_is_sdk_logger_instance(self) -> None: + """Should expose a module-level SDKLogger instance.""" + # When/Then: sdk_logger is a SDKLogger instance + assert isinstance(sdk_logger, SDKLogger) + + def test_sdk_logger_default_name(self) -> None: + """Should default module-level SDKLogger name to 'connectors_sdk'.""" + # When/Then: sdk_logger name is correct + assert sdk_logger._logger.name == _SDK_LOGGER_NAME diff --git a/connectors-sdk/tests/test_settings/conftest.py b/connectors-sdk/tests/test_settings/conftest.py index 42e4cd3371a..59fbbe84cc6 100644 --- a/connectors-sdk/tests/test_settings/conftest.py +++ b/connectors-sdk/tests/test_settings/conftest.py @@ -1,19 +1,8 @@ -import sys from pathlib import Path -from types import SimpleNamespace import pytest -@pytest.fixture -def mock_main_path(monkeypatch): - """Mock the path of `__main__.__file__` for `_SettingsLoader._get_connector_main_path` calls.""" - - monkeypatch.setitem( - sys.modules, "__main__", SimpleNamespace(__file__="/app/src/main.py") - ) - - @pytest.fixture def mock_environment(monkeypatch): """Mock `os.environ` for `_SettingsLoader` and `BaseConnectorSettings` calls.""" @@ -24,7 +13,7 @@ def mock_environment(monkeypatch): monkeypatch.setenv("CONNECTOR_NAME", "Test Connector") monkeypatch.setenv("CONNECTOR_SCOPE", "test") monkeypatch.setenv("CONNECTOR_DURATION_PERIOD", "PT5M") - monkeypatch.setenv("CONNECTOR_LOG_LEVEL", "error") + monkeypatch.setenv("CONNECTOR_LOG_LEVEL", "debug") @pytest.fixture @@ -32,7 +21,7 @@ def mock_config_yml_file_presence(monkeypatch): """Mock the path of `config.yml` for `_SettingsLoader` and `BaseConnectorSettings` calls.""" def get_config_yml_file_path(): - return Path(__file__).parent / "data" / "config.test.yml" + return Path(__file__).parent.parent / "data" / "config.test.yml" monkeypatch.setattr( "connectors_sdk.settings.base_settings._SettingsLoader._get_config_yml_file_path", @@ -45,7 +34,7 @@ def mock_dot_env_file_presence(monkeypatch): """Mock the path of `.env` for `_SettingsLoader` and `BaseConnectorSettings` calls.""" def get_dot_env_file_path(): - return Path(__file__).parent / "data" / ".env.test" + return Path(__file__).parent.parent / "data" / ".env.test" monkeypatch.setattr( "connectors_sdk.settings.base_settings._SettingsLoader._get_dot_env_file_path", diff --git a/connectors-sdk/tests/test_settings/test_base_settings.py b/connectors-sdk/tests/test_settings/test_base_settings.py index 2d8f3f9db9c..96fb71cafb4 100644 --- a/connectors-sdk/tests/test_settings/test_base_settings.py +++ b/connectors-sdk/tests/test_settings/test_base_settings.py @@ -1,13 +1,10 @@ -import sys -from pathlib import Path from typing import Annotated -from unittest.mock import patch import pytest +from connectors_sdk.logging.logger import Logger from connectors_sdk.settings.base_settings import ( BaseConfigModel, BaseConnectorSettings, - _SettingsLoader, ) from connectors_sdk.settings.deprecations import Deprecate, DeprecatedField from connectors_sdk.settings.exceptions import ConfigValidationError @@ -101,241 +98,12 @@ class TestConfig(BaseConfigModel): } -def test_settings_loader_should_get_connector_main_path(mock_main_path): - """ - Test that `_SettingsLoader._get_connector_main_path` locates connector's `main.py`. - For testing purpose, a fake path is assigned to `sys.modules[__main__].__file__`. - """ - - # Given: The connector main module path is available - # When: The main path resolver is executed - main_path = _SettingsLoader._get_connector_main_path() - - # Then: The resolved main.py path matches the expected connector location - assert main_path == Path("/app/src/main.py").resolve() - - -def test_settings_loader_should_raise_when_main_module_misses_file_attribute( - mock_main_path, -): - """ - Test that `_SettingsLoader._get_connector_main_path` raises a meaningful error in case `__main__.__file__` is missing. - For testing purpose, `sys.modules[__main__].__file__` is set to `None`. - """ - - # Given: The __main__.__file__ attribute is missing - sys.modules["__main__"].__file__ = None - - # When: The main path resolver is executed - # Then: A runtime error is raised to signal invalid execution context - with pytest.raises(RuntimeError): - _SettingsLoader._get_connector_main_path() - - -def test_settings_loader_should_get_legacy_config_yml_file_path( - mock_main_path, -): - """ - Test that `_SettingsLoader._get_config_yml_file_path` locates connector's `config.yml` (legacy path). - For testing purpose, a fake path is assigned to `sys.modules[__main__].__file__`. - """ - - def is_file(self: Path) -> bool: - return self.name == "config.yml" - - # Given: Legacy config file (/src/config.yml) is present - with patch("pathlib.Path.is_file", is_file): - # When: The config.yml path resolver is executed - config_yml_file_path = _SettingsLoader._get_config_yml_file_path() - - # Then: The legacy config.yml path is returned - assert config_yml_file_path == Path("/app/src/config.yml").resolve() - - -def test_settings_loader_should_get_config_yml_file_path(mock_main_path): - """ - Test that `_SettingsLoader._get_config_yml_file_path` locates connector's `config.yml` (new path). - For testing purpose, a fake path is assigned to `sys.modules[__main__].__file__`. - """ - - def is_file(self: Path) -> bool: - return self.name == "config.yml" and self.parent.name != "src" - - # Given: Root config file (/config.yml) is present - with patch("pathlib.Path.is_file", is_file): - # When: The config.yml path resolver is executed - config_yml_file_path = _SettingsLoader._get_config_yml_file_path() - - # Then: The new config.yml path is returned - assert config_yml_file_path == Path("/app/config.yml").resolve() - - -def test_settings_loader_should_get_dot_env_file_path(mock_main_path): - """ - Test that `_SettingsLoader._get_dot_env_file_path` locates connector's `.env`. - For testing purpose, a fake path is assigned to `sys.modules[__main__].__file__`. - """ - - def is_file(self: Path) -> bool: - return self.name == ".env" - - # Given: Root env file (/.env) is present - with patch("pathlib.Path.is_file", is_file): - # When: The .env path resolver is executed - dot_env_file_path = _SettingsLoader._get_dot_env_file_path() - - # Then: The .env file path is returned - assert dot_env_file_path == Path("/app/.env").resolve() - - -def test_settings_loader_should_parse_config_yml_file(mock_config_yml_file_presence): - """ - Test that `_SettingsLoader()` parses config vars in `config.yml`. - For testing purpose, the path of `config.yml` file is `tests/test_settings/data/config.test.yml`. - """ - # Given: A valid config.yml - # When: The settings loader is instantiated and dumped - settings_loader = _SettingsLoader() - settings_dict = settings_loader.model_dump() - - # Then: Parsed nested settings match expected config.yml values - assert settings_dict == { - "opencti": { - "url": "http://localhost:8080", - "token": "changeme", - }, - "connector": { - "id": "connector-poc--uid", - "name": "Test Connector", - "duration_period": "PT5M", - "log_level": "error", - "scope": "test", - }, - } - +def test_base_connector_settings_init_subclass(): + class MySettings(BaseConnectorSettings): + pass -def test_settings_loader_should_parse_dot_env_file(mock_dot_env_file_presence): - """ - Test that `_SettingsLoader()` parses env vars in `.env`. - For testing purpose, the path of `.env` file is `tests/test_settings/data/.env.test`. - """ - - # Given: A valid .env file - # When: The settings loader is instantiated and dumped - settings_loader = _SettingsLoader() - settings_dict = settings_loader.model_dump() - - # Then: Parsed flat settings match expected environment variables - assert settings_dict == { - "opencti_url": "http://localhost:8080", - "opencti_token": "changeme", - "connector_id": "connector-poc--uid", - "connector_name": "Test Connector", - "connector_duration_period": "PT5M", - "connector_log_level": "error", - "connector_scope": "test", - } - - -def test_settings_loader_should_not_parse_os_environ(mock_environment): - """ - Test that `_SettingsLoader()` does not parse env vars from `os.environ` (for security purposes). - For testing purpose, `os.environ` is patched. - """ - - # Given: Valid environment variables - # When: The settings loader is instantiated and dumped - settings_loader = _SettingsLoader() - settings_dict = settings_loader.model_dump() - - # Then: No implicit values are parsed - assert settings_dict == {} - - -def test_settings_loader_should_parse_config_yml_from_model( - mock_config_yml_file_presence, -): - """ - Test that `_SettingsLoader.build_loader_from_model` returns a `BaseSettings` subclass - capable of parsing `config.yml` according to the given `BaseModel`. - For testing purpose, the path of `config.yml` file is `tests/test_settings/data/config.test.yml`. - """ - - # Given: A model-aware loader is built for BaseConnectorSettings with config.yml fixture - # When: The loader instance parses and dumps values - settings_loader = _SettingsLoader.build_loader_from_model(BaseConnectorSettings) - settings_dict = settings_loader().model_dump() - - # Then: Parsed nested settings expose expected OpenCTI and connector values - assert settings_dict["opencti"]["url"] == "http://localhost:8080" - assert settings_dict["opencti"]["token"] == "changeme" - assert settings_dict["connector"]["id"] == "connector-poc--uid" - assert settings_dict["connector"]["name"] == "Test Connector" - assert settings_dict["connector"]["scope"] == "test" - assert settings_dict["connector"]["log_level"] == "error" - - -def test_settings_loader_should_parse_dot_env_from_model(mock_dot_env_file_presence): - """ - Test that `_SettingsLoader.build_loader_from_model` returns a `BaseSettings` subclass - capable of parsing `.env` according to the given `BaseModel`. - For testing purpose, the path of `.env` file is `tests/test_settings/data/.env.test`. - """ - - # Given: A model-aware loader is built for BaseConnectorSettings with .env fixture - # When: The loader instance parses and dumps values - settings_loader = _SettingsLoader.build_loader_from_model(BaseConnectorSettings) - settings_dict = settings_loader().model_dump() - - # Then: Parsed nested settings expose expected OpenCTI and connector values - assert settings_dict["opencti"]["url"] == "http://localhost:8080" - assert settings_dict["opencti"]["token"] == "changeme" - assert settings_dict["connector"]["id"] == "connector-poc--uid" - assert settings_dict["connector"]["name"] == "Test Connector" - assert settings_dict["connector"]["scope"] == "test" - assert settings_dict["connector"]["log_level"] == "error" - - -def test_settings_loader_should_parse_os_environ_from_model(mock_environment): - """ - Test that `_SettingsLoader.build_loader_from_model` returns a `BaseSettings` subclass - capable of parsing `os.environ` according to the given `BaseModel`. - For testing purpose, `os.environ` is patched. - """ - - # Given: A model-aware loader is built for BaseConnectorSettings with patched os.environ - # When: The loader instance parses and dumps values - settings_loader = _SettingsLoader.build_loader_from_model(BaseConnectorSettings) - settings_dict = settings_loader().model_dump() - - # Then: Parsed nested settings expose expected OpenCTI and connector values - assert settings_dict["opencti"]["url"] == "http://localhost:8080" - assert settings_dict["opencti"]["token"] == "changeme" - assert settings_dict["connector"]["id"] == "connector-poc--uid" - assert settings_dict["connector"]["name"] == "Test Connector" - assert settings_dict["connector"]["scope"] == "test" - assert settings_dict["connector"]["log_level"] == "error" - - -def test_base_connector_settings_should_validate_settings_from_config_yaml_file( - mock_config_yml_file_presence, -): - """ - Test that `BaseConnectorSettings` casts and validates config vars in `config.yml`. - For testing purpose, the path of `config.yml` file is `tests/test_settings/data/config.test.yml`. - """ - - # Given: Valid connector settings are provided through config.yml fixture - # When: BaseConnectorSettings is instantiated - settings = BaseConnectorSettings() - - # Then: Values are validated and cast to expected runtime types - assert settings.opencti.url == HttpUrl("http://localhost:8080/") - assert settings.opencti.token == "changeme" - assert settings.connector.id == "connector-poc--uid" - assert settings.connector.name == "Test Connector" - assert settings.connector.scope == ["test"] - assert settings.connector.log_level == "error" + assert isinstance(MySettings.logger, Logger) + assert MySettings.logger._logger.name.endswith(".MySettings") def test_base_connector_settings_should_validate_settings_from_dot_env_file( @@ -356,7 +124,7 @@ def test_base_connector_settings_should_validate_settings_from_dot_env_file( assert settings.connector.id == "connector-poc--uid" assert settings.connector.name == "Test Connector" assert settings.connector.scope == ["test"] - assert settings.connector.log_level == "error" + assert settings.connector.log_level == "debug" def test_base_connector_settings_should_validate_settings_from_os_environ( @@ -377,7 +145,7 @@ def test_base_connector_settings_should_validate_settings_from_os_environ( assert settings.connector.id == "connector-poc--uid" assert settings.connector.name == "Test Connector" assert settings.connector.scope == ["test"] - assert settings.connector.log_level == "error" + assert settings.connector.log_level == "debug" def test_base_connector_settings_should_raise_when_missing_mandatory_env_vars(): @@ -405,7 +173,7 @@ def test_base_connector_settings_should_provide_helper_config(mock_environment): "connector": { "duration_period": "PT5M", "id": "connector-poc--uid", - "log_level": "error", + "log_level": "debug", "name": "Test Connector", "scope": "test", }, diff --git a/connectors-sdk/tests/test_settings/test_settings_loader.py b/connectors-sdk/tests/test_settings/test_settings_loader.py new file mode 100644 index 00000000000..7d13b433165 --- /dev/null +++ b/connectors-sdk/tests/test_settings/test_settings_loader.py @@ -0,0 +1,247 @@ +import sys +from pathlib import Path +from unittest.mock import patch + +import pytest +from connectors_sdk.settings.base_settings import ( + BaseConnectorSettings, + _SettingsLoader, +) +from pydantic import HttpUrl + + +def test_settings_loader_should_get_connector_main_path(mock_main_path): + """ + Test that `_SettingsLoader._get_connector_main_path` locates connector's `main.py`. + For testing purpose, a fake path is assigned to `sys.modules[__main__].__file__`. + """ + + # Given: The connector main module path is available + # When: The main path resolver is executed + main_path = _SettingsLoader._get_connector_main_path() + + # Then: The resolved main.py path matches the expected connector location + assert main_path == Path("/app/src/main.py").resolve() + + +def test_settings_loader_should_raise_when_main_module_misses_file_attribute( + mock_main_path, +): + """ + Test that `_SettingsLoader._get_connector_main_path` raises a meaningful error in case `__main__.__file__` is missing. + For testing purpose, `sys.modules[__main__].__file__` is set to `None`. + """ + + # Given: The __main__.__file__ attribute is missing + sys.modules["__main__"].__file__ = None + + # When: The main path resolver is executed + # Then: A runtime error is raised to signal invalid execution context + with pytest.raises(RuntimeError): + _SettingsLoader._get_connector_main_path() + + +def test_settings_loader_should_get_legacy_config_yml_file_path( + mock_main_path, +): + """ + Test that `_SettingsLoader._get_config_yml_file_path` locates connector's `config.yml` (legacy path). + For testing purpose, a fake path is assigned to `sys.modules[__main__].__file__`. + """ + + def is_file(self: Path) -> bool: + return self.name == "config.yml" + + # Given: Legacy config file (/src/config.yml) is present + with patch("pathlib.Path.is_file", is_file): + # When: The config.yml path resolver is executed + config_yml_file_path = _SettingsLoader._get_config_yml_file_path() + + # Then: The legacy config.yml path is returned + assert config_yml_file_path == Path("/app/src/config.yml").resolve() + + +def test_settings_loader_should_get_config_yml_file_path(mock_main_path): + """ + Test that `_SettingsLoader._get_config_yml_file_path` locates connector's `config.yml` (new path). + For testing purpose, a fake path is assigned to `sys.modules[__main__].__file__`. + """ + + def is_file(self: Path) -> bool: + return self.name == "config.yml" and self.parent.name != "src" + + # Given: Root config file (/config.yml) is present + with patch("pathlib.Path.is_file", is_file): + # When: The config.yml path resolver is executed + config_yml_file_path = _SettingsLoader._get_config_yml_file_path() + + # Then: The new config.yml path is returned + assert config_yml_file_path == Path("/app/config.yml").resolve() + + +def test_settings_loader_should_get_dot_env_file_path(mock_main_path): + """ + Test that `_SettingsLoader._get_dot_env_file_path` locates connector's `.env`. + For testing purpose, a fake path is assigned to `sys.modules[__main__].__file__`. + """ + + def is_file(self: Path) -> bool: + return self.name == ".env" + + # Given: Root env file (/.env) is present + with patch("pathlib.Path.is_file", is_file): + # When: The .env path resolver is executed + dot_env_file_path = _SettingsLoader._get_dot_env_file_path() + + # Then: The .env file path is returned + assert dot_env_file_path == Path("/app/.env").resolve() + + +def test_settings_loader_should_parse_config_yml_file(mock_config_yml_file_presence): + """ + Test that `_SettingsLoader()` parses config vars in `config.yml`. + For testing purpose, the path of `config.yml` file is `tests/test_settings/data/config.test.yml`. + """ + # Given: A valid config.yml + # When: The settings loader is instantiated and dumped + settings_loader = _SettingsLoader() + settings_dict = settings_loader.model_dump() + + # Then: Parsed nested settings match expected config.yml values + assert settings_dict == { + "opencti": { + "url": "http://localhost:8080", + "token": "changeme", + }, + "connector": { + "id": "connector-poc--uid", + "name": "Test Connector", + "duration_period": "PT5M", + "log_level": "debug", + "scope": "test", + }, + } + + +def test_settings_loader_should_parse_dot_env_file(mock_dot_env_file_presence): + """ + Test that `_SettingsLoader()` parses env vars in `.env`. + For testing purpose, the path of `.env` file is `tests/test_settings/data/.env.test`. + """ + + # Given: A valid .env file + # When: The settings loader is instantiated and dumped + settings_loader = _SettingsLoader() + settings_dict = settings_loader.model_dump() + + # Then: Parsed flat settings match expected environment variables + assert settings_dict == { + "opencti_url": "http://localhost:8080", + "opencti_token": "changeme", + "connector_id": "connector-poc--uid", + "connector_name": "Test Connector", + "connector_duration_period": "PT5M", + "connector_log_level": "debug", + "connector_scope": "test", + } + + +def test_settings_loader_should_not_parse_os_environ(mock_environment): + """ + Test that `_SettingsLoader()` does not parse env vars from `os.environ` (for security purposes). + For testing purpose, `os.environ` is patched. + """ + + # Given: Valid environment variables + # When: The settings loader is instantiated and dumped + settings_loader = _SettingsLoader() + settings_dict = settings_loader.model_dump() + + # Then: No implicit values are parsed + assert settings_dict == {} + + +def test_settings_loader_should_parse_config_yml_from_model( + mock_config_yml_file_presence, +): + """ + Test that `_SettingsLoader.build_loader_from_model` returns a `BaseSettings` subclass + capable of parsing `config.yml` according to the given `BaseModel`. + For testing purpose, the path of `config.yml` file is `tests/test_settings/data/config.test.yml`. + """ + + # Given: A model-aware loader is built for BaseConnectorSettings with config.yml fixture + # When: The loader instance parses and dumps values + settings_loader = _SettingsLoader.build_loader_from_model(BaseConnectorSettings) + settings_dict = settings_loader().model_dump() + + # Then: Parsed nested settings expose expected OpenCTI and connector values + assert settings_dict["opencti"]["url"] == "http://localhost:8080" + assert settings_dict["opencti"]["token"] == "changeme" + assert settings_dict["connector"]["id"] == "connector-poc--uid" + assert settings_dict["connector"]["name"] == "Test Connector" + assert settings_dict["connector"]["scope"] == "test" + assert settings_dict["connector"]["log_level"] == "debug" + + +def test_settings_loader_should_parse_dot_env_from_model(mock_dot_env_file_presence): + """ + Test that `_SettingsLoader.build_loader_from_model` returns a `BaseSettings` subclass + capable of parsing `.env` according to the given `BaseModel`. + For testing purpose, the path of `.env` file is `tests/test_settings/data/.env.test`. + """ + + # Given: A model-aware loader is built for BaseConnectorSettings with .env fixture + # When: The loader instance parses and dumps values + settings_loader = _SettingsLoader.build_loader_from_model(BaseConnectorSettings) + settings_dict = settings_loader().model_dump() + + # Then: Parsed nested settings expose expected OpenCTI and connector values + assert settings_dict["opencti"]["url"] == "http://localhost:8080" + assert settings_dict["opencti"]["token"] == "changeme" + assert settings_dict["connector"]["id"] == "connector-poc--uid" + assert settings_dict["connector"]["name"] == "Test Connector" + assert settings_dict["connector"]["scope"] == "test" + assert settings_dict["connector"]["log_level"] == "debug" + + +def test_settings_loader_should_parse_os_environ_from_model(mock_environment): + """ + Test that `_SettingsLoader.build_loader_from_model` returns a `BaseSettings` subclass + capable of parsing `os.environ` according to the given `BaseModel`. + For testing purpose, `os.environ` is patched. + """ + + # Given: A model-aware loader is built for BaseConnectorSettings with patched os.environ + # When: The loader instance parses and dumps values + settings_loader = _SettingsLoader.build_loader_from_model(BaseConnectorSettings) + settings_dict = settings_loader().model_dump() + + # Then: Parsed nested settings expose expected OpenCTI and connector values + assert settings_dict["opencti"]["url"] == "http://localhost:8080" + assert settings_dict["opencti"]["token"] == "changeme" + assert settings_dict["connector"]["id"] == "connector-poc--uid" + assert settings_dict["connector"]["name"] == "Test Connector" + assert settings_dict["connector"]["scope"] == "test" + assert settings_dict["connector"]["log_level"] == "debug" + + +def test_base_connector_settings_should_validate_settings_from_config_yaml_file( + mock_config_yml_file_presence, +): + """ + Test that `BaseConnectorSettings` casts and validates config vars in `config.yml`. + For testing purpose, the path of `config.yml` file is `tests/test_settings/data/config.test.yml`. + """ + + # Given: Valid connector settings are provided through config.yml fixture + # When: BaseConnectorSettings is instantiated + settings = BaseConnectorSettings() + + # Then: Values are validated and cast to expected runtime types + assert settings.opencti.url == HttpUrl("http://localhost:8080/") + assert settings.opencti.token == "changeme" + assert settings.connector.id == "connector-poc--uid" + assert settings.connector.name == "Test Connector" + assert settings.connector.scope == ["test"] + assert settings.connector.log_level == "debug" diff --git a/connectors-sdk/tests/tests_states/test_base_state.py b/connectors-sdk/tests/tests_states/test_base_state.py index 52fc78f6c86..ad63989b380 100644 --- a/connectors-sdk/tests/tests_states/test_base_state.py +++ b/connectors-sdk/tests/tests_states/test_base_state.py @@ -3,6 +3,7 @@ from datetime import datetime, timezone import pytest +from connectors_sdk.logging.logger import Logger from connectors_sdk.states._base_state import BaseConnectorState from pydantic import BaseModel, ValidationError @@ -22,6 +23,14 @@ def dummy_state(mock_opencti_connector_helper) -> DummyConnectorState: return state +def test_base_connector_state_init_subclass(): + class MyConnectorState(BaseConnectorState): + pass + + assert isinstance(MyConnectorState.logger, Logger) + assert MyConnectorState.logger._logger.name.endswith(".MyConnectorState") + + def test_base_connector_state_is_pydantic_model() -> None: """Test that `BaseConnectorState` is a Pydantic model.""" assert issubclass(BaseConnectorState, BaseModel) From 63b8b7977a585394a6cc542f2180ea58330102fa Mon Sep 17 00:00:00 2001 From: Powlinett Date: Thu, 28 May 2026 19:10:14 +0200 Subject: [PATCH 07/13] doc: update code examples --- ...ping_and_validation_of_connector_state_with_pydantic.md | 7 ++++++- .../docs/Dummy_end_to_end_test_octi_models_example.md | 6 ++++-- 2 files changed, 10 insertions(+), 3 deletions(-) diff --git a/connectors-sdk/TDRs/2026-04-14-Typing_and_validation_of_connector_state_with_pydantic.md b/connectors-sdk/TDRs/2026-04-14-Typing_and_validation_of_connector_state_with_pydantic.md index bed3ea9fcbf..a5bcef05a67 100644 --- a/connectors-sdk/TDRs/2026-04-14-Typing_and_validation_of_connector_state_with_pydantic.md +++ b/connectors-sdk/TDRs/2026-04-14-Typing_and_validation_of_connector_state_with_pydantic.md @@ -101,7 +101,7 @@ state.attach_opencti_connector_helper(helper) # establish the connection with Op state.load() if state.last_run: - self.helper.connector_logger.info("Last run:", {"last_run": state.last_run}) + logger.info("Last run:", {"last_run": state.last_run}) state.last_run = datetime.now(tz=timezone.utc) state.save() @@ -203,3 +203,8 @@ The `states` module has no dependency on other SDK modules (`settings`, `models` - Related TDR: [Typing and validation of configurations with Pydantic Settings](https://github.com/OpenCTI-Platform/connectors/blob/master/connectors-sdk/TDRs/2025-10-01-Typing_and_validation_of_configurations_with_Pydantic_Settings.md) - [Pydantic BaseModel documentation](https://docs.pydantic.dev/latest/concepts/models/) - [Pydantic serialization (JSON mode)](https://docs.pydantic.dev/latest/concepts/serialization/#json-mode) + + +## Update (2026-05-28) + +Update logging in code examples. No technical changes. \ No newline at end of file diff --git a/connectors-sdk/docs/Dummy_end_to_end_test_octi_models_example.md b/connectors-sdk/docs/Dummy_end_to_end_test_octi_models_example.md index 9575e034a0c..2b74cad89a5 100644 --- a/connectors-sdk/docs/Dummy_end_to_end_test_octi_models_example.md +++ b/connectors-sdk/docs/Dummy_end_to_end_test_octi_models_example.md @@ -19,10 +19,12 @@ from connectors_sdk.models import ( TLPMarking, ) from connectors_sdk.models.octi import based_on, related_to +from connectors_sdk import Logger if TYPE_CHECKING: from connectors_sdk.models import BaseObject +logger = Logger("connector") class ConnectorExample: """Example OpenCTI connector using connectors-sdk.""" @@ -123,13 +125,13 @@ class ConnectorExample: self.helper.api.work.to_processed(work_id, "Done") except (KeyboardInterrupt, SystemExit): - self.helper.connector_logger.info( + logger.info( "[CONNECTOR] Connector stopped...", {"connector_name": self.helper.connect_name}, ) sys.exit(0) except Exception as err: - self.helper.connector_logger.error(str(err)) + logger.error(str(err)) def run(self) -> None: """Run the connector.""" From 23a3b7e479c7d8ffa9e8f35c1a7e91f4635605aa Mon Sep 17 00:00:00 2001 From: Powlinett Date: Fri, 3 Jul 2026 16:07:48 +0200 Subject: [PATCH 08/13] fix: fix docstrings and typos --- .../connectors/external_import/_work_manager.py | 4 ++-- .../connectors/external_import/base_data_processor.py | 2 +- .../connectors/external_import/external_import_connector.py | 4 ++-- connectors-sdk/connectors_sdk/logging/_base_logger.py | 1 - connectors-sdk/connectors_sdk/settings/base_settings.py | 2 +- connectors-sdk/connectors_sdk/states/_base_state.py | 2 +- connectors-sdk/tests/test_logging/test_base_logger.py | 4 ++-- 7 files changed, 9 insertions(+), 10 deletions(-) diff --git a/connectors-sdk/connectors_sdk/connectors/external_import/_work_manager.py b/connectors-sdk/connectors_sdk/connectors/external_import/_work_manager.py index 4668b51be1f..660af3666d6 100644 --- a/connectors-sdk/connectors_sdk/connectors/external_import/_work_manager.py +++ b/connectors-sdk/connectors_sdk/connectors/external_import/_work_manager.py @@ -61,7 +61,7 @@ def __init__( self._closed = False self._has_sent_bundles = False - self.logger.debug(f"{self.__class__.__name__} instantiated succesfully") + self.logger.debug(f"{self.__class__.__name__} instantiated successfully") @classmethod def create(cls, helper: OpenCTIConnectorHelper, work_name: str) -> _Work: @@ -200,7 +200,7 @@ def __init__(self, helper: OpenCTIConnectorHelper) -> None: self._current_work: _Work | None = None self._active = False - self.logger.debug(f"{self.__class__.__name__} instantiated succesfully") + self.logger.debug(f"{self.__class__.__name__} instantiated successfully") def __enter__(self) -> WorkManager: """Enter the context manager.""" diff --git a/connectors-sdk/connectors_sdk/connectors/external_import/base_data_processor.py b/connectors-sdk/connectors_sdk/connectors/external_import/base_data_processor.py index 654b3d14d06..0179679ff0e 100644 --- a/connectors-sdk/connectors_sdk/connectors/external_import/base_data_processor.py +++ b/connectors-sdk/connectors_sdk/connectors/external_import/base_data_processor.py @@ -89,7 +89,7 @@ def __init_subclass__(cls, **kwargs: Any) -> None: def __init__(self) -> None: """Initialize the processor.""" - self.logger.debug(f"{self.__class__.__name__} instantiated succesfully") + self.logger.debug(f"{self.__class__.__name__} instantiated successfully") def inject_dependencies( self, diff --git a/connectors-sdk/connectors_sdk/connectors/external_import/external_import_connector.py b/connectors-sdk/connectors_sdk/connectors/external_import/external_import_connector.py index 914a459ed11..0404b9d4be2 100644 --- a/connectors-sdk/connectors_sdk/connectors/external_import/external_import_connector.py +++ b/connectors-sdk/connectors_sdk/connectors/external_import/external_import_connector.py @@ -8,7 +8,7 @@ ExternalImportConnector ├── OpenCTIConnectorHelper → pycti bridge (created in _init_dependencies) - ├── Logger → Logging (wraps helper's AppLogger) + ├── Logger → Logging (stdlib logging with pycti's CustomJsonFormatter) ├── ExternalImportConnectorState → State persistence (last_run, custom fields) └── BaseDataProcessor[] → process(): with work_manager: send(transform(collect())) └── WorkManager → context manager: open work → send → close work @@ -106,7 +106,7 @@ def __init__( self.state = state if state is not None else ExternalImportConnectorState() self.logger.debug( - f"{self.__class__.__name__} instantiated succesfully with {len(data_processors)} processor(s)", + f"{self.__class__.__name__} instantiated successfully with {len(data_processors)} processor(s)", {"data_processors": [p.__class__.__name__ for p in data_processors]}, ) diff --git a/connectors-sdk/connectors_sdk/logging/_base_logger.py b/connectors-sdk/connectors_sdk/logging/_base_logger.py index 5cf8fccbdce..2a372cc1fe6 100644 --- a/connectors-sdk/connectors_sdk/logging/_base_logger.py +++ b/connectors-sdk/connectors_sdk/logging/_base_logger.py @@ -32,7 +32,6 @@ def _prepare_meta(meta: dict[str, Any] | None = None) -> dict[str, Any] | None: The format is the same as the one used in pycti's `AppLogger`, in order to maintain consistency. Args: - message: The log message. meta: Optional metadata dict. """ return None if meta is None else {"attributes": meta} diff --git a/connectors-sdk/connectors_sdk/settings/base_settings.py b/connectors-sdk/connectors_sdk/settings/base_settings.py index d348f7e76e6..0e2bb65fea8 100644 --- a/connectors-sdk/connectors_sdk/settings/base_settings.py +++ b/connectors-sdk/connectors_sdk/settings/base_settings.py @@ -255,7 +255,7 @@ def __init__(self) -> None: raise ConfigValidationError("Error validating configuration.") from e self.logger.debug( - f"{self.__class__.__name__} instantiated succesfully", + f"{self.__class__.__name__} instantiated successfully", {"settings": self.model_dump(mode="json")}, ) diff --git a/connectors-sdk/connectors_sdk/states/_base_state.py b/connectors-sdk/connectors_sdk/states/_base_state.py index a76ca8e04ce..1f0ff2c1ad1 100644 --- a/connectors-sdk/connectors_sdk/states/_base_state.py +++ b/connectors-sdk/connectors_sdk/states/_base_state.py @@ -48,7 +48,7 @@ def __init__(self, helper: OpenCTIConnectorHelper) -> None: """ self._helper = helper - self.logger.debug(f"{self.__class__.__name__} instantiated succesfully") + self.logger.debug(f"{self.__class__.__name__} instantiated successfully") def load_state(self) -> dict[str, Any]: """Get connector's state stored on OpenCTI.""" diff --git a/connectors-sdk/tests/test_logging/test_base_logger.py b/connectors-sdk/tests/test_logging/test_base_logger.py index f2eb2a10005..a65b66455e9 100644 --- a/connectors-sdk/tests/test_logging/test_base_logger.py +++ b/connectors-sdk/tests/test_logging/test_base_logger.py @@ -356,10 +356,10 @@ def test_error_emits_record_for_exception(self, logger_factory) -> None: capture = self._capture_records_for(logger) # When: error is called with exception - logger.error(ValueError("An error occured")) + logger.error(ValueError("An error occurred")) # Then: record contains exception message - assert any("An error occured" in r.getMessage() for r in capture.buffer) + assert any("An error occurred" in r.getMessage() for r in capture.buffer) def test_error_sets_exc_info(self, logger_factory) -> None: """Should set exc_info on error log records.""" From f5dec1bc2f3edcd70c12ae5dd81998756336fcc1 Mon Sep 17 00:00:00 2001 From: Powlinett Date: Fri, 3 Jul 2026 16:26:53 +0200 Subject: [PATCH 09/13] fix: remove sensitive debug log comment https://github.com/OpenCTI-Platform/connectors/pull/6559#discussion_r3319603224 --- connectors-sdk/connectors_sdk/settings/base_settings.py | 5 ----- 1 file changed, 5 deletions(-) diff --git a/connectors-sdk/connectors_sdk/settings/base_settings.py b/connectors-sdk/connectors_sdk/settings/base_settings.py index 0e2bb65fea8..65672242d88 100644 --- a/connectors-sdk/connectors_sdk/settings/base_settings.py +++ b/connectors-sdk/connectors_sdk/settings/base_settings.py @@ -422,11 +422,6 @@ def _load_config_dict( # Get config/env vars as dict to send for validation config_dict: dict[str, Any] = settings_loader().model_dump() - cls.logger.debug( - "Raw configuration data loaded for validation", - {"config_data": config_dict}, - ) - return handler(config_dict) def to_helper_config(self) -> dict[str, Any]: From e5631b5fe18b13feb036461d4e86e111ec3f7b37 Mon Sep 17 00:00:00 2001 From: Powlinett Date: Mon, 6 Jul 2026 10:34:27 +0200 Subject: [PATCH 10/13] fix!: use `SecretStr` type for `OPENCTI_TOKEN` env var (avoid leaking sensitive data in logs) comment https://github.com/OpenCTI-Platform/connectors/pull/6559#discussion_r3319603205 --- .../connectors_sdk/settings/base_settings.py | 21 ++++++++++++++++++- .../tests/test_settings/test_base_settings.py | 10 ++++++--- 2 files changed, 27 insertions(+), 4 deletions(-) diff --git a/connectors-sdk/connectors_sdk/settings/base_settings.py b/connectors-sdk/connectors_sdk/settings/base_settings.py index 65672242d88..bc333055a22 100644 --- a/connectors-sdk/connectors_sdk/settings/base_settings.py +++ b/connectors-sdk/connectors_sdk/settings/base_settings.py @@ -29,9 +29,13 @@ BaseModel, ConfigDict, Field, + FieldSerializationInfo, HttpUrl, ModelWrapValidatorHandler, + SecretStr, + SerializerFunctionWrapHandler, ValidationError, + field_serializer, model_validator, ) from pydantic.fields import FieldInfo @@ -87,10 +91,25 @@ class _OpenCTIConfig(BaseConfigModel): url: HttpUrl = Field( description="The base URL of the OpenCTI instance.", ) - token: str = Field( + token: SecretStr = Field( description="The API token to connect to OpenCTI.", ) + @field_serializer("token", mode="wrap", when_used="json") + def _serialize_token( + self, + value: Any, + handler: SerializerFunctionWrapHandler, + info: FieldSerializationInfo, + ) -> str: + """Get token secret value when serializing for `pycti.OpenCTIConnectorHelper` only. + Otherwise, return the redacted value, i.e. `"********"`. + """ + mode = info.context.get("mode") if info.context else None + if isinstance(value, SecretStr) and mode == "pycti": + return value.get_secret_value() + return handler(value) # type: ignore[no-any-return] # actually return `str` + class _BaseConnectorConfig(BaseConfigModel, ABC): """Base class for connector configuration. diff --git a/connectors-sdk/tests/test_settings/test_base_settings.py b/connectors-sdk/tests/test_settings/test_base_settings.py index 96fb71cafb4..6bd9f073d20 100644 --- a/connectors-sdk/tests/test_settings/test_base_settings.py +++ b/connectors-sdk/tests/test_settings/test_base_settings.py @@ -8,7 +8,7 @@ ) from connectors_sdk.settings.deprecations import Deprecate, DeprecatedField from connectors_sdk.settings.exceptions import ConfigValidationError -from pydantic import Field, HttpUrl +from pydantic import Field, HttpUrl, SecretStr def test_base_config_model_should_retrieve_deprecated_fields(): @@ -120,7 +120,7 @@ def test_base_connector_settings_should_validate_settings_from_dot_env_file( # Then: Values are validated and cast to expected runtime types assert settings.opencti.url == HttpUrl("http://localhost:8080/") - assert settings.opencti.token == "changeme" + assert settings.opencti.token == SecretStr("changeme") assert settings.connector.id == "connector-poc--uid" assert settings.connector.name == "Test Connector" assert settings.connector.scope == ["test"] @@ -141,7 +141,7 @@ def test_base_connector_settings_should_validate_settings_from_os_environ( # Then: Values are validated and cast to expected runtime types assert settings.opencti.url == HttpUrl("http://localhost:8080/") - assert settings.opencti.token == "changeme" + assert settings.opencti.token == SecretStr("changeme") assert settings.connector.id == "connector-poc--uid" assert settings.connector.name == "Test Connector" assert settings.connector.scope == ["test"] @@ -166,8 +166,12 @@ def test_base_connector_settings_should_provide_helper_config(mock_environment): # Given: A valid BaseConnectorSettings instance built from patched environment # When: OpenCTIConnectorHelper config dict is generated settings = BaseConnectorSettings() + json_dump = settings.model_dump(mode="json") opencti_dict = settings.to_helper_config() + # Then: The regular JSON dump of settings does not expose the secret token value + assert json_dump["opencti"]["token"] == "**********" + # Then: The resulting helper config dict matches expected structure and values assert opencti_dict == { "connector": { From d358dac896904e9e2a15aba8401c8dfb43214d71 Mon Sep 17 00:00:00 2001 From: Powlinett Date: Mon, 6 Jul 2026 10:36:27 +0200 Subject: [PATCH 11/13] refactor: use `field_serializer` for `CONNECTOR_SCOPE` for consistency --- .../settings/annotated_types.py | 49 ------------------- .../connectors_sdk/settings/base_settings.py | 17 ++++++- connectors-sdk/tests/data/.env.test | 2 +- connectors-sdk/tests/data/config.test.yml | 2 +- .../tests/test_settings/conftest.py | 2 +- .../test_settings/test_annotated_types.py | 40 --------------- .../tests/test_settings/test_base_settings.py | 36 +++++++++++--- .../test_deprecation_migration.py | 14 +++--- .../test_settings/test_settings_loader.py | 32 ++---------- 9 files changed, 60 insertions(+), 134 deletions(-) diff --git a/connectors-sdk/connectors_sdk/settings/annotated_types.py b/connectors-sdk/connectors_sdk/settings/annotated_types.py index 880a53f9eaf..be0474ef338 100644 --- a/connectors-sdk/connectors_sdk/settings/annotated_types.py +++ b/connectors-sdk/connectors_sdk/settings/annotated_types.py @@ -6,7 +6,6 @@ from pydantic import ( BeforeValidator, PlainSerializer, - SerializationInfo, TypeAdapter, ) @@ -35,58 +34,17 @@ def parse_comma_separated_list(value: str | list[str]) -> list[str]: return value -def serialize_list_of_strings( - value: list[str], info: SerializationInfo -) -> str | list[str]: - """Serialize a list[str] as a comma-separated string when the Pydantic - serialization context requests "pycti" mode; otherwise, return the list - unchanged. - - This serializer is intended for use with Pydantic v2 `PlainSerializer` and - is typically activated only during JSON serialization (`when_used="json"`), - so the in-memory Python value remains a `list[str]` while the JSON output - can be a single string when required by external systems. - - Parameters - - value: The value to serialize. Expected to be a list of strings. - - info: Serialization context provided by Pydantic. If `info.context` - contains `{"mode": "pycti"}`, the list will be joined into a single - comma-separated string. - - Returns: - - A comma-separated string if context mode is "pycti" and `value` is a list. - - The original value `value` unchanged in all other cases. - - Notes: - - Joining does not insert spaces; e.g., ["a", "b", "c"] -> "a,b,c". - - If any element contains commas, those commas are not escaped. - - Examples: - - info.context={"mode": "pycti"} and value=["e1", "e2"] -> "e1,e2" - - info.context is None or mode != "pycti" -> ["e1", "e2"] - """ - if info.context and info.context.get("mode") == "pycti": - return ",".join(value) # [ "e1", "e2", "e3" ] -> "e1,e2,e3" - return value - - ListFromString = Annotated[ list[str], # Final type BeforeValidator(parse_comma_separated_list), - PlainSerializer(serialize_list_of_strings, when_used="json"), """Annotated list[str] that: - Validates: Accepts a comma-separated string (e.g., "a,b,c") or a list[str]. If a string is provided, it is split on commas and whitespace is trimmed for each item. -- Serializes (JSON): When the Pydantic serialization context includes - {"mode": "pycti"}, the list is serialized as a single comma-separated string - (e.g., ["a","b"] -> "a,b"). Otherwise, it serializes as a JSON array by default. Components - BeforeValidator(parse_comma_separated_list): Converts input strings to list[str] early in validation. -- PlainSerializer(serialize_list_of_strings, when_used="json"): Produces the "pycti" - string form only for JSON serialization. Examples - Validation: @@ -97,12 +55,6 @@ class Model(BaseModel): Model.model_validate({"tags": "a, b , c"}).tags # -> ["a", "b", "c"] Model.model_validate({"tags": ["x", "y"]}).tags # -> ["x", "y"] - -- Serialization: - m = Model.model_validate({"tags": ["e1", "e2"]}) - m.model_dump() # -> {'tags': ['e1', 'e2']} - m.model_dump_json() # -> {"tags":["e1","e2"]} - m.model_dump_json(context={"mode": "pycti"}) # -> {"tags":"e1,e2"} """, ] @@ -172,6 +124,5 @@ class Model(BaseModel): m = Model.model_validate({"start_date": datetime(2023, 10, 01, 0, 0, tzinfo=timezone.utc)}) m.model_dump() # -> {'start_date': datetime(2023, 10, 01, 0, 0, tzinfo=timezone.utc)} m.model_dump_json() # -> {"start_date": "2023-10-01T00:00:00+00:00"} - m.model_dump_json(context={"mode": "pycti"}) # -> {"start_date": "2023-10-01T00:00:00+00:00"} """, ] diff --git a/connectors-sdk/connectors_sdk/settings/base_settings.py b/connectors-sdk/connectors_sdk/settings/base_settings.py index bc333055a22..fcfb90bc7a3 100644 --- a/connectors-sdk/connectors_sdk/settings/base_settings.py +++ b/connectors-sdk/connectors_sdk/settings/base_settings.py @@ -129,13 +129,28 @@ class _BaseConnectorConfig(BaseConfigModel, ABC): description="The name of the connector.", ) scope: ListFromString = Field( - description="The scope of the connector, e.g. 'flashpoint'." + description="The scope of the connector, e.g. 'indicator, vulnerability'." ) log_level: Literal["debug", "info", "warn", "warning", "error"] = Field( description="The minimum level of logs to display.", default="error", ) + @field_serializer("scope", mode="wrap", when_used="json") + def _serialize_scope( + self, + value: Any, + handler: SerializerFunctionWrapHandler, + info: FieldSerializationInfo, + ) -> str | list[str]: + """Serialize scope as a comma-separated string when serializing for `pycti.OpenCTIConnectorHelper` only. + Otherwise, return the list of strings. + """ + mode = info.context.get("mode") if info.context else None + if isinstance(value, list) and mode == "pycti": + return ",".join(value) # [ "e1", "e2", "e3" ] -> "e1,e2,e3" + return handler(value) # type: ignore[no-any-return] # actually return `list[str]` + class BaseExternalImportConnectorConfig(_BaseConnectorConfig): """Settings class for external import connectors. diff --git a/connectors-sdk/tests/data/.env.test b/connectors-sdk/tests/data/.env.test index f4b59c4df05..8c426344946 100644 --- a/connectors-sdk/tests/data/.env.test +++ b/connectors-sdk/tests/data/.env.test @@ -2,6 +2,6 @@ OPENCTI_URL=http://localhost:8080 OPENCTI_TOKEN=changeme CONNECTOR_ID=connector-poc--uid CONNECTOR_NAME=Test Connector -CONNECTOR_SCOPE=test +CONNECTOR_SCOPE=scope1,scope2 CONNECTOR_LOG_LEVEL=debug CONNECTOR_DURATION_PERIOD=PT5M \ No newline at end of file diff --git a/connectors-sdk/tests/data/config.test.yml b/connectors-sdk/tests/data/config.test.yml index 89ad916a168..bea9f9d5014 100644 --- a/connectors-sdk/tests/data/config.test.yml +++ b/connectors-sdk/tests/data/config.test.yml @@ -5,7 +5,7 @@ opencti: connector: id: connector-poc--uid name: Test Connector - scope: test + scope: scope1,scope2 log_level: debug duration_period: PT5M \ No newline at end of file diff --git a/connectors-sdk/tests/test_settings/conftest.py b/connectors-sdk/tests/test_settings/conftest.py index 59fbbe84cc6..b5373118ab1 100644 --- a/connectors-sdk/tests/test_settings/conftest.py +++ b/connectors-sdk/tests/test_settings/conftest.py @@ -11,7 +11,7 @@ def mock_environment(monkeypatch): monkeypatch.setenv("OPENCTI_TOKEN", "changeme") monkeypatch.setenv("CONNECTOR_ID", "connector-poc--uid") monkeypatch.setenv("CONNECTOR_NAME", "Test Connector") - monkeypatch.setenv("CONNECTOR_SCOPE", "test") + monkeypatch.setenv("CONNECTOR_SCOPE", "scope1,scope2") monkeypatch.setenv("CONNECTOR_DURATION_PERIOD", "PT5M") monkeypatch.setenv("CONNECTOR_LOG_LEVEL", "debug") diff --git a/connectors-sdk/tests/test_settings/test_annotated_types.py b/connectors-sdk/tests/test_settings/test_annotated_types.py index bc611dca536..620205271d2 100644 --- a/connectors-sdk/tests/test_settings/test_annotated_types.py +++ b/connectors-sdk/tests/test_settings/test_annotated_types.py @@ -1,5 +1,4 @@ from datetime import datetime, timedelta, timezone -from types import SimpleNamespace import freezegun import pytest @@ -8,7 +7,6 @@ ListFromString, parse_comma_separated_list, parse_iso_string, - serialize_list_of_strings, ) from pydantic import TypeAdapter @@ -45,20 +43,6 @@ def test_parse_comma_separated_list_passthrough() -> None: assert parse_comma_separated_list(["a", "b"]) == ["a", "b"] -def test_serialize_list_of_strings_handles_pycti_mode() -> None: - info = SimpleNamespace(context={"mode": "pycti"}) - assert serialize_list_of_strings(["a", "b"], info) == "a,b" - - -@pytest.mark.parametrize("context", [None, {}, {"mode": "other"}]) -def test_serialize_list_of_strings_handles_non_pycti_modes( - context: dict[str, str] | None, -) -> None: - info = SimpleNamespace(context=context) - value = ["a", "b"] - assert serialize_list_of_strings(value, info) == value - - def test_list_from_string_accepts_string_input() -> None: value = TypeAdapter(ListFromString).validate_python("a,b,c") assert value == ["a", "b", "c"] @@ -74,30 +58,6 @@ def test_list_from_string_dumps_valid_json() -> None: assert value == ["a", "b"] -@pytest.mark.parametrize( - "input,expected", - [ - pytest.param( - ["a", "b", "c"], - "a,b,c", - id="list_of_strings", - ), - pytest.param( - [], - "", - id="empty_list", - ), # empty list -> empty string - ], -) -def test_list_from_string_dumps_valid_json_in_pycti_mode( - input: list[str], expected: str -) -> None: - value = TypeAdapter(ListFromString).dump_python( - input, mode="json", context={"mode": "pycti"} - ) - assert value == expected - - # DatetimeFromIsoString diff --git a/connectors-sdk/tests/test_settings/test_base_settings.py b/connectors-sdk/tests/test_settings/test_base_settings.py index 6bd9f073d20..4ab1fc6ab6f 100644 --- a/connectors-sdk/tests/test_settings/test_base_settings.py +++ b/connectors-sdk/tests/test_settings/test_base_settings.py @@ -123,7 +123,7 @@ def test_base_connector_settings_should_validate_settings_from_dot_env_file( assert settings.opencti.token == SecretStr("changeme") assert settings.connector.id == "connector-poc--uid" assert settings.connector.name == "Test Connector" - assert settings.connector.scope == ["test"] + assert settings.connector.scope == ["scope1", "scope2"] assert settings.connector.log_level == "debug" @@ -144,7 +144,28 @@ def test_base_connector_settings_should_validate_settings_from_os_environ( assert settings.opencti.token == SecretStr("changeme") assert settings.connector.id == "connector-poc--uid" assert settings.connector.name == "Test Connector" - assert settings.connector.scope == ["test"] + assert settings.connector.scope == ["scope1", "scope2"] + assert settings.connector.log_level == "debug" + + +def test_base_connector_settings_should_validate_settings_from_config_yaml_file( + mock_config_yml_file_presence, +): + """ + Test that `BaseConnectorSettings` casts and validates config vars in `config.yml`. + For testing purpose, the path of `config.yml` file is `tests/test_settings/data/config.test.yml`. + """ + + # Given: Valid connector settings are provided through config.yml fixture + # When: BaseConnectorSettings is instantiated + settings = BaseConnectorSettings() + + # Then: Values are validated and cast to expected runtime types + assert settings.opencti.url == HttpUrl("http://localhost:8080/") + assert settings.opencti.token == SecretStr("changeme") + assert settings.connector.id == "connector-poc--uid" + assert settings.connector.name == "Test Connector" + assert settings.connector.scope == ["scope1", "scope2"] assert settings.connector.log_level == "debug" @@ -171,19 +192,20 @@ def test_base_connector_settings_should_provide_helper_config(mock_environment): # Then: The regular JSON dump of settings does not expose the secret token value assert json_dump["opencti"]["token"] == "**********" + assert json_dump["connector"]["scope"] == ["scope1", "scope2"] # Then: The resulting helper config dict matches expected structure and values assert opencti_dict == { + "opencti": { + "token": "changeme", # clear token + "url": "http://localhost:8080/", + }, "connector": { "duration_period": "PT5M", "id": "connector-poc--uid", "log_level": "debug", "name": "Test Connector", - "scope": "test", - }, - "opencti": { - "token": "changeme", - "url": "http://localhost:8080/", + "scope": "scope1,scope2", # comma-separated string }, } diff --git a/connectors-sdk/tests/test_settings/test_deprecation_migration.py b/connectors-sdk/tests/test_settings/test_deprecation_migration.py index 1a28a51149e..a6188ba49b3 100644 --- a/connectors-sdk/tests/test_settings/test_deprecation_migration.py +++ b/connectors-sdk/tests/test_settings/test_deprecation_migration.py @@ -36,7 +36,7 @@ class TestSettings(BaseConnectorSettings): monkeypatch.setenv("OPENCTI_TOKEN", "test-token") monkeypatch.setenv("CONNECTOR_ID", "test-id") monkeypatch.setenv("CONNECTOR_NAME", "Test") - monkeypatch.setenv("CONNECTOR_SCOPE", "test") + monkeypatch.setenv("CONNECTOR_SCOPE", "scope1,scope2") monkeypatch.setenv("CONNECTOR_DURATION_PERIOD", "PT5M") monkeypatch.setenv("CONNECTOR_OLD_FIELD", "old_value") @@ -73,7 +73,7 @@ class TestSettings(BaseConnectorSettings): monkeypatch.setenv("OPENCTI_TOKEN", "test-token") monkeypatch.setenv("CONNECTOR_ID", "test-id") monkeypatch.setenv("CONNECTOR_NAME", "Test") - monkeypatch.setenv("CONNECTOR_SCOPE", "test") + monkeypatch.setenv("CONNECTOR_SCOPE", "scope1,scope2") monkeypatch.setenv("CONNECTOR_DURATION_PERIOD", "PT5M") # When: Settings initialization evaluates migration metadata @@ -108,7 +108,7 @@ class TestSettings(BaseConnectorSettings): monkeypatch.setenv("OPENCTI_TOKEN", "test-token") monkeypatch.setenv("CONNECTOR_ID", "test-id") monkeypatch.setenv("CONNECTOR_NAME", "Test") - monkeypatch.setenv("CONNECTOR_SCOPE", "test") + monkeypatch.setenv("CONNECTOR_SCOPE", "scope1,scope2") monkeypatch.setenv("CONNECTOR_DURATION_PERIOD", "PT5M") monkeypatch.setenv("CONNECTOR_OLD_VALUE", "5") @@ -139,7 +139,7 @@ class TestSettings(BaseConnectorSettings): monkeypatch.setenv("OPENCTI_TOKEN", "test-token") monkeypatch.setenv("CONNECTOR_ID", "test-id") monkeypatch.setenv("CONNECTOR_NAME", "Test") - monkeypatch.setenv("CONNECTOR_SCOPE", "test") + monkeypatch.setenv("CONNECTOR_SCOPE", "scope1,scope2") monkeypatch.setenv("CONNECTOR_DURATION_PERIOD", "PT5M") # When: TestSettings initialization evaluates namespace migration metadata @@ -171,7 +171,7 @@ class TestSettings(BaseConnectorSettings): monkeypatch.setenv("OPENCTI_TOKEN", "test-token") monkeypatch.setenv("OLD_CONNECTOR_ID", "test-id") monkeypatch.setenv("OLD_CONNECTOR_NAME", "Test") - monkeypatch.setenv("OLD_CONNECTOR_SCOPE", "test") + monkeypatch.setenv("OLD_CONNECTOR_SCOPE", "scope1,scope2") monkeypatch.setenv("OLD_CONNECTOR_DURATION_PERIOD", "PT5M") # When: TestSettings initialization validates deprecated namespace migration @@ -203,7 +203,7 @@ class TestSettings(BaseConnectorSettings): monkeypatch.setenv("OPENCTI_TOKEN", "test-token") monkeypatch.setenv("CONNECTOR_ID", "test-id") monkeypatch.setenv("CONNECTOR_NAME", "Test") - monkeypatch.setenv("CONNECTOR_SCOPE", "test") + monkeypatch.setenv("CONNECTOR_SCOPE", "scope1,scope2") monkeypatch.setenv("CONNECTOR_DURATION_PERIOD", "PT5M") monkeypatch.setenv("CONNECTOR_SPECIAL_FIELD", "special_value") @@ -237,7 +237,7 @@ class TestSettings(BaseConnectorSettings): monkeypatch.setenv("OPENCTI_TOKEN", "test-token") monkeypatch.setenv("CONNECTOR_ID", "test-id") monkeypatch.setenv("CONNECTOR_NAME", "Test") - monkeypatch.setenv("CONNECTOR_SCOPE", "test") + monkeypatch.setenv("CONNECTOR_SCOPE", "scope1,scope2") monkeypatch.setenv("CONNECTOR_DURATION_PERIOD", "PT5M") monkeypatch.setenv("CONNECTOR_OLD_FIELD", "migrated_value") diff --git a/connectors-sdk/tests/test_settings/test_settings_loader.py b/connectors-sdk/tests/test_settings/test_settings_loader.py index 7d13b433165..c2a41c0d060 100644 --- a/connectors-sdk/tests/test_settings/test_settings_loader.py +++ b/connectors-sdk/tests/test_settings/test_settings_loader.py @@ -7,7 +7,6 @@ BaseConnectorSettings, _SettingsLoader, ) -from pydantic import HttpUrl def test_settings_loader_should_get_connector_main_path(mock_main_path): @@ -118,7 +117,7 @@ def test_settings_loader_should_parse_config_yml_file(mock_config_yml_file_prese "name": "Test Connector", "duration_period": "PT5M", "log_level": "debug", - "scope": "test", + "scope": "scope1,scope2", }, } @@ -142,7 +141,7 @@ def test_settings_loader_should_parse_dot_env_file(mock_dot_env_file_presence): "connector_name": "Test Connector", "connector_duration_period": "PT5M", "connector_log_level": "debug", - "connector_scope": "test", + "connector_scope": "scope1,scope2", } @@ -180,7 +179,7 @@ def test_settings_loader_should_parse_config_yml_from_model( assert settings_dict["opencti"]["token"] == "changeme" assert settings_dict["connector"]["id"] == "connector-poc--uid" assert settings_dict["connector"]["name"] == "Test Connector" - assert settings_dict["connector"]["scope"] == "test" + assert settings_dict["connector"]["scope"] == "scope1,scope2" assert settings_dict["connector"]["log_level"] == "debug" @@ -201,7 +200,7 @@ def test_settings_loader_should_parse_dot_env_from_model(mock_dot_env_file_prese assert settings_dict["opencti"]["token"] == "changeme" assert settings_dict["connector"]["id"] == "connector-poc--uid" assert settings_dict["connector"]["name"] == "Test Connector" - assert settings_dict["connector"]["scope"] == "test" + assert settings_dict["connector"]["scope"] == "scope1,scope2" assert settings_dict["connector"]["log_level"] == "debug" @@ -222,26 +221,5 @@ def test_settings_loader_should_parse_os_environ_from_model(mock_environment): assert settings_dict["opencti"]["token"] == "changeme" assert settings_dict["connector"]["id"] == "connector-poc--uid" assert settings_dict["connector"]["name"] == "Test Connector" - assert settings_dict["connector"]["scope"] == "test" + assert settings_dict["connector"]["scope"] == "scope1,scope2" assert settings_dict["connector"]["log_level"] == "debug" - - -def test_base_connector_settings_should_validate_settings_from_config_yaml_file( - mock_config_yml_file_presence, -): - """ - Test that `BaseConnectorSettings` casts and validates config vars in `config.yml`. - For testing purpose, the path of `config.yml` file is `tests/test_settings/data/config.test.yml`. - """ - - # Given: Valid connector settings are provided through config.yml fixture - # When: BaseConnectorSettings is instantiated - settings = BaseConnectorSettings() - - # Then: Values are validated and cast to expected runtime types - assert settings.opencti.url == HttpUrl("http://localhost:8080/") - assert settings.opencti.token == "changeme" - assert settings.connector.id == "connector-poc--uid" - assert settings.connector.name == "Test Connector" - assert settings.connector.scope == ["test"] - assert settings.connector.log_level == "debug" From f5341f7dd4da97179a060d07cf291a29a3d105bc Mon Sep 17 00:00:00 2001 From: Powlinett Date: Mon, 6 Jul 2026 20:18:59 +0200 Subject: [PATCH 12/13] fix: solve serialization issue by making deprecated fields optional allow `model_dump()` method to work with `exclude_none` arg --- .../settings/_settings_loader.py | 21 ++- .../connectors_sdk/settings/base_settings.py | 132 ++++++++++++------ .../tests/test_settings/test_base_settings.py | 14 +- .../test_settings/test_settings_loader.py | 43 ++++++ 4 files changed, 164 insertions(+), 46 deletions(-) diff --git a/connectors-sdk/connectors_sdk/settings/_settings_loader.py b/connectors-sdk/connectors_sdk/settings/_settings_loader.py index b23d2cfc6aa..f69c3c23614 100644 --- a/connectors-sdk/connectors_sdk/settings/_settings_loader.py +++ b/connectors-sdk/connectors_sdk/settings/_settings_loader.py @@ -3,7 +3,8 @@ import sys from copy import deepcopy from pathlib import Path -from typing import TYPE_CHECKING, Any, ClassVar +from types import UnionType +from typing import TYPE_CHECKING, Any, ClassVar, Union, get_args, get_origin from connectors_sdk.logging.sdk_logger import sdk_logger from pydantic import BaseModel, create_model @@ -149,7 +150,23 @@ class SettingsLoader(_SettingsLoader): ... model_fields = deepcopy(connector_settings.model_fields) for field_info in model_fields.values(): annotation = field_info.annotation - if annotation and issubclass(annotation, BaseModel): + + # Unwrap `BaseModel | None` / `Optional[BaseModel]` annotations + annotation_origin = get_origin(annotation) + if annotation_origin in (Union, UnionType): + base_model_annotation = next( + ( + arg + for arg in get_args(annotation) + if isinstance(arg, type) and issubclass(arg, BaseModel) + ), + None, + ) + if base_model_annotation: + annotation = base_model_annotation + + # Keep only `BaseModel` model fields names (accept any value) + if isinstance(annotation, type) and issubclass(annotation, BaseModel): fields: dict[str, Any] = dict.fromkeys( annotation.model_fields.keys(), Any ) diff --git a/connectors-sdk/connectors_sdk/settings/base_settings.py b/connectors-sdk/connectors_sdk/settings/base_settings.py index fcfb90bc7a3..b150ef96dc3 100644 --- a/connectors-sdk/connectors_sdk/settings/base_settings.py +++ b/connectors-sdk/connectors_sdk/settings/base_settings.py @@ -10,7 +10,17 @@ from abc import ABC from datetime import timedelta -from typing import TYPE_CHECKING, Any, ClassVar, Literal, Self +from types import UnionType +from typing import ( + TYPE_CHECKING, + Any, + ClassVar, + Literal, + Self, + Union, + get_args, + get_origin, +) from connectors_sdk.logging.logger import Logger from connectors_sdk.logging.sdk_logger import sdk_logger @@ -49,7 +59,11 @@ class BaseConfigModel(BaseModel, ABC): To prevent attributes from being modified after initialization. """ - model_config = ConfigDict(extra="allow", frozen=True, validate_default=True) + model_config = ConfigDict( + extra="allow", + frozen=True, + validate_default=True, + ) _model_deprecated_fields: ClassVar[dict[str, FieldInfo]] = {} @@ -63,13 +77,17 @@ def __pydantic_init_subclass__(cls, **kwargs: Any) -> None: for name, field in cls.model_fields.items(): for meta in field.metadata: if isinstance(meta, Deprecate): - # Change validation behavior - if not field.deprecated: - field.deprecated = True + # Make the field optional (accept `None`) + if isinstance(field.annotation, type): + field.annotation = field.annotation | None # type: ignore[assignment] field.default = None field.default_factory = None field.validate_default = False + # Mark as deprecated (in case of missing/empty deprecation message) + if not field.deprecated: + field.deprecated = True + # Add deprecation info to JSON schema if not field.json_schema_extra: field.json_schema_extra = {} @@ -129,7 +147,7 @@ class _BaseConnectorConfig(BaseConfigModel, ABC): description="The name of the connector.", ) scope: ListFromString = Field( - description="The scope of the connector, e.g. 'indicator, vulnerability'." + description="The scope of the connector, e.g. 'indicator, vulnerability'.", ) log_level: Literal["debug", "info", "warn", "warning", "error"] = Field( description="The minimum level of logs to display.", @@ -318,6 +336,34 @@ def make_schema_generator( mode=mode, ) + @classmethod + def _extract_base_config_model_type(cls, annotation: Any) -> Any: + """Extract `BaseConfigModel` type from a field's annotation. + + Args: + annotation: The field's annotation to extract from. + + Returns: + The extracted `BaseConfigModel` type if present, otherwise `None`. + """ + # Handle `field_name: BaseConfigModel` annotations + if isinstance(annotation, type) and issubclass(annotation, BaseConfigModel): + return annotation + + # Handle `field_name: BaseConfigModel | None` / `Optional[BaseConfigModel]` annotations + annotation_origin = get_origin(annotation) + if annotation_origin in (Union, UnionType): + base_config_model_type = next( + ( + arg + for arg in get_args(annotation) + if isinstance(arg, type) and issubclass(arg, BaseConfigModel) + ), + None, + ) + if base_config_model_type: + return base_config_model_type + @classmethod def _migrate_deprecated_namespaces(cls, data: dict[str, Any]) -> dict[str, Any]: """Migrate deprecated namespaces in the configuration data. @@ -329,9 +375,8 @@ def _migrate_deprecated_namespaces(cls, data: dict[str, Any]) -> dict[str, Any]: Migrated configuration data. """ for field_name, field in cls._model_deprecated_fields.items(): - annotation = field.annotation - is_namespace = isinstance(annotation, type) and issubclass( - annotation, BaseConfigModel + is_namespace = ( + cls._extract_base_config_model_type(field.annotation) is not None ) deprecate_metadata = next( m for m in field.metadata if isinstance(m, Deprecate) @@ -372,39 +417,40 @@ def _migrate_deprecated_variables(cls, data: dict[str, Any]) -> dict[str, Any]: Migrated configuration data. """ for field_name, field in cls.model_fields.items(): - annotation = field.annotation - is_namespace = isinstance(annotation, type) and issubclass( - annotation, BaseConfigModel + base_config_model_type = cls._extract_base_config_model_type( + field.annotation ) - if is_namespace: - for ( - sub_field_name, - sub_field, - ) in annotation._model_deprecated_fields.items(): # type: ignore[union-attr] - deprecate_metadata = next( - m for m in sub_field.metadata if isinstance(m, Deprecate) - ) - new_namespace = deprecate_metadata.new_namespace - new_namespaced_var = deprecate_metadata.new_namespaced_var - new_value_factory = deprecate_metadata.new_value_factory - removal_date = deprecate_metadata.removal_date - - if new_namespaced_var: - if not isinstance(new_namespaced_var, str): - raise ValueError( - f"`new_namespaced_var` for field {sub_field_name} must be a string." - ) - - migrate_deprecated_variable( - data, - old_name=sub_field_name, - new_name=new_namespaced_var, - current_namespace=field_name, - new_namespace=new_namespace, - new_value_factory=new_value_factory, - removal_date=removal_date, + if not base_config_model_type: + continue # not a namespace, skip + + for ( + sub_field_name, + sub_field, + ) in base_config_model_type._model_deprecated_fields.items(): + deprecate_metadata = next( + m for m in sub_field.metadata if isinstance(m, Deprecate) + ) + new_namespace = deprecate_metadata.new_namespace + new_namespaced_var = deprecate_metadata.new_namespaced_var + new_value_factory = deprecate_metadata.new_value_factory + removal_date = deprecate_metadata.removal_date + + if new_namespaced_var: + if not isinstance(new_namespaced_var, str): + raise ValueError( + f"`new_namespaced_var` for field {sub_field_name} must be a string." ) + migrate_deprecated_variable( + data, + old_name=sub_field_name, + new_name=new_namespaced_var, + current_namespace=field_name, + new_namespace=new_namespace, + new_value_factory=new_value_factory, + removal_date=removal_date, + ) + return data @model_validator(mode="wrap") @@ -463,8 +509,8 @@ def to_helper_config(self) -> dict[str, Any]: return self.model_dump( mode="json", context={"mode": "pycti"}, - # Deprecated fields can be set to `None` despite their type (due to `Deprecate` annotation). - # To avoid `PydanticSerializationError`, we exclude all fields set to `None` during serialization. - # OpenCTIConnectorHelper handles missing fields with default values or internal logic. - exclude_none=True, + # # Deprecated fields can be set to `None` despite their type (due to `Deprecate` annotation). + # # To avoid `PydanticSerializationError`, we exclude all fields set to `None` during serialization. + # # OpenCTIConnectorHelper handles missing fields with default values or internal logic. + # exclude_none=True, ) diff --git a/connectors-sdk/tests/test_settings/test_base_settings.py b/connectors-sdk/tests/test_settings/test_base_settings.py index 4ab1fc6ab6f..8116aca4f49 100644 --- a/connectors-sdk/tests/test_settings/test_base_settings.py +++ b/connectors-sdk/tests/test_settings/test_base_settings.py @@ -45,12 +45,24 @@ class TestConfig(BaseConfigModel): assert "old_field" in TestConfig._model_deprecated_fields +def test_base_config_model_should_make_deprecated_fields_optional(): + """Test that `BaseConfigModel` subclasses set `default` to `None` for deprecated fields.""" + + # Given: A deprecated field explicitly defined as required (non-optional) + class TestConfig(BaseConfigModel): + old_field: str = DeprecatedField() # type should be overwritten to `str | None` + + # When: The model field definitions are built + # Then: Deprecated field annotation is normalized to `str | None` to make it optional + assert TestConfig.model_fields["old_field"].annotation == str | None + assert TestConfig._model_deprecated_fields["old_field"].annotation == str | None + + def test_base_config_model_should_set_default_to_none_for_deprecated_fields(): """Test that `BaseConfigModel` subclasses set `default` to `None` for deprecated fields.""" # Given: A deprecated field explicitly defines a non-None default class TestConfig(BaseConfigModel): - test_field: str = Field(default="test") old_field: str = DeprecatedField( default="deprecated default" # should be overwritten to None ) diff --git a/connectors-sdk/tests/test_settings/test_settings_loader.py b/connectors-sdk/tests/test_settings/test_settings_loader.py index c2a41c0d060..1c1b70cc864 100644 --- a/connectors-sdk/tests/test_settings/test_settings_loader.py +++ b/connectors-sdk/tests/test_settings/test_settings_loader.py @@ -4,9 +4,12 @@ import pytest from connectors_sdk.settings.base_settings import ( + BaseConfigModel, BaseConnectorSettings, _SettingsLoader, ) +from connectors_sdk.settings.deprecations import DeprecatedField +from pydantic import Field def test_settings_loader_should_get_connector_main_path(mock_main_path): @@ -223,3 +226,43 @@ def test_settings_loader_should_parse_os_environ_from_model(mock_environment): assert settings_dict["connector"]["name"] == "Test Connector" assert settings_dict["connector"]["scope"] == "scope1,scope2" assert settings_dict["connector"]["log_level"] == "debug" + + +def test_settings_loader_should_parse_os_environ_from_model_with_deprecated_fields( + monkeypatch, mock_environment +): + """ + Test that `_SettingsLoader.build_loader_from_model` returns a `BaseSettings` subclass + capable of parsing `os.environ` according to the given `BaseModel` with deprecated fields. + For testing purpose, `os.environ` is patched. + """ + + monkeypatch.setenv("DEPRECATED_NAMESPACE_TEST_FIELD", "deprecated_value") + + # Given: Connector settings model with deprecated fields + class DeprecatedConfig(BaseConfigModel): + test_field: str = Field( + description="This is a test field.", + ) + + class DeprecatedConnectorSettings(BaseConnectorSettings): + """A connector settings model with deprecated namespaces.""" + + deprecated_namespace: DeprecatedConfig = DeprecatedField( + deprecated="This namespace is deprecated.", + ) + + # When: The settings loader instance parses and dumps values + settings_loader = _SettingsLoader.build_loader_from_model( + DeprecatedConnectorSettings + ) + settings_dict = settings_loader().model_dump() + + # Then: Parsed nested settings expose expected OpenCTI and connector values + assert settings_dict["opencti"]["url"] == "http://localhost:8080" + assert settings_dict["opencti"]["token"] == "changeme" + assert settings_dict["connector"]["id"] == "connector-poc--uid" + assert settings_dict["connector"]["name"] == "Test Connector" + assert settings_dict["connector"]["scope"] == "scope1,scope2" + assert settings_dict["connector"]["log_level"] == "debug" + assert settings_dict["deprecated_namespace"]["test_field"] == "deprecated_value" From aacc0e323ee13115dc8a8a22b3da553df76dc4d9 Mon Sep 17 00:00:00 2001 From: Powlinett Date: Mon, 6 Jul 2026 12:07:03 +0200 Subject: [PATCH 13/13] ci: update connectors tests after `OPENCTI_TOKEN` type change --- .../tests/test_connector/test_settings.py | 3 ++- .../ctm360-cyna-feed/tests/test_connector/test_settings.py | 3 ++- .../tests/test_connector/test_settings.py | 3 ++- external-import/flare/tests/test_connector/test_settings.py | 2 +- .../censys-enrichment/tests/censys_enrichment/test_config.py | 2 +- internal-enrichment/polyswarm-sandbox/tests/test_connector.py | 2 +- .../tests/test_connector/test_settings.py | 3 ++- .../tests/microsoft_sentinel_intel/test_config.py | 4 ++-- 8 files changed, 13 insertions(+), 9 deletions(-) diff --git a/external-import/ctm360-cyberblindspot-feed/tests/test_connector/test_settings.py b/external-import/ctm360-cyberblindspot-feed/tests/test_connector/test_settings.py index 5518d96c50f..5cdbae4acd9 100644 --- a/external-import/ctm360-cyberblindspot-feed/tests/test_connector/test_settings.py +++ b/external-import/ctm360-cyberblindspot-feed/tests/test_connector/test_settings.py @@ -24,7 +24,8 @@ def test_opencti_token(self): """OpenCTI token should match the environment variable.""" settings = ConnectorSettings() assert ( - settings.opencti.token == "test-token-00000000-0000-0000-0000-000000000000" + settings.opencti.token.get_secret_value() + == "test-token-00000000-0000-0000-0000-000000000000" ) def test_connector_id(self): diff --git a/external-import/ctm360-cyna-feed/tests/test_connector/test_settings.py b/external-import/ctm360-cyna-feed/tests/test_connector/test_settings.py index 14dc46d0006..42a706e91ce 100644 --- a/external-import/ctm360-cyna-feed/tests/test_connector/test_settings.py +++ b/external-import/ctm360-cyna-feed/tests/test_connector/test_settings.py @@ -24,7 +24,8 @@ def test_opencti_token(self): """OpenCTI token should match the environment variable.""" settings = ConnectorSettings() assert ( - settings.opencti.token == "test-token-00000000-0000-0000-0000-000000000000" + settings.opencti.token.get_secret_value() + == "test-token-00000000-0000-0000-0000-000000000000" ) def test_connector_id(self): diff --git a/external-import/ctm360-hackerview-feed/tests/test_connector/test_settings.py b/external-import/ctm360-hackerview-feed/tests/test_connector/test_settings.py index 32f51957923..16ccd5f1285 100644 --- a/external-import/ctm360-hackerview-feed/tests/test_connector/test_settings.py +++ b/external-import/ctm360-hackerview-feed/tests/test_connector/test_settings.py @@ -28,7 +28,8 @@ def test_opencti_token(self): """OpenCTI token should match the environment variable.""" settings = ConnectorSettings() assert ( - settings.opencti.token == "test-token-00000000-0000-0000-0000-000000000000" + settings.opencti.token.get_secret_value() + == "test-token-00000000-0000-0000-0000-000000000000" ) def test_connector_id(self): diff --git a/external-import/flare/tests/test_connector/test_settings.py b/external-import/flare/tests/test_connector/test_settings.py index 11367222d52..14a7ea89cb6 100644 --- a/external-import/flare/tests/test_connector/test_settings.py +++ b/external-import/flare/tests/test_connector/test_settings.py @@ -102,7 +102,7 @@ def test_to_helper_config_structure(self, required_env: None) -> None: settings = ConnectorSettings() config = settings.to_helper_config() assert config["opencti"]["url"] == str(settings.opencti.url) - assert config["opencti"]["token"] == settings.opencti.token + assert config["opencti"]["token"] == settings.opencti.token.get_secret_value() assert config["connector"]["id"] == settings.connector.id assert config["connector"]["type"] == "EXTERNAL_IMPORT" assert config["connector"]["name"] == settings.connector.name diff --git a/internal-enrichment/censys-enrichment/tests/censys_enrichment/test_config.py b/internal-enrichment/censys-enrichment/tests/censys_enrichment/test_config.py index 46a1b1e3126..40d87f264c1 100644 --- a/internal-enrichment/censys-enrichment/tests/censys_enrichment/test_config.py +++ b/internal-enrichment/censys-enrichment/tests/censys_enrichment/test_config.py @@ -10,7 +10,7 @@ def test_config() -> None: # Test config from env assert config.opencti.url == HttpUrl("http://test") - assert config.opencti.token == "opencti-token" + assert config.opencti.token.get_secret_value() == "opencti-token" assert ( config.censys_enrichment.organisation_id.get_secret_value() diff --git a/internal-enrichment/polyswarm-sandbox/tests/test_connector.py b/internal-enrichment/polyswarm-sandbox/tests/test_connector.py index 3f681718afe..379bea3991d 100644 --- a/internal-enrichment/polyswarm-sandbox/tests/test_connector.py +++ b/internal-enrichment/polyswarm-sandbox/tests/test_connector.py @@ -131,7 +131,7 @@ def test_config_loads_with_env_vars(self, monkeypatch): config = ConnectorSettings() assert str(config.opencti.url).rstrip("/") == "http://localhost:8080" - assert config.opencti.token == "test-token" + assert config.opencti.token.get_secret_value() == "test-token" assert config.polyswarm.api_key.get_secret_value() == "test-api-key" def test_config_defaults(self, monkeypatch): diff --git a/internal-enrichment/qualys-cve-enrichment/tests/test_connector/test_settings.py b/internal-enrichment/qualys-cve-enrichment/tests/test_connector/test_settings.py index a59c09f4967..4bc06d093fe 100644 --- a/internal-enrichment/qualys-cve-enrichment/tests/test_connector/test_settings.py +++ b/internal-enrichment/qualys-cve-enrichment/tests/test_connector/test_settings.py @@ -28,7 +28,8 @@ def test_opencti_token(self): """OpenCTI token should match the environment variable.""" settings = ConnectorSettings() assert ( - settings.opencti.token == "test-token-00000000-0000-0000-0000-000000000000" + settings.opencti.token.get_secret_value() + == "test-token-00000000-0000-0000-0000-000000000000" ) def test_connector_id(self): diff --git a/stream/microsoft-sentinel-intel/tests/microsoft_sentinel_intel/test_config.py b/stream/microsoft-sentinel-intel/tests/microsoft_sentinel_intel/test_config.py index 9a159fcb73a..4df45aac150 100644 --- a/stream/microsoft-sentinel-intel/tests/microsoft_sentinel_intel/test_config.py +++ b/stream/microsoft-sentinel-intel/tests/microsoft_sentinel_intel/test_config.py @@ -1,6 +1,6 @@ import pytest from microsoft_sentinel_intel.settings import ConnectorSettings -from pydantic import HttpUrl +from pydantic import HttpUrl, SecretStr @pytest.mark.usefixtures("mock_microsoft_sentinel_intel_config") @@ -8,7 +8,7 @@ def test_config() -> None: config = ConnectorSettings().model_dump() assert config["opencti"]["url"] == HttpUrl("http://test-opencti-url/") - assert config["opencti"]["token"] == "test-opencti-token" + assert config["opencti"]["token"] == SecretStr("test-opencti-token") assert config["connector"]["id"] == "test-connector-id" assert config["connector"]["name"] == "External Import Connector Template"