From ca9a47427849f04a7f19ae12d6224871717f48d4 Mon Sep 17 00:00:00 2001 From: Gustavo Date: Thu, 25 Jun 2026 15:56:02 -0300 Subject: [PATCH 01/36] add timeout for queries --- ros_typedb/ros_typedb/ros_typedb_interface.py | 16 +- ros_typedb/ros_typedb/typedb_interface.py | 235 +++++++--- ros_typedb/test/test_ros_typedb_interface.py | 78 ++++ ros_typedb/test/test_typedb_interface.py | 418 +++++++++++++++++- ros_typedb_msgs/srv/Query.srv | 1 + 5 files changed, 685 insertions(+), 63 deletions(-) diff --git a/ros_typedb/ros_typedb/ros_typedb_interface.py b/ros_typedb/ros_typedb/ros_typedb_interface.py index 74388a1..78f7219 100644 --- a/ros_typedb/ros_typedb/ros_typedb_interface.py +++ b/ros_typedb/ros_typedb/ros_typedb_interface.py @@ -320,6 +320,7 @@ def __init__(self, node_name: str, **kwargs): self.declare_parameter('force_data', True) self.declare_parameter('infer', True) self.declare_parameter('driver_timeout_s', 10.0) + self.declare_parameter('query_timeout_s', 0.0) self.default_schema_path = '' self.declare_parameter('schema_path', ['']) @@ -341,7 +342,8 @@ def init_typedb_interface( force_data: Optional[bool] = False, infer: Optional[bool] = False, sort_fetch_results: Optional[bool] = False, - driver_timeout_s: Optional[float] = 10.0) -> None: + driver_timeout_s: Optional[float] = 10.0, + query_timeout_s: Optional[float] = None) -> None: """ Initialize self.typedb_interface. @@ -356,6 +358,8 @@ def init_typedb_interface( recursively sorted. :param driver_timeout_s: seconds to wait for driver connection before timing out. + :param query_timeout_s: default per-query timeout in seconds. + None means no limit. """ self.typedb_interface = self.typedb_interface_class( address, @@ -366,7 +370,8 @@ def init_typedb_interface( force_data, infer, sort_fetch_results, - driver_timeout_s + driver_timeout_s, + query_timeout_s=query_timeout_s ) self.typedb_interface.insert_data_event = self.insert_data_event @@ -418,7 +423,9 @@ def on_configure(self, state: State) -> TransitionCallbackReturn: infer=self.get_parameter('infer').value, sort_fetch_results=( self.get_parameter('sort_fetch_results').value), - driver_timeout_s=self.get_parameter('driver_timeout_s').value + driver_timeout_s=self.get_parameter('driver_timeout_s').value, + query_timeout_s=( + self.get_parameter('query_timeout_s').value or None) ) self.event_pub = self.create_lifecycle_publisher( @@ -510,7 +517,8 @@ def query_service_cb( response.error_message = f'Unknown query type: {req.query_type}' return response - query_result = query_func(req.query) + per_call_timeout = req.timeout_s if req.timeout_s > 0 else None + query_result = query_func(req.query, timeout=per_call_timeout) response = query_result_to_ros_msg(req.query_type, query_result) if query_result is None: response.success = False diff --git a/ros_typedb/ros_typedb/typedb_interface.py b/ros_typedb/ros_typedb/typedb_interface.py index 272b530..c944d9e 100644 --- a/ros_typedb/ros_typedb/typedb_interface.py +++ b/ros_typedb/ros_typedb/typedb_interface.py @@ -175,7 +175,8 @@ def __init__( force_data: Optional[bool] = False, infer: Optional[bool] = False, sort_fetch_results: Optional[bool] = False, - driver_timeout_s: Optional[float] = 10.0) -> None: + driver_timeout_s: Optional[float] = 10.0, + query_timeout_s: Optional[float] = None) -> None: """ Connect to a typeDB server and interacts with it. @@ -195,12 +196,17 @@ def __init__( :param driver_timeout_s: seconds to wait for driver connection before timing out. Set to None or a non-positive value to wait without a timeout. + :param query_timeout_s: default per-query timeout in seconds. + Set to None or a non-positive value to wait without a timeout. """ self.logger = logging.getLogger() self._database_query_lock = Lock() self.last_error: str = '' self._address = address self._driver_timeout_s = driver_timeout_s + self._query_timeout_s = ( + query_timeout_s if (query_timeout_s and query_timeout_s > 0) + else None) self._infer = infer self._sort_fetch_results = bool(sort_fetch_results) self.connect_driver(address, timeout_s=driver_timeout_s) @@ -303,38 +309,6 @@ def close_late_driver(): else: raise result - def delete_database(self, database_name: str = None) -> None: - """ - Delete database. - - :param database_name: database name. - """ - if database_name is None: - database_name = self.database_name - if self.driver.databases.contains(database_name): - self.driver.databases.get(database_name).delete() - - def create_database( - self, database_name: str, force: Optional[bool] = False) -> None: - """ - Create database. - - :param database_name: database name. - :param force: if database should override an existing database - """ - if force: - self.delete_database(database_name) - - self.database_name = database_name - if self.driver.databases.contains(database_name): - self.logger.warning( - 'The database with the name %s already exists. ' - 'Ignoring create_database request.', - database_name) - return - - self.driver.databases.create(database_name) - def is_server_alive(self) -> bool: """Return True if the current driver can reach the TypeDB server.""" try: @@ -374,6 +348,38 @@ def ensure_server_alive(self) -> None: else: self.ensure_database_exists() + def delete_database(self, database_name: str = None) -> None: + """ + Delete database. + + :param database_name: database name. + """ + if database_name is None: + database_name = self.database_name + if self.driver.databases.contains(database_name): + self.driver.databases.get(database_name).delete() + + def create_database( + self, database_name: str, force: Optional[bool] = False) -> None: + """ + Create database. + + :param database_name: database name. + :param force: if database should override an existing database + """ + if force: + self.delete_database(database_name) + + self.database_name = database_name + if self.driver.databases.contains(database_name): + self.logger.warning( + 'The database with the name %s already exists. ' + 'Ignoring create_database request.', + database_name) + return + + self.driver.databases.create(database_name) + def create_session( self, database_name: str, @@ -391,34 +397,40 @@ def create_session( # Read/write database # Generic query method - def database_query( + def _database_query_impl( self, session_type: SessionType, transaction_type: TransactionType, query_type: Literal[ - 'define', 'insert', 'delete', 'fetch', 'get', 'get_aggregate', 'update'], + 'define', 'insert', 'delete', 'fetch', 'get', + 'get_aggregate', 'update'], query: str, - options: Optional[TypeDBOptions] = TypeDBOptions() + options: TypeDBOptions, + effective_timeout: Optional[float] = None ) -> Literal[True] | Iterator[ConceptMap] | \ list[dict[str, MatchResultDict]] | None | int | float: """ - Query database. + Query database (internal implementation). - Helper method to query the database, it handles creating a session and - managaging a transaction. It can perform queries of the type define, - insert, fetch, get_aggregate, or delete. + Acquires _database_query_lock and performs the full session/transaction + lifecycle; called by database_query() which adds the deadline wrapper. :param session_type: TypeDB session type. :param transaction_type: TypeDB transaction type. :param query_type: TypeDB query type. :param query: Query to be performed. - :param options: TypeDB options. + :param options: TypeDB options (modified in-place). + :param effective_timeout: resolved timeout in seconds; sets + transaction_timeout_millis when positive. :return: Query result, type depends on the query_type. """ with self._database_query_lock: self.ensure_server_alive() with self.create_session( self.database_name, session_type) as session: + if effective_timeout and effective_timeout > 0: + options.transaction_timeout_millis = int( + effective_timeout * 1000) options.infer = self._infer options.parallel = True with session.transaction( @@ -445,6 +457,85 @@ def database_query( return None return list(query_answer) + def database_query( + self, + session_type: SessionType, + transaction_type: TransactionType, + query_type: Literal[ + 'define', 'insert', 'delete', 'fetch', 'get', + 'get_aggregate', 'update'], + query: str, + options: Optional[TypeDBOptions] = None, + timeout: Optional[float] = None + ) -> Literal[True] | Iterator[ConceptMap] | \ + list[dict[str, MatchResultDict]] | None | int | float: + """ + Query database. + + Wraps _database_query_impl with an optional client-side deadline. + When the deadline fires, the stale driver is closed in a background + thread (interrupting the blocked gRPC call) so _database_query_lock + is released for subsequent callers. + + :param session_type: TypeDB session type. + :param transaction_type: TypeDB transaction type. + :param query_type: TypeDB query type. + :param query: Query to be performed. + :param options: TypeDB options; defaults to a fresh TypeDBOptions(). + :param timeout: per-call timeout in seconds. Falls back to + self._query_timeout_s. If both are None or non-positive, no + client-side deadline is applied. + :return: Query result, type depends on the query_type. + """ + if options is None: + options = TypeDBOptions() + + effective_timeout = ( + timeout if timeout is not None else self._query_timeout_s) + if not effective_timeout or effective_timeout <= 0: + return self._database_query_impl( + session_type, transaction_type, query_type, query, + options, effective_timeout) + + result_queue = queue.Queue(maxsize=1) + + def run_query(): + try: + result = self._database_query_impl( + session_type, transaction_type, query_type, query, + options, effective_timeout) + result_queue.put((True, result)) + except BaseException as exc: # noqa: B902 + result_queue.put((False, exc)) + + thread = threading.Thread(target=run_query, daemon=True) + thread.start() + thread.join(timeout=effective_timeout) + + if thread.is_alive(): + stale_driver = self.driver + + def close_stale(): + try: + stale_driver.close() + except Exception: # noqa: B902 + pass + thread.join() + try: + result_queue.get_nowait() + except queue.Empty: + pass + + threading.Thread(target=close_stale, daemon=True).start() + raise TimeoutError( + f'Query timed out after {effective_timeout:g}s' + ) + + succeeded, result = result_queue.get_nowait() + if succeeded: + return result + raise result + def write_database_file( self, session_type: SessionType, @@ -528,36 +619,44 @@ def delete_data_event_wrapper(*args, **kwargs): # Events end # @insert_data_event_ - def insert_database(self, query: str) -> Iterator[ConceptMap] | None: + def insert_database( + self, query: str, + timeout: Optional[float] = None) -> Iterator[ConceptMap] | None: """ Perform insert query. :param query: Query to be performed. + :param timeout: per-call timeout in seconds; overrides default. :return: Query result, if query fails return None. """ self.last_error = '' result = None try: result = self.database_query( - SessionType.DATA, TransactionType.WRITE, 'insert', query) + SessionType.DATA, TransactionType.WRITE, 'insert', query, + timeout=timeout) except Exception as err: self.logger.warning( 'Error with insert query! Exception retrieved: %s', err) self.last_error = str(err) return result - def update_database(self, query: str) -> Iterator[ConceptMap] | None: + def update_database( + self, query: str, + timeout: Optional[float] = None) -> Iterator[ConceptMap] | None: """ Perform update query. :param query: Query to be performed. + :param timeout: per-call timeout in seconds; overrides default. :return: Query result, if query fails return None. """ self.last_error = '' result = None try: result = self.database_query( - SessionType.DATA, TransactionType.WRITE, 'update', query) + SessionType.DATA, TransactionType.WRITE, 'update', query, + timeout=timeout) except Exception as err: self.logger.warning( 'Error with update query! Exception retrieved: %s', err) @@ -565,31 +664,43 @@ def update_database(self, query: str) -> Iterator[ConceptMap] | None: return result # @delete_data_event_ - def delete_from_database(self, query: str) -> Literal[True] | None: + def delete_from_database( + self, query: str, + timeout: Optional[float] = None) -> Literal[True] | None: """ Perform delete query. :param query: Query to be performed. + :param timeout: per-call timeout in seconds; overrides default. :return: Query result, if query fails return None. """ self.last_error = '' result = None try: result = self.database_query( - SessionType.DATA, TransactionType.WRITE, 'delete', query) + SessionType.DATA, TransactionType.WRITE, 'delete', query, + timeout=timeout) except Exception as err: self.logger.warning( 'Error with delete query! Exception retrieved: %s', err) self.last_error = str(err) return result - def define_database(self, query: str) -> Literal[True] | None: - """Perform define query.""" + def define_database( + self, query: str, + timeout: Optional[float] = None) -> Literal[True] | None: + """ + Perform define query. + + :param query: Query to be performed. + :param timeout: per-call timeout in seconds; overrides default. + """ self.last_error = '' result = None try: result = self.database_query( - SessionType.SCHEMA, TransactionType.WRITE, 'define', query) + SessionType.SCHEMA, TransactionType.WRITE, 'define', query, + timeout=timeout) except Exception as err: self.logger.warning('Error with define query! Exception retrieved: %s', err) self.last_error = str(err) @@ -598,13 +709,16 @@ def define_database(self, query: str) -> Literal[True] | None: def fetch_database( self, query: str, - sort_result: Optional[bool] = None) -> list[dict[str, MatchResultDict]] | None: + sort_result: Optional[bool] = None, + timeout: Optional[float] = None + ) -> list[dict[str, MatchResultDict]] | None: """ Perform match query. :param query: Query to be performed. :param sort_result: if query result should be recursively sorted. If None, uses the interface default policy. + :param timeout: per-call timeout in seconds; overrides default. :return: Query result, if query fails return None. """ self.last_error = '' @@ -617,7 +731,8 @@ def fetch_database( TransactionType.READ, 'fetch', query, - options) + options, + timeout=timeout) should_sort = self._sort_fetch_results if sort_result is None else sort_result if should_sort: result = recursively_sort_dict(result) @@ -646,11 +761,14 @@ def fetch_database_ordered( """ return self.fetch_database(query, sort_result=True) - def get_database(self, query: str) -> int | float | None: + def get_database( + self, query: str, + timeout: Optional[float] = None) -> int | float | None: """ Perform get query. :param query: Query to be performed. + :param timeout: per-call timeout in seconds; overrides default. :return: Query result. """ self.last_error = '' @@ -663,18 +781,22 @@ def get_database(self, query: str) -> int | float | None: TransactionType.READ, 'get', query, - options) + options, + timeout=timeout) except Exception as err: self.logger.warning( 'Error with get query! Exception retrieved: %s', err) self.last_error = str(err) return result - def get_aggregate_database(self, query: str) -> int | float | None: + def get_aggregate_database( + self, query: str, + timeout: Optional[float] = None) -> int | float | None: """ Perform get aggregate query. :param query: Query to be performed. + :param timeout: per-call timeout in seconds; overrides default. :return: Query result. """ self.last_error = '' @@ -687,7 +809,8 @@ def get_aggregate_database(self, query: str) -> int | float | None: TransactionType.READ, 'get_aggregate', query, - options) + options, + timeout=timeout) except Exception as err: self.logger.warning( 'Error with get_aggregate query! Exception retrieved: %s', err) diff --git a/ros_typedb/test/test_ros_typedb_interface.py b/ros_typedb/test/test_ros_typedb_interface.py index 9980c10..4814b1b 100644 --- a/ros_typedb/test/test_ros_typedb_interface.py +++ b/ros_typedb/test/test_ros_typedb_interface.py @@ -75,6 +75,84 @@ def insert_query(): return query_req +def test_query_srv_has_timeout_s_field(): + """Query.Request must have a timeout_s field defaulting to 0.0.""" + req = Query.Request() + assert hasattr(req, 'timeout_s') + assert req.timeout_s == 0.0 + + +def test_init_typedb_interface_accepts_query_timeout_s(): + """init_typedb_interface passes query_timeout_s to TypeDBInterface constructor.""" + captured = {} + + class FakeTypeDBInterface: + + def __init__(self, *args, **kwargs): + captured.update(kwargs) + self.insert_data_event = None + self.delete_data_event = None + + node = ROSTypeDBInterface.__new__(ROSTypeDBInterface) + node.typedb_interface_class = FakeTypeDBInterface + node.init_typedb_interface( + address='localhost:1729', + database_name='test', + query_timeout_s=15.0) + + assert captured.get('query_timeout_s') == 15.0 + + +def test_query_service_cb_passes_timeout_to_wrapper(): + """query_service_cb extracts req.timeout_s and passes it as timeout= to the wrapper.""" + captured = {} + + class FakeTypeDBInterface: + last_error = '' + + def fetch_database(self, query, timeout=None): + captured['timeout'] = timeout + return [] + + node = ROSTypeDBInterface.__new__(ROSTypeDBInterface) + node.typedb_interface = FakeTypeDBInterface() + + req = Query.Request() + req.query_type = Query.Request.FETCH + req.query = 'match $x isa thing; fetch $x;' + req.timeout_s = 5.0 + + response = Query.Response() + node.query_service_cb(req, response) + + assert captured.get('timeout') == 5.0 + + +def test_query_service_cb_passes_none_when_timeout_s_is_zero(): + """query_service_cb converts timeout_s=0.0 to timeout=None (use node default).""" + captured = {} + + class FakeTypeDBInterface: + last_error = '' + + def fetch_database(self, query, timeout=None): + captured['timeout'] = timeout + return [] + + node = ROSTypeDBInterface.__new__(ROSTypeDBInterface) + node.typedb_interface = FakeTypeDBInterface() + + req = Query.Request() + req.query_type = Query.Request.FETCH + req.query = 'match $x isa thing; fetch $x;' + req.timeout_s = 0.0 + + response = Query.Response() + node.query_service_cb(req, response) + + assert captured.get('timeout') is None + + @launch_pytest.fixture def generate_test_description(): path_to_test = Path(__file__).parents[1] diff --git a/ros_typedb/test/test_typedb_interface.py b/ros_typedb/test/test_typedb_interface.py index ee43253..15d082e 100644 --- a/ros_typedb/test/test_typedb_interface.py +++ b/ros_typedb/test/test_typedb_interface.py @@ -13,6 +13,9 @@ # limitations under the License. from datetime import datetime +import logging +import threading +from threading import Lock import time import pytest @@ -21,6 +24,353 @@ from ros_typedb.typedb_interface import string_to_string_array from ros_typedb.typedb_interface import TypeDBInterface +from typedb.driver import SessionType +from typedb.driver import TransactionType + + +# --------------------------------------------------------------------------- +# Helpers shared by timeout tests +# --------------------------------------------------------------------------- + +def _make_hung_tdb(query_timeout_s): + """ + Fake TypeDBInterface (no __init__) with an unresponsive server. + + FakeDriver.close() unblocks the hung session so the daemon thread can exit. + _query_timeout_s is set so the FIXED code can apply its timeout; the + current (unfixed) database_query ignores the attribute and hangs forever. + """ + blocked = threading.Event() + + class HungTransaction: + + def __enter__(self): return self + def __exit__(self, *args): return False + + class query: + + @staticmethod + def fetch(q): + blocked.wait(timeout=10) + return [] + + def commit(self): pass + + class HungSession: + + def __enter__(self): return self + def __exit__(self, *args): return False + def transaction(self, *args, **kwargs): return HungTransaction() + + class FakeDriver: + + class databases: + + @staticmethod + def contains(name): return True + + def session(self, *args, **kwargs): return HungSession() + def close(self): blocked.set() + + tdb = TypeDBInterface.__new__(TypeDBInterface) + tdb._database_query_lock = Lock() + tdb._infer = False + tdb._sort_fetch_results = False + tdb.database_name = 'test_database' + tdb.last_error = '' + tdb._address = 'localhost:1729' + tdb._driver_timeout_s = 10.0 + tdb.logger = logging.getLogger() + tdb.driver = FakeDriver() + tdb._query_timeout_s = query_timeout_s + return tdb + + +def _call_in_thread(fn, wall_clock_s=1.5): + """ + Run fn() in a daemon thread, returning (thread, exc_box). + + Joins for wall_clock_s. Caller checks thread.is_alive() to detect a hang. + """ + exc_box = [None] + + def run(): + try: + fn() + except Exception as e: # noqa: B902 + exc_box[0] = e + + t = threading.Thread(target=run, daemon=True) + t.start() + t.join(timeout=wall_clock_s) + return t, exc_box + + +def test_database_query_raises_timeout_error_on_stalled_server(): + """ + database_query raises TimeoutError when the server stops responding. + + When _query_timeout_s is set and the TypeDB server stalls mid-query, + database_query must raise TimeoutError within the deadline instead of + blocking the caller indefinitely. + """ + tdb = _make_hung_tdb(query_timeout_s=0.05) + + thread, exc_box = _call_in_thread( + lambda: tdb.database_query( + SessionType.DATA, TransactionType.READ, 'fetch', + 'match $x isa thing; fetch $x;'), + wall_clock_s=1.5) + + if thread.is_alive(): + pytest.fail( + 'database_query blocked for 1.5 s on a stalled server — ' + 'timeout mechanism is not working; _query_timeout_s was 0.05 s.') + + assert isinstance(exc_box[0], TimeoutError), ( + f'Expected TimeoutError, got {exc_box[0]!r}') + + +def test_database_query_releases_lock_after_timeout_on_stalled_server(): + """ + Lock is released after timeout so subsequent callers are not blocked. + + When a query times out because the server stopped responding, closing the + stale driver must unblock the worker thread so it can exit the lock + context. If the lock is never released, all subsequent database_query + calls pile up indefinitely. + """ + tdb = _make_hung_tdb(query_timeout_s=0.05) + + thread, _ = _call_in_thread( + lambda: tdb.database_query( + SessionType.DATA, TransactionType.READ, 'fetch', + 'match $x isa thing; fetch $x;'), + wall_clock_s=1.5) + + if thread.is_alive(): + pytest.fail( + 'database_query blocked for 1.5 s on a stalled server — ' + 'cannot verify lock release because the call never returned.') + + acquired = tdb._database_query_lock.acquire(timeout=2.0) + assert acquired, ( + '_database_query_lock not released after TimeoutError; ' + 'subsequent callers would block indefinitely.') + tdb._database_query_lock.release() + + +def test_fetch_database_returns_none_on_stalled_server(): + """ + fetch_database returns None and sets last_error when the server stalls. + + fetch_database wraps database_query and must catch TimeoutError, populate + last_error, and return None — it must not propagate the exception or block. + """ + tdb = _make_hung_tdb(query_timeout_s=0.05) + + result_box = [None] + thread, exc_box = _call_in_thread( + lambda: result_box.__setitem__(0, tdb.fetch_database( + 'match $x isa thing; fetch $x;')), + wall_clock_s=1.5) + + if thread.is_alive(): + pytest.fail( + 'fetch_database blocked for 1.5 s on a stalled server — ' + 'did not return within the query timeout.') + + assert result_box[0] is None, ( + f'fetch_database should return None on timeout, got {result_box[0]!r}') + assert exc_box[0] is None, ( + f'fetch_database should not propagate exceptions, got {exc_box[0]!r}') + assert tdb.last_error, 'last_error should be populated after a timeout' + + +def _make_tdb_with_slow_driver(timeout_s): + """Fake TypeDBInterface whose driver blocks until closed.""" + blocked = threading.Event() + + class SlowTransaction: + + def __enter__(self): return self + def __exit__(self, *args): return False + + class query: + + @staticmethod + def fetch(q): + blocked.wait(timeout=5) + return [] + + def commit(self): pass + + class SlowSession: + + def __enter__(self): return self + def __exit__(self, *args): return False + def transaction(self, *args, **kwargs): return SlowTransaction() + + class FakeDriver: + + class databases: + + @staticmethod + def contains(name): return True + + def session(self, *args, **kwargs): return SlowSession() + def close(self): blocked.set() + + tdb = TypeDBInterface.__new__(TypeDBInterface) + tdb._query_timeout_s = timeout_s + tdb._database_query_lock = Lock() + tdb._infer = False + tdb._sort_fetch_results = False + tdb.database_name = 'test_database' + tdb.last_error = '' + tdb._address = 'localhost:1729' + tdb._driver_timeout_s = 10.0 + tdb.logger = logging.getLogger() + tdb.driver = FakeDriver() + return tdb + + +def test_database_query_raises_timeout_error(): + """database_query raises TimeoutError when query hangs past timeout.""" + tdb = _make_tdb_with_slow_driver(timeout_s=0.05) + + start = time.monotonic() + with pytest.raises(TimeoutError): + tdb.database_query( + SessionType.DATA, TransactionType.READ, 'fetch', + 'match $x isa thing; fetch $x;') + assert time.monotonic() - start < 1.0 + + +def test_database_query_lock_released_after_timeout(): + """After a timeout, lock is released so the next call can proceed.""" + tdb = _make_tdb_with_slow_driver(timeout_s=0.05) + + with pytest.raises(TimeoutError): + tdb.database_query( + SessionType.DATA, TransactionType.READ, 'fetch', + 'match $x isa thing; fetch $x;') + + acquired = tdb._database_query_lock.acquire(timeout=2.0) + assert acquired, 'lock was not released within 2 seconds after timeout' + tdb._database_query_lock.release() + + +def test_database_query_no_timeout_when_query_timeout_s_is_none(): + """Query runs without deadline when no timeout is configured.""" + calls = [] + + class FastTransaction: + + def __enter__(self): return self + def __exit__(self, *args): return False + + class query: + + @staticmethod + def fetch(q): + calls.append(q) + return [] + + def commit(self): pass + + class FastSession: + + def __enter__(self): return self + def __exit__(self, *args): return False + def transaction(self, *args, **kwargs): return FastTransaction() + + class FakeDriver: + + class databases: + + @staticmethod + def contains(name): return True + + def session(self, *args, **kwargs): return FastSession() + + tdb = TypeDBInterface.__new__(TypeDBInterface) + tdb._query_timeout_s = None + tdb._database_query_lock = Lock() + tdb._infer = False + tdb.database_name = 'test_database' + tdb.last_error = '' + tdb.logger = logging.getLogger() + tdb.driver = FakeDriver() + + result = tdb.database_query( + SessionType.DATA, TransactionType.READ, 'fetch', + 'match $x isa thing; fetch $x;') + assert result == [] + assert calls == ['match $x isa thing; fetch $x;'] + + +def test_database_query_per_call_timeout_overrides_default(): + """A per-call timeout takes precedence over _query_timeout_s=None.""" + tdb = _make_tdb_with_slow_driver(timeout_s=None) + + with pytest.raises(TimeoutError): + tdb.database_query( + SessionType.DATA, TransactionType.READ, 'fetch', + 'match $x isa thing; fetch $x;', + timeout=0.05) + + +def test_database_query_sets_transaction_timeout_millis(): + """transaction_timeout_millis is set to effective_timeout * 1000.""" + captured_options = [] + + class CapturingTransaction: + + def __enter__(self): return self + def __exit__(self, *args): return False + + class query: + + @staticmethod + def fetch(q): return [] + + def commit(self): pass + + class CapturingSession: + + def __enter__(self): return self + def __exit__(self, *args): return False + + def transaction(self, txn_type, options): + captured_options.append(options) + return CapturingTransaction() + + class FakeDriver: + + class databases: + + @staticmethod + def contains(name): return True + + def session(self, *args, **kwargs): return CapturingSession() + + tdb = TypeDBInterface.__new__(TypeDBInterface) + tdb._query_timeout_s = None + tdb._database_query_lock = Lock() + tdb._infer = False + tdb.database_name = 'test_database' + tdb.last_error = '' + tdb.logger = logging.getLogger() + tdb.driver = FakeDriver() + + tdb.database_query( + SessionType.DATA, TransactionType.READ, 'fetch', + 'match $x isa thing; fetch $x;', + timeout=2.5) + + assert captured_options[0].transaction_timeout_millis == 2500 + def test_string_to_string_array_preserves_raw_path_with_comma(): path = '/tmp/schema,with-comma.tql' @@ -40,6 +390,66 @@ def test_string_to_string_array_keeps_unbracketed_strings_as_single_values(): ] +@pytest.mark.parametrize('method_name,extra_kwargs', [ + ('insert_database', {}), + ('update_database', {}), + ('delete_from_database', {}), + ('define_database', {}), + ('fetch_database', {}), + ('get_database', {}), + ('get_aggregate_database', {}), +]) +def test_raw_wrapper_passes_timeout_to_database_query( + monkeypatch, method_name, extra_kwargs): + """Each raw wrapper forwards timeout= to database_query.""" + tdb = TypeDBInterface.__new__(TypeDBInterface) + tdb.last_error = '' + tdb._sort_fetch_results = False + tdb._infer = False + tdb.logger = logging.getLogger() + + captured = {} + + def fake_database_query(*args, **kwargs): + captured['timeout'] = kwargs.get('timeout') + return ( + [] if method_name in ('fetch_database', 'get_database') + else True) + + monkeypatch.setattr(tdb, 'database_query', fake_database_query) + getattr(tdb, method_name)('match $x isa thing; fetch $x;', timeout=7.0) + + assert captured.get('timeout') == 7.0 + + +@pytest.mark.parametrize('method_name', [ + 'insert_database', + 'update_database', + 'delete_from_database', + 'define_database', + 'fetch_database', + 'get_database', + 'get_aggregate_database', +]) +def test_raw_wrapper_catches_timeout_error_and_sets_last_error( + monkeypatch, method_name): + """Each wrapper catches TimeoutError, sets last_error, and returns None.""" + tdb = TypeDBInterface.__new__(TypeDBInterface) + tdb.last_error = '' + tdb._sort_fetch_results = False + tdb._infer = False + tdb.logger = logging.getLogger() + + def raise_timeout(*args, **kwargs): + raise TimeoutError('Query timed out after 0.05s') + + monkeypatch.setattr(tdb, 'database_query', raise_timeout) + result = getattr(tdb, method_name)('match $x isa thing; fetch $x;') + + assert result is None + assert 'timed out' in tdb.last_error.lower() + + @pytest.fixture def typedb_interface(): typedb_interface = TypeDBInterface( @@ -156,7 +566,8 @@ def test_create_and_delete_database(): def test_define_query(typedb_interface): assert typedb_interface.define_database('define MyEntity sub entity;') - assert typedb_interface.define_database('define MyEntity aa entity') is None + assert typedb_interface.define_database( + 'define MyEntity aa entity') is None def test_insert_entity(typedb_interface): @@ -485,7 +896,7 @@ def test_insert_attributes(typedb_interface, match_dict, r_dict): ) ]) def test_delete_attributes( - typedb_interface, insert_dict, match_dict): + typedb_interface, insert_dict, match_dict): query = typedb_interface.dict_to_query(insert_dict) insert_result = typedb_interface.insert_database('insert ' + query) @@ -701,7 +1112,8 @@ def __exit__(self, exc_type, exc, traceback): result = typedb_interface.fetch_database('match $p isa person; fetch $p;') - assert result == [{'person': {'type': {'root': 'entity', 'label': 'person'}}}] + assert result == [ + {'person': {'type': {'root': 'entity', 'label': 'person'}}}] assert stale_driver.closed is True assert stale_driver.session_count == 0 assert typedb_interface.driver.session_count == 1 diff --git a/ros_typedb_msgs/srv/Query.srv b/ros_typedb_msgs/srv/Query.srv index e32279e..df40ff6 100644 --- a/ros_typedb_msgs/srv/Query.srv +++ b/ros_typedb_msgs/srv/Query.srv @@ -9,6 +9,7 @@ uint8 UPDATE=6 uint8 query_type string query +float64 timeout_s --- bool success string error_message From f38b506809e9565b0c1630249d881d0ce431bcd9 Mon Sep 17 00:00:00 2001 From: Gustavo Date: Thu, 25 Jun 2026 15:56:31 -0300 Subject: [PATCH 02/36] update Dockerfile --- Dockerfile | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/Dockerfile b/Dockerfile index 8799621..96493ae 100644 --- a/Dockerfile +++ b/Dockerfile @@ -63,8 +63,7 @@ RUN echo 'alias typedb-studio="/opt/typedb-studio/bin/TypeDB\ Studio"' >> ~/.bas ## Create, install deps, and build ros_typedb workspace RUN mkdir -p typedb_ws/src -WORKDIR $HOME/typedb_ws/src -RUN git clone https://github.com/kas-lab/ros_typedb.git +COPY . $HOME/typedb_ws/src/ros_typedb WORKDIR $HOME/typedb_ws/ RUN ["/bin/bash", "-c", "source /opt/ros/humble/setup.bash \ @@ -74,7 +73,7 @@ RUN ["/bin/bash", "-c", "source /opt/ros/humble/setup.bash \ && sudo rm -rf /var/lib/apt/lists/"] ## There is a bug with the default setuptools and packaging versions -RUN USER=ubuntu-user python3 -m pip install setuptools==75.8.2 packaging==24.1 empy==3.3.4 pandas==2.0.2 scipy==1.15.2 numpy==1.26.4 +RUN USER=ubuntu-user python3 -m pip install setuptools==75.8.2 packaging==24.1 empy==3.3.4 pandas==2.0.2 scipy==1.15.2 numpy==1.23.5 RUN ["/bin/bash", "-c", "source /opt/ros/humble/setup.bash \ && colcon build --symlink-install \ && echo 'source ~/typedb_ws/install/setup.bash' >> ~/.bashrc"] From 9c121374a20b4ce75d1e41100effbf9f35f7776e Mon Sep 17 00:00:00 2001 From: Gustavo Date: Thu, 25 Jun 2026 16:38:23 -0300 Subject: [PATCH 03/36] fix: missing database recreation --- ros_typedb/ros_typedb/typedb_interface.py | 212 +++++++++++++++++----- ros_typedb/test/test_typedb_interface.py | 78 +++++++- 2 files changed, 239 insertions(+), 51 deletions(-) diff --git a/ros_typedb/ros_typedb/typedb_interface.py b/ros_typedb/ros_typedb/typedb_interface.py index c944d9e..74ee61b 100644 --- a/ros_typedb/ros_typedb/typedb_interface.py +++ b/ros_typedb/ros_typedb/typedb_interface.py @@ -121,6 +121,20 @@ def string_to_string_array(string: str) -> list[str]: return [string] +def normalize_path_list(paths: Optional[list[str] | str]) -> list[str]: + """ + Normalize optional path input to a list of non-empty paths. + + :param paths: path, path list, or None. + :return: normalized list of paths. + """ + if isinstance(paths, str): + paths = string_to_string_array(paths) + if isinstance(paths, list): + return [path for path in paths if path != ''] + return [] + + def recursively_sort_dict(obj): """ Recursively sort dict keys (and nested dicts) in a dict or list. @@ -209,20 +223,16 @@ def __init__( else None) self._infer = infer self._sort_fetch_results = bool(sort_fetch_results) + self._schema_paths = normalize_path_list(schema_path) + self._data_paths = normalize_path_list(data_path) self.connect_driver(address, timeout_s=driver_timeout_s) self.create_database(database_name, force=force_database) - if isinstance(schema_path, str): - schema_path = string_to_string_array(schema_path) - if isinstance(schema_path, list): - for path in schema_path: - self.load_schema(path) + for path in self._schema_paths: + self.load_schema(path) if force_data: self.delete_all_data() - if isinstance(data_path, str): - data_path = string_to_string_array(data_path) - if isinstance(data_path, list): - for path in data_path: - self.load_data(path) + for path in self._data_paths: + self.load_data(path) def __del__(self): try: @@ -321,12 +331,34 @@ def is_server_alive(self) -> bool: return False def ensure_database_exists(self) -> None: - """Create the configured database if it is missing.""" - if not self.driver.databases.contains(self.database_name): - self.logger.warning( - 'Database %s was not found; creating it.', - self.database_name) - self.driver.databases.create(self.database_name) + """Recreate and initialize the configured database if it is missing.""" + if self.driver.databases.contains(self.database_name): + return + + if not self._schema_paths and not self._data_paths: + raise RuntimeError( + 'Database {} is missing and cannot be recreated because no ' + 'schema or data paths are configured.'.format( + self.database_name)) + + self.logger.warning( + 'Database %s was not found; recreating it from configured files.', + self.database_name) + self.driver.databases.create(self.database_name) + try: + for path in self._schema_paths: + self._load_schema_unlocked(path) + for path in self._data_paths: + self._load_data_unlocked(path) + except Exception: + try: + self.delete_database() + except Exception as exc: + self.logger.warning( + 'Failed to remove partially initialized database %s: %s', + self.database_name, + exc) + raise def reconnect_driver(self) -> None: """Reconnect the TypeDB driver after a lost server connection.""" @@ -395,6 +427,64 @@ def create_session( """ return self.driver.session(database_name, session_type, options) + def _database_query_unlocked( + self, + session_type: SessionType, + transaction_type: TransactionType, + query_type: Literal[ + 'define', 'insert', 'delete', 'fetch', 'get', + 'get_aggregate', 'update'], + query: str, + options: TypeDBOptions, + effective_timeout: Optional[float] = None + ) -> Literal[True] | Iterator[ConceptMap] | \ + list[dict[str, MatchResultDict]] | None | int | float: + """ + Query database without acquiring the database query lock. + + Callers must hold _database_query_lock and must have already ensured + the database exists. + + :param session_type: TypeDB session type. + :param transaction_type: TypeDB transaction type. + :param query_type: TypeDB query type. + :param query: Query to be performed. + :param options: TypeDB options (modified in-place). + :param effective_timeout: resolved timeout in seconds; sets + transaction_timeout_millis when positive. + :return: Query result, type depends on the query_type. + """ + with self.create_session( + self.database_name, session_type) as session: + if effective_timeout and effective_timeout > 0: + options.transaction_timeout_millis = int( + effective_timeout * 1000) + options.infer = self._infer + options.parallel = True + with session.transaction( + transaction_type, options) as transaction: + transaction_query_function = getattr( + transaction.query, query_type) + query_answer = transaction_query_function(query) + if transaction_type == TransactionType.WRITE: + if query_type == 'insert': + query_answer = list(query_answer) + transaction.commit() + + if query_type == 'delete' or query_type == 'define': + return True # delete and define always return None + + return query_answer + elif transaction_type == TransactionType.READ: + if query_type == 'get_aggregate': + answer = query_answer.resolve() + if answer.is_long(): + return answer.as_long() + if answer.is_float(): + return answer.as_float() + return None + return list(query_answer) + # Read/write database # Generic query method def _database_query_impl( @@ -412,8 +502,8 @@ def _database_query_impl( """ Query database (internal implementation). - Acquires _database_query_lock and performs the full session/transaction - lifecycle; called by database_query() which adds the deadline wrapper. + Acquires _database_query_lock, checks server/database health, and + performs the session/transaction lifecycle. :param session_type: TypeDB session type. :param transaction_type: TypeDB transaction type. @@ -426,36 +516,13 @@ def _database_query_impl( """ with self._database_query_lock: self.ensure_server_alive() - with self.create_session( - self.database_name, session_type) as session: - if effective_timeout and effective_timeout > 0: - options.transaction_timeout_millis = int( - effective_timeout * 1000) - options.infer = self._infer - options.parallel = True - with session.transaction( - transaction_type, options) as transaction: - transaction_query_function = getattr( - transaction.query, query_type) - query_answer = transaction_query_function(query) - if transaction_type == TransactionType.WRITE: - if query_type == 'insert': - query_answer = list(query_answer) - transaction.commit() - - if query_type == 'delete' or query_type == 'define': - return True # delete and define always return None - - return query_answer - elif transaction_type == TransactionType.READ: - if query_type == 'get_aggregate': - answer = query_answer.resolve() - if answer.is_long(): - return answer.as_long() - if answer.is_float(): - return answer.as_float() - return None - return list(query_answer) + return self._database_query_unlocked( + session_type, + transaction_type, + query_type, + query, + options, + effective_timeout) def database_query( self, @@ -554,6 +621,42 @@ def write_database_file( self.database_query( session_type, TransactionType.WRITE, query_type, query) + def _write_database_file_unlocked( + self, + session_type: SessionType, + query_type: Literal['define', 'insert'], + file_path: str) -> None: + """ + Write a .tql file while the caller already holds the query lock. + + :param session_type: session type, e.g., schema or data. + :param query_type: query type, e.g., 'define' or 'insert'. + :param file_path: .tql file path. + """ + with open(file_path, mode='r') as file: + query = file.read() + + self._database_query_unlocked( + session_type, + TransactionType.WRITE, + query_type, + query, + TypeDBOptions(), + self._query_timeout_s) + + def _load_schema_unlocked(self, schema_path: str) -> None: + """ + Load a schema file while the caller already holds the query lock. + + :param schema_path: .tql file path. + """ + if schema_path is not None and schema_path != '': + return self._write_database_file_unlocked( + SessionType.SCHEMA, + 'define', + schema_path + ) + def load_schema(self, schema_path: str) -> None: """ Load .tql schema file to database. @@ -590,6 +693,19 @@ def load_data(self, data_path: str, force: bool = False) -> None: data_path ) + def _load_data_unlocked(self, data_path: str) -> None: + """ + Load a data file while the caller already holds the query lock. + + :param data_path: .tql file path. + """ + if data_path is not None and data_path != '': + self._write_database_file_unlocked( + SessionType.DATA, + 'insert', + data_path + ) + # Events begining def insert_data_event(self): """Insert data event.""" diff --git a/ros_typedb/test/test_typedb_interface.py b/ros_typedb/test/test_typedb_interface.py index 15d082e..266e673 100644 --- a/ros_typedb/test/test_typedb_interface.py +++ b/ros_typedb/test/test_typedb_interface.py @@ -450,6 +450,77 @@ def raise_timeout(*args, **kwargs): assert 'timed out' in tdb.last_error.lower() +def test_ensure_database_exists_recreates_from_configured_paths(monkeypatch): + """Missing database is recreated with configured schema and data files.""" + class FakeDatabases: + + def __init__(self): + self.exists = False + self.created = [] + + def contains(self, name): + return self.exists + + def create(self, name): + self.created.append(name) + self.exists = True + + fake_databases = FakeDatabases() + tdb = TypeDBInterface.__new__(TypeDBInterface) + tdb.database_name = 'test_database' + tdb.driver = type('FakeDriver', (), {'databases': fake_databases})() + tdb.logger = logging.getLogger() + tdb._schema_paths = ['schema.tql'] + tdb._data_paths = ['data.tql'] + loaded = [] + + monkeypatch.setattr( + tdb, + '_load_schema_unlocked', + lambda path: loaded.append(('schema', path))) + monkeypatch.setattr( + tdb, + '_load_data_unlocked', + lambda path: loaded.append(('data', path))) + monkeypatch.setattr( + tdb, + 'database_query', + lambda *args, **kwargs: pytest.fail( + 'ensure_database_exists must not call database_query')) + + tdb.ensure_database_exists() + + assert fake_databases.created == ['test_database'] + assert loaded == [('schema', 'schema.tql'), ('data', 'data.tql')] + + +def test_ensure_database_exists_refuses_empty_recreate(): + """Missing database without init files must fail instead of going blank.""" + class FakeDatabases: + + def __init__(self): + self.created = [] + + def contains(self, name): + return False + + def create(self, name): + self.created.append(name) + + fake_databases = FakeDatabases() + tdb = TypeDBInterface.__new__(TypeDBInterface) + tdb.database_name = 'test_database' + tdb.driver = type('FakeDriver', (), {'databases': fake_databases})() + tdb.logger = logging.getLogger() + tdb._schema_paths = [] + tdb._data_paths = [] + + with pytest.raises(RuntimeError, match='cannot be recreated'): + tdb.ensure_database_exists() + + assert fake_databases.created == [] + + @pytest.fixture def typedb_interface(): typedb_interface = TypeDBInterface( @@ -1119,7 +1190,7 @@ def __exit__(self, exc_type, exc, traceback): assert typedb_interface.driver.session_count == 1 -def test_ensure_server_alive_creates_missing_database(monkeypatch): +def test_ensure_server_alive_refuses_empty_recreate(monkeypatch): class FakeDatabases: """Fake database collection tracking database creation.""" @@ -1149,6 +1220,7 @@ def close(self): typedb_interface = TypeDBInterface('localhost:1729', 'test_database') typedb_interface.driver.databases.created_database = None - typedb_interface.ensure_server_alive() + with pytest.raises(RuntimeError, match='cannot be recreated'): + typedb_interface.ensure_server_alive() - assert typedb_interface.driver.databases.created_database == 'test_database' + assert typedb_interface.driver.databases.created_database is None From cc4c15e9dab119ee76380121be14130d1412a615 Mon Sep 17 00:00:00 2001 From: Gustavo Date: Thu, 25 Jun 2026 17:39:58 -0300 Subject: [PATCH 04/36] Add reload schema control --- ros_typedb/ros_typedb/ros_typedb_interface.py | 8 ++- ros_typedb/ros_typedb/typedb_interface.py | 19 ++++--- ros_typedb/test/test_ros_typedb_interface.py | 21 ++++++++ ros_typedb/test/test_typedb_interface.py | 52 +++++++++++++++++++ 4 files changed, 93 insertions(+), 7 deletions(-) diff --git a/ros_typedb/ros_typedb/ros_typedb_interface.py b/ros_typedb/ros_typedb/ros_typedb_interface.py index 78f7219..1828fb4 100644 --- a/ros_typedb/ros_typedb/ros_typedb_interface.py +++ b/ros_typedb/ros_typedb/ros_typedb_interface.py @@ -318,6 +318,7 @@ def __init__(self, node_name: str, **kwargs): self.declare_parameter('database_name', 'ros_typedb') self.declare_parameter('force_database', True) self.declare_parameter('force_data', True) + self.declare_parameter('reload_schema', True) self.declare_parameter('infer', True) self.declare_parameter('driver_timeout_s', 10.0) self.declare_parameter('query_timeout_s', 0.0) @@ -340,6 +341,7 @@ def init_typedb_interface( data_path: Optional[list[str] | str] = None, force_database: Optional[bool] = False, force_data: Optional[bool] = False, + reload_schema: Optional[bool] = True, infer: Optional[bool] = False, sort_fetch_results: Optional[bool] = False, driver_timeout_s: Optional[float] = 10.0, @@ -353,6 +355,8 @@ def init_typedb_interface( :param data_path: list with paths to data files (.tql). :param force_database: if database should override an existing database :param force_data: if the database data should be overriden. + :param reload_schema: if schema files should be reapplied when the + database already exists. :param infer: if inference engine should be used. :param sort_fetch_results: if fetch query results should be recursively sorted. @@ -371,7 +375,8 @@ def init_typedb_interface( infer, sort_fetch_results, driver_timeout_s, - query_timeout_s=query_timeout_s + query_timeout_s=query_timeout_s, + reload_schema=reload_schema ) self.typedb_interface.insert_data_event = self.insert_data_event @@ -420,6 +425,7 @@ def on_configure(self, state: State) -> TransitionCallbackReturn: data_path=self.get_parameter('data_path').value, force_database=self.get_parameter('force_database').value, force_data=self.get_parameter('force_data').value, + reload_schema=self.get_parameter('reload_schema').value, infer=self.get_parameter('infer').value, sort_fetch_results=( self.get_parameter('sort_fetch_results').value), diff --git a/ros_typedb/ros_typedb/typedb_interface.py b/ros_typedb/ros_typedb/typedb_interface.py index 74ee61b..bae21a9 100644 --- a/ros_typedb/ros_typedb/typedb_interface.py +++ b/ros_typedb/ros_typedb/typedb_interface.py @@ -190,7 +190,8 @@ def __init__( infer: Optional[bool] = False, sort_fetch_results: Optional[bool] = False, driver_timeout_s: Optional[float] = 10.0, - query_timeout_s: Optional[float] = None) -> None: + query_timeout_s: Optional[float] = None, + reload_schema: Optional[bool] = True) -> None: """ Connect to a typeDB server and interacts with it. @@ -212,6 +213,8 @@ def __init__( Set to None or a non-positive value to wait without a timeout. :param query_timeout_s: default per-query timeout in seconds. Set to None or a non-positive value to wait without a timeout. + :param reload_schema: if schema files should be reapplied when the + database already exists. """ self.logger = logging.getLogger() self._database_query_lock = Lock() @@ -226,9 +229,11 @@ def __init__( self._schema_paths = normalize_path_list(schema_path) self._data_paths = normalize_path_list(data_path) self.connect_driver(address, timeout_s=driver_timeout_s) - self.create_database(database_name, force=force_database) - for path in self._schema_paths: - self.load_schema(path) + database_created = self.create_database( + database_name, force=force_database) + if reload_schema or database_created: + for path in self._schema_paths: + self.load_schema(path) if force_data: self.delete_all_data() for path in self._data_paths: @@ -392,12 +397,13 @@ def delete_database(self, database_name: str = None) -> None: self.driver.databases.get(database_name).delete() def create_database( - self, database_name: str, force: Optional[bool] = False) -> None: + self, database_name: str, force: Optional[bool] = False) -> bool: """ Create database. :param database_name: database name. :param force: if database should override an existing database + :return: True if a database was created. """ if force: self.delete_database(database_name) @@ -408,9 +414,10 @@ def create_database( 'The database with the name %s already exists. ' 'Ignoring create_database request.', database_name) - return + return False self.driver.databases.create(database_name) + return True def create_session( self, diff --git a/ros_typedb/test/test_ros_typedb_interface.py b/ros_typedb/test/test_ros_typedb_interface.py index 4814b1b..d561361 100644 --- a/ros_typedb/test/test_ros_typedb_interface.py +++ b/ros_typedb/test/test_ros_typedb_interface.py @@ -103,6 +103,27 @@ def __init__(self, *args, **kwargs): assert captured.get('query_timeout_s') == 15.0 +def test_init_typedb_interface_accepts_reload_schema(): + """init_typedb_interface passes reload_schema to TypeDBInterface.""" + captured = {} + + class FakeTypeDBInterface: + + def __init__(self, *args, **kwargs): + captured.update(kwargs) + self.insert_data_event = None + self.delete_data_event = None + + node = ROSTypeDBInterface.__new__(ROSTypeDBInterface) + node.typedb_interface_class = FakeTypeDBInterface + node.init_typedb_interface( + address='localhost:1729', + database_name='test', + reload_schema=False) + + assert captured.get('reload_schema') is False + + def test_query_service_cb_passes_timeout_to_wrapper(): """query_service_cb extracts req.timeout_s and passes it as timeout= to the wrapper.""" captured = {} diff --git a/ros_typedb/test/test_typedb_interface.py b/ros_typedb/test/test_typedb_interface.py index 266e673..c5b2a24 100644 --- a/ros_typedb/test/test_typedb_interface.py +++ b/ros_typedb/test/test_typedb_interface.py @@ -521,6 +521,58 @@ def create(self, name): assert fake_databases.created == [] +def test_init_skips_schema_reload_when_database_is_reused(monkeypatch): + """reload_schema=False skips define queries for an existing database.""" + loaded_schema_paths = [] + + monkeypatch.setattr( + TypeDBInterface, 'connect_driver', lambda self, *args, **kwargs: None) + monkeypatch.setattr( + TypeDBInterface, 'create_database', + lambda self, *args, **kwargs: False) + monkeypatch.setattr( + TypeDBInterface, 'load_schema', + lambda self, path: loaded_schema_paths.append(path)) + monkeypatch.setattr( + TypeDBInterface, 'delete_all_data', lambda self: None) + monkeypatch.setattr( + TypeDBInterface, 'load_data', lambda self, path: None) + + TypeDBInterface( + 'localhost:1729', + 'test_database', + schema_path=['schema.tql'], + reload_schema=False) + + assert loaded_schema_paths == [] + + +def test_init_loads_schema_for_new_database_when_reload_disabled(monkeypatch): + """Fresh databases still get schema even when reload_schema=False.""" + loaded_schema_paths = [] + + monkeypatch.setattr( + TypeDBInterface, 'connect_driver', lambda self, *args, **kwargs: None) + monkeypatch.setattr( + TypeDBInterface, 'create_database', + lambda self, *args, **kwargs: True) + monkeypatch.setattr( + TypeDBInterface, 'load_schema', + lambda self, path: loaded_schema_paths.append(path)) + monkeypatch.setattr( + TypeDBInterface, 'delete_all_data', lambda self: None) + monkeypatch.setattr( + TypeDBInterface, 'load_data', lambda self, path: None) + + TypeDBInterface( + 'localhost:1729', + 'test_database', + schema_path=['schema.tql'], + reload_schema=False) + + assert loaded_schema_paths == ['schema.tql'] + + @pytest.fixture def typedb_interface(): typedb_interface = TypeDBInterface( From f88e9af7588f3cc7c0d08f25812154f59b3f305d Mon Sep 17 00:00:00 2001 From: Gustavo Date: Thu, 25 Jun 2026 18:07:02 -0300 Subject: [PATCH 05/36] Fail on incomplete data deletion --- ros_typedb/ros_typedb/typedb_interface.py | 16 +++-- ros_typedb/test/test_typedb_interface.py | 72 +++++++++++++++++++++++ 2 files changed, 82 insertions(+), 6 deletions(-) diff --git a/ros_typedb/ros_typedb/typedb_interface.py b/ros_typedb/ros_typedb/typedb_interface.py index bae21a9..1fb8d13 100644 --- a/ros_typedb/ros_typedb/typedb_interface.py +++ b/ros_typedb/ros_typedb/typedb_interface.py @@ -679,12 +679,16 @@ def load_schema(self, schema_path: str) -> None: def delete_all_data(self) -> None: """Delete all data from database.""" - self.delete_from_database( - 'match $e isa entity; delete $e isa entity;') - self.delete_from_database( - 'match $r isa relation; delete $r isa relation;') - self.delete_from_database( - 'match $a isa attribute; delete $a isa attribute;') + delete_queries = [ + 'match $e isa entity; delete $e isa entity;', + 'match $r isa relation; delete $r isa relation;', + 'match $a isa attribute; delete $a isa attribute;', + ] + for query in delete_queries: + if self.delete_from_database(query) is None: + raise RuntimeError( + 'Failed to delete all TypeDB data: {}'.format( + self.last_error)) def load_data(self, data_path: str, force: bool = False) -> None: """ diff --git a/ros_typedb/test/test_typedb_interface.py b/ros_typedb/test/test_typedb_interface.py index c5b2a24..4438372 100644 --- a/ros_typedb/test/test_typedb_interface.py +++ b/ros_typedb/test/test_typedb_interface.py @@ -588,6 +588,78 @@ def typedb_interface(): typedb_interface.delete_database() +def _fetch_entities(typedb_interface): + return typedb_interface.get_aggregate_database( + 'match $e isa entity; get $e; count;') + + +def _fetch_relations(typedb_interface): + return typedb_interface.get_aggregate_database( + 'match $r isa relation; get $r; count;') + + +def _fetch_attributes(typedb_interface): + return typedb_interface.get_aggregate_database( + 'match $a isa attribute; get $a; count;') + + +def test_delete_all_data_removes_entities_relations_and_attributes( + typedb_interface): + """delete_all_data removes all live fixture data.""" + assert _fetch_entities(typedb_interface) + assert _fetch_relations(typedb_interface) + assert _fetch_attributes(typedb_interface) + + typedb_interface.delete_all_data() + + assert _fetch_entities(typedb_interface) == 0 + assert _fetch_relations(typedb_interface) == 0 + assert _fetch_attributes(typedb_interface) == 0 + + +def test_delete_all_data_raises_when_delete_step_fails(monkeypatch): + """delete_all_data stops instead of loading data over residual data.""" + typedb_interface = TypeDBInterface.__new__(TypeDBInterface) + typedb_interface.last_error = 'delete failed' + queries = [] + + def fake_delete_from_database(query): + queries.append(query) + return None + + monkeypatch.setattr( + typedb_interface, 'delete_from_database', fake_delete_from_database) + + with pytest.raises(RuntimeError, match='delete failed'): + typedb_interface.delete_all_data() + + assert queries == ['match $e isa entity; delete $e isa entity;'] + + +def test_delete_entities_before_relations_leaves_no_residual_data( + typedb_interface): + """Document TypeDB behavior for the current delete_all_data order.""" + assert _fetch_relations(typedb_interface) + + typedb_interface.delete_from_database( + 'match $e isa entity; delete $e isa entity;') + after_entity_delete_error = typedb_interface.last_error + entities_after_entity_delete = _fetch_entities(typedb_interface) + relations_after_entity_delete = _fetch_relations(typedb_interface) + + typedb_interface.delete_from_database( + 'match $r isa relation; delete $r isa relation;') + typedb_interface.delete_from_database( + 'match $a isa attribute; delete $a isa attribute;') + + assert after_entity_delete_error == '' + assert entities_after_entity_delete == 0 + assert relations_after_entity_delete == 0 + assert _fetch_entities(typedb_interface) == 0 + assert _fetch_relations(typedb_interface) == 0 + assert _fetch_attributes(typedb_interface) == 0 + + def test_convert_py_type_to_query_type_escapes_strings(): assert convert_py_type_to_query_type("O'Brien") == "'O\\'Brien'" assert convert_py_type_to_query_type(r'C:\\tmp') == r"'C:\\\\tmp'" From 0d2c9d5586e3d2b6e034b46b391a07304c8d0199 Mon Sep 17 00:00:00 2001 From: Gustavo Date: Thu, 25 Jun 2026 18:19:59 -0300 Subject: [PATCH 06/36] split test_typedb_interface into 2 test files --- .../test/test_typedb_interface_integration.py | 563 ++++++++++++++++++ ...rface.py => test_typedb_interface_unit.py} | 544 ----------------- 2 files changed, 563 insertions(+), 544 deletions(-) create mode 100644 ros_typedb/test/test_typedb_interface_integration.py rename ros_typedb/test/{test_typedb_interface.py => test_typedb_interface_unit.py} (58%) diff --git a/ros_typedb/test/test_typedb_interface_integration.py b/ros_typedb/test/test_typedb_interface_integration.py new file mode 100644 index 0000000..0adbbf2 --- /dev/null +++ b/ros_typedb/test/test_typedb_interface_integration.py @@ -0,0 +1,563 @@ +# Copyright 2023 Gustavo Rezende Silva +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from datetime import datetime + +import pytest + +from ros_typedb.typedb_interface import TypeDBInterface + + +@pytest.fixture +def typedb_interface(): + typedb_interface = TypeDBInterface( + 'localhost:1729', + 'test_database', + force_database=True, + schema_path=['test/typedb_test_data/schema.tql'], + data_path=['test/typedb_test_data/data.tql'], + force_data=True, + sort_fetch_results=True + ) + yield typedb_interface + typedb_interface.delete_database() + + +def _fetch_entities(typedb_interface): + return typedb_interface.get_aggregate_database( + 'match $e isa entity; get $e; count;') + + +def _fetch_relations(typedb_interface): + return typedb_interface.get_aggregate_database( + 'match $r isa relation; get $r; count;') + + +def _fetch_attributes(typedb_interface): + return typedb_interface.get_aggregate_database( + 'match $a isa attribute; get $a; count;') + + +def test_delete_all_data_removes_entities_relations_and_attributes( + typedb_interface): + """delete_all_data removes all live fixture data.""" + assert _fetch_entities(typedb_interface) + assert _fetch_relations(typedb_interface) + assert _fetch_attributes(typedb_interface) + + typedb_interface.delete_all_data() + + assert _fetch_entities(typedb_interface) == 0 + assert _fetch_relations(typedb_interface) == 0 + assert _fetch_attributes(typedb_interface) == 0 + + +def test_delete_entities_before_relations_leaves_no_residual_data( + typedb_interface): + """Document TypeDB behavior for the current delete_all_data order.""" + assert _fetch_relations(typedb_interface) + + typedb_interface.delete_from_database( + 'match $e isa entity; delete $e isa entity;') + after_entity_delete_error = typedb_interface.last_error + entities_after_entity_delete = _fetch_entities(typedb_interface) + relations_after_entity_delete = _fetch_relations(typedb_interface) + + typedb_interface.delete_from_database( + 'match $r isa relation; delete $r isa relation;') + typedb_interface.delete_from_database( + 'match $a isa attribute; delete $a isa attribute;') + + assert after_entity_delete_error == '' + assert entities_after_entity_delete == 0 + assert relations_after_entity_delete == 0 + assert _fetch_entities(typedb_interface) == 0 + assert _fetch_relations(typedb_interface) == 0 + assert _fetch_attributes(typedb_interface) == 0 + + +def test_load_bad_data_raises(typedb_interface): + """Regression: load_data must raise on invalid TypeQL, not silently pass.""" + with pytest.raises(Exception): + typedb_interface.load_data('test/typedb_test_data/bad_data.tql') + + +def test_create_and_delete_database(): + typedb_interface = TypeDBInterface( + 'localhost:1729', + 'test_database', + force_database=True, + schema_path=['test/typedb_test_data/schema.tql'], + data_path=['test/typedb_test_data/data.tql'], + force_data=True, + ) + + assert typedb_interface.driver.databases.contains('test_database') + + typedb_interface.delete_database() + assert not typedb_interface.driver.databases.contains('test_database') + + +def test_define_query(typedb_interface): + assert typedb_interface.define_database('define MyEntity sub entity;') + assert typedb_interface.define_database( + 'define MyEntity aa entity') is None + + +def test_insert_entity(typedb_interface): + result_insert = typedb_interface.insert_entity( + 'person', [('email', 'test@email.test'), ('nickname', 't')]) + assert result_insert is not None + query = """ + match $entity isa person, + has email "test@email.test", + has nickname "t"; + get $entity; + count; + """ + result = typedb_interface.get_aggregate_database(query) + assert result > 0 + + result_insert = typedb_interface.insert_entity( + 'something_wrong', [('email', 'test@email.test'), ('nickname', 't')]) + assert result_insert is None + + +def test_delete_thing(typedb_interface): + typedb_interface.insert_entity('person', [('email', 'test@email.test')]) + typedb_interface.delete_thing('person', 'email', 'test@email.test') + query = """ + match $entity isa person, has email "test@email.test"; + get $entity; + """ + result = typedb_interface.get_database(query) + assert len(result) == 0 + + wrong_result = typedb_interface.delete_thing( + 'something_wrong', 'email', 'test@email.test') + assert wrong_result is None + + +@pytest.mark.parametrize('attr, attr_value', [ + ('nickname', 't'), + ('alive', True), + ('age', 33), + ('height', 3.237), + ('birth-date', datetime.now()), +]) +def test_insert_attribute_in_thing(typedb_interface, attr, attr_value): + typedb_interface.insert_entity('person', [('email', 'test@email.test')]) + typedb_interface.insert_attribute_in_thing( + 'person', 'email', 'test@email.test', attr, attr_value) + result = typedb_interface.fetch_attribute_from_thing( + 'person', [('email', 'test@email.test')], attr) + assert result + + +@pytest.mark.parametrize('attr, attr_value', [ + ('nickname', 't'), + ('alive', True), + ('age', 33), + ('height', 3.237), + ('birth-date', datetime.fromisoformat( + datetime.now().isoformat(timespec='milliseconds'))), +]) +def test_fetch_attribute_from_thing(typedb_interface, attr, attr_value): + typedb_interface.insert_entity( + 'person', + [('email', 'test@email.test'), ('gender', 'male')] + ) + typedb_interface.insert_attribute_in_thing( + 'person', 'email', 'test@email.test', attr, attr_value) + + result = typedb_interface.fetch_attribute_from_thing( + 'person', [('email', 'test@email.test'), ('gender', 'male')], attr) + + assert result[0] == attr_value + + +@pytest.mark.parametrize('attr, attr_value', [ + ('nickname', 't'), + ('alive', True), + ('age', 33), + ('height', 3.237), + ('birth-date', datetime.fromisoformat( + datetime.now().isoformat(timespec='milliseconds'))), +]) +def test_delete_attribute_from_thing(typedb_interface, attr, attr_value): + typedb_interface.insert_entity('person', [('email', 'test@email.test')]) + typedb_interface.insert_attribute_in_thing( + 'person', 'email', 'test@email.test', attr, attr_value) + + typedb_interface.delete_attribute_from_thing( + 'person', 'email', 'test@email.test', attr) + + result = typedb_interface.fetch_attribute_from_thing( + 'person', [('email', 'test@email.test')], attr) + + assert len(result) == 0 + + +@pytest.mark.parametrize('attr, attr_value, new_v', [ + ('nickname', 't', 'new_t'), + ('alive', True, False), + ('age', 33, 56), + ('height', 3.237, 1.66), + ('birth-date', datetime.fromisoformat( + datetime.now().isoformat(timespec='milliseconds')), + datetime.fromisoformat( + datetime.now().isoformat(timespec='milliseconds'))), +]) +def test_update_attribute_in_thing(typedb_interface, attr, attr_value, new_v): + typedb_interface.insert_entity('person', [('email', 'test@email.test')]) + typedb_interface.insert_attribute_in_thing( + 'person', 'email', 'test@email.test', attr, attr_value) + result_update = typedb_interface.update_attribute_in_thing( + 'person', 'email', 'test@email.test', attr, new_v) + result = typedb_interface.fetch_attribute_from_thing( + 'person', [('email', 'test@email.test')], attr) + + assert result_update is not None and result[0] == new_v + + +def test_insert_relationship(typedb_interface): + typedb_interface.insert_entity('person', [('email', 'test@email.test')]) + typedb_interface.insert_entity('person', [('email', 'test2@email.test')]) + typedb_interface.insert_entity('person', [('email', 'test3@email.test')]) + typedb_interface.insert_entity('person', [('email', 'test4@email.test')]) + + related_things_dict = { + 'employee': [ + ('person', 'email', 'test@email.test'), + ('person', 'email', 'test2@email.test')], + 'employer': [ + ('person', 'email', 'test3@email.test'), + ('person', 'email', 'test4@email.test') + ] + } + attribute_list = [ + ('salary', 2333), ('role-name', 'boss'), ('role-name', 'super boss')] + typedb_interface.insert_relationship( + 'employment', related_things_dict, attribute_list) + + query = """ + match $r (employee:$ee, employer:$er)isa employment, + has salary 2333, + has role-name "boss", + has role-name "super boss"; + get $r; + count; + """ + result = typedb_interface.get_aggregate_database(query) + assert result > 0 + + +@pytest.mark.parametrize('things_dict', [ + ({ + 'person': [ + { + 'prefix': 'p1', + 'attributes': { + 'email': 'test@email.test', + 'nickname': 't', + 'height': 1.80, + 'age': 18, + 'alive': True, + 'birth-date': datetime.now() + } + }, + { + 'prefix': 'p2', + 'attributes': { + 'email': 'test2@email.test', + 'nickname': 't2', + 'height': 1.33, + 'age': 128, + 'alive': False, + 'birth-date': datetime.now() + } + }, + ], + 'robot': [ + { + 'prefix': 'r', + 'attributes': { + 'full-name': 'robot123', + 'height': 1.0, + 'age': 0, + 'alive': True, + 'birth-date': datetime.now() + } + }, + ], + 'employment': [ + { + 'prefix': 'e', + 'attributes': { + 'salary': 2333, + 'role-name': ['boss', 'super boss'], + }, + 'relationship': { + 'employee': 'p1', + 'employer': 'p2' + } + }, + ] + }), +]) +def test_dict_to_query(typedb_interface, things_dict): + query = typedb_interface.dict_to_query(things_dict) + insert_result = typedb_interface.insert_database('insert ' + query) + assert insert_result is not None + + +@pytest.mark.parametrize('match_dict, r_dict', [ + ( + { + 'person': [ + { + 'prefix': 'p1', + 'attributes': { + 'email': 'test_person@test.test', + }, + 'insert_attributes': { + 'nickname': 't', + 'height': 1.80, + 'age': 18, + 'alive': True, + 'birth-date': datetime.now() + } + }, + ], + }, + { + 'person': [ + { + 'prefix': 'p1', + 'attributes': { + 'email': 'test_person@test.test', + 'nickname': 't', + 'height': 1.80, + 'age': 18, + 'alive': True, + 'birth-date': datetime.now() + } + }, + ], + }, + ), +]) +def test_insert_attributes(typedb_interface, match_dict, r_dict): + r = typedb_interface.insert_attributes_in_thing(match_dict) + query = typedb_interface.dict_to_query(r_dict) + match_result = typedb_interface.get_aggregate_database( + 'match ' + query + 'get; count;') + assert r is not None and r is not False and match_result > 0 + + +@pytest.mark.parametrize('insert_dict, match_dict', [ + ( + { + 'person': [ + { + 'prefix': 'p1', + 'attributes': { + 'email': 'test@test.test', + 'nickname': 't', + 'height': 1.80, + 'age': 18, + 'alive': True, + 'birth-date': datetime.now() + } + }, + ], + 'robot': [ + { + 'prefix': 'r', + 'attributes': { + 'full-name': 'robot123', + 'height': 1.0, + 'age': 0, + 'alive': True, + 'birth-date': datetime.now(), + 'robot-type': 'auv' + } + }, + ], + }, + { + 'person': [ + { + 'prefix': 'p1', + 'attributes': { + 'email': 'test@test.test', + }, + 'delete_attributes': [ + 'nickname', 'height', 'age', 'alive', 'birth-date'] + }, + ], + 'robot': [ + { + 'attributes': { + 'full-name': 'robot123', + }, + 'delete_attributes': ['robot-type'] + }, + ], + }, + ) +]) +def test_delete_attributes( + typedb_interface, insert_dict, match_dict): + + query = typedb_interface.dict_to_query(insert_dict) + insert_result = typedb_interface.insert_database('insert ' + query) + assert insert_result is not None + assert len(insert_result) > 0 + + delete_result = typedb_interface.delete_attributes_from_thing(match_dict) + assert delete_result is True + + +@pytest.mark.parametrize('insert_dict, update_dict, r_dict', [ + ( + { + 'person': [ + { + 'prefix': 'p1', + 'attributes': { + 'email': 'test@test.test', + 'nickname': 't', + 'height': 1.80, + 'age': 18, + 'alive': True, + 'birth-date': datetime.now() + } + }, + ], + }, + { + 'person': [ + { + 'prefix': 'p1', + 'attributes': { + 'email': 'test@test.test', + }, + 'update_attributes': { + 'nickname': 't2', + 'height': 1.50, + 'age': 17, + 'alive': False, + } + }, + ], + }, + { + 'person': [ + { + 'prefix': 'p1', + 'attributes': { + 'email': 'test@test.test', + 'nickname': 't2', + 'height': 1.50, + 'age': 17, + 'alive': False, + } + }, + ], + }, + ), +]) +def test_update_attributes(typedb_interface, insert_dict, update_dict, r_dict): + query = typedb_interface.dict_to_query(insert_dict) + typedb_interface.insert_database('insert ' + query) + + r = typedb_interface.update_attributes_in_thing(update_dict) + query = typedb_interface.dict_to_query(r_dict) + match_result = typedb_interface.get_aggregate_database( + 'match ' + query + 'get; count;') + assert r is not None and r is not False and match_result > 0 + + +def test_get_query(typedb_interface): + query = """ + match + $p isa person, has full-name $name, has email $email; + get $name, $email; + sort $name asc; limit 3; + """ + result = typedb_interface.get_database(query) + name = result[0].get('name').as_attribute().get_value() + assert len(result) == 3 and name == 'Ahmed Frazier' + + +def test_fetch_query(typedb_interface): + query = """ + match + $company isa company, has name "TU Delft"; + fetch + employee_names: { + match + $e (employer: $company, employee: $employee) isa employment; + fetch + $employee: full-name, email; + $e: salary; + }; + """ + result = typedb_interface.fetch_database(query) + assert len(result) == 1 + assert len(result[0]['employee_names']) == 3 + + expected = [ + { + 'salary': 30000, + 'email': 'phd@tudelft.nl', + 'full-name': 'PhD candidate 1' + }, + { + 'salary': 999999, + 'email': 'boss@tudelft.nl', + 'full-name': 'Big Boss' + }, + { + 'salary': 0, + 'email': 'guest@tudelft.nl', + 'full-name': 'Random guest' + } + ] + + actual = [] + for emp in result[0]['employee_names']: + actual.append({ + 'salary': emp['e']['salary'][0]['value'], + 'email': emp['employee']['email'][0]['value'], + 'full-name': emp['employee']['full-name'][0]['value'] + }) + + assert {tuple(sorted(d.items())) for d in actual} == \ + {tuple(sorted(d.items())) for d in expected} + + +def test_register_method(typedb_interface): + def get_name_email(self, name): + query = f""" + match + $p isa person, has full-name '{name}'; + fetch + $p: email; + """ + return typedb_interface.fetch_database( + query)[0]['p']['email'][0]['value'] + typedb_interface.register_method('get_name_email', get_name_email) + assert typedb_interface.get_name_email('Big Boss') == 'boss@tudelft.nl' diff --git a/ros_typedb/test/test_typedb_interface.py b/ros_typedb/test/test_typedb_interface_unit.py similarity index 58% rename from ros_typedb/test/test_typedb_interface.py rename to ros_typedb/test/test_typedb_interface_unit.py index 4438372..d2c8add 100644 --- a/ros_typedb/test/test_typedb_interface.py +++ b/ros_typedb/test/test_typedb_interface_unit.py @@ -573,50 +573,6 @@ def test_init_loads_schema_for_new_database_when_reload_disabled(monkeypatch): assert loaded_schema_paths == ['schema.tql'] -@pytest.fixture -def typedb_interface(): - typedb_interface = TypeDBInterface( - 'localhost:1729', - 'test_database', - force_database=True, - schema_path=['test/typedb_test_data/schema.tql'], - data_path=['test/typedb_test_data/data.tql'], - force_data=True, - sort_fetch_results=True - ) - yield typedb_interface - typedb_interface.delete_database() - - -def _fetch_entities(typedb_interface): - return typedb_interface.get_aggregate_database( - 'match $e isa entity; get $e; count;') - - -def _fetch_relations(typedb_interface): - return typedb_interface.get_aggregate_database( - 'match $r isa relation; get $r; count;') - - -def _fetch_attributes(typedb_interface): - return typedb_interface.get_aggregate_database( - 'match $a isa attribute; get $a; count;') - - -def test_delete_all_data_removes_entities_relations_and_attributes( - typedb_interface): - """delete_all_data removes all live fixture data.""" - assert _fetch_entities(typedb_interface) - assert _fetch_relations(typedb_interface) - assert _fetch_attributes(typedb_interface) - - typedb_interface.delete_all_data() - - assert _fetch_entities(typedb_interface) == 0 - assert _fetch_relations(typedb_interface) == 0 - assert _fetch_attributes(typedb_interface) == 0 - - def test_delete_all_data_raises_when_delete_step_fails(monkeypatch): """delete_all_data stops instead of loading data over residual data.""" typedb_interface = TypeDBInterface.__new__(TypeDBInterface) @@ -636,30 +592,6 @@ def fake_delete_from_database(query): assert queries == ['match $e isa entity; delete $e isa entity;'] -def test_delete_entities_before_relations_leaves_no_residual_data( - typedb_interface): - """Document TypeDB behavior for the current delete_all_data order.""" - assert _fetch_relations(typedb_interface) - - typedb_interface.delete_from_database( - 'match $e isa entity; delete $e isa entity;') - after_entity_delete_error = typedb_interface.last_error - entities_after_entity_delete = _fetch_entities(typedb_interface) - relations_after_entity_delete = _fetch_relations(typedb_interface) - - typedb_interface.delete_from_database( - 'match $r isa relation; delete $r isa relation;') - typedb_interface.delete_from_database( - 'match $a isa attribute; delete $a isa attribute;') - - assert after_entity_delete_error == '' - assert entities_after_entity_delete == 0 - assert relations_after_entity_delete == 0 - assert _fetch_entities(typedb_interface) == 0 - assert _fetch_relations(typedb_interface) == 0 - assert _fetch_attributes(typedb_interface) == 0 - - def test_convert_py_type_to_query_type_escapes_strings(): assert convert_py_type_to_query_type("O'Brien") == "'O\\'Brien'" assert convert_py_type_to_query_type(r'C:\\tmp') == r"'C:\\\\tmp'" @@ -683,12 +615,6 @@ def fake_delete_from_database(query): assert 'has name "O\'Brien"' not in captured_query -def test_load_bad_data_raises(typedb_interface): - """Regression: load_data must raise on invalid TypeQL, not silently pass.""" - with pytest.raises(Exception): - typedb_interface.load_data('test/typedb_test_data/bad_data.tql') - - def test_driver_connection_timeout(monkeypatch): def slow_core_driver(address): time.sleep(1.0) @@ -743,62 +669,6 @@ def slow_core_driver(address): assert late_driver.closed -def test_create_and_delete_database(): - typedb_interface = TypeDBInterface( - 'localhost:1729', - 'test_database', - force_database=True, - schema_path=['test/typedb_test_data/schema.tql'], - data_path=['test/typedb_test_data/data.tql'], - force_data=True, - ) - - assert typedb_interface.driver.databases.contains('test_database') - - typedb_interface.delete_database() - assert not typedb_interface.driver.databases.contains('test_database') - - -def test_define_query(typedb_interface): - assert typedb_interface.define_database('define MyEntity sub entity;') - assert typedb_interface.define_database( - 'define MyEntity aa entity') is None - - -def test_insert_entity(typedb_interface): - result_insert = typedb_interface.insert_entity( - 'person', [('email', 'test@email.test'), ('nickname', 't')]) - assert result_insert is not None - query = """ - match $entity isa person, - has email "test@email.test", - has nickname "t"; - get $entity; - count; - """ - result = typedb_interface.get_aggregate_database(query) - assert result > 0 - - result_insert = typedb_interface.insert_entity( - 'something_wrong', [('email', 'test@email.test'), ('nickname', 't')]) - assert result_insert is None - - -def test_delete_thing(typedb_interface): - typedb_interface.insert_entity('person', [('email', 'test@email.test')]) - typedb_interface.delete_thing('person', 'email', 'test@email.test') - query = """ - match $entity isa person, has email "test@email.test"; - get $entity; - """ - result = typedb_interface.get_database(query) - assert len(result) == 0 - - wrong_result = typedb_interface.delete_thing( - 'something_wrong', 'email', 'test@email.test') - assert wrong_result is None - - @pytest.mark.parametrize('key_value, expected_query_value', [ ('test@email.test', "'test@email.test'"), (33, '33'), @@ -821,420 +691,6 @@ def capture_delete(query): assert f'has email {expected_query_value};' in queries[0] -@pytest.mark.parametrize('attr, attr_value', [ - ('nickname', 't'), - ('alive', True), - ('age', 33), - ('height', 3.237), - ('birth-date', datetime.now()), -]) -def test_insert_attribute_in_thing(typedb_interface, attr, attr_value): - typedb_interface.insert_entity('person', [('email', 'test@email.test')]) - typedb_interface.insert_attribute_in_thing( - 'person', 'email', 'test@email.test', attr, attr_value) - result = typedb_interface.fetch_attribute_from_thing( - 'person', [('email', 'test@email.test')], attr) - assert result - - -@pytest.mark.parametrize('attr, attr_value', [ - ('nickname', 't'), - ('alive', True), - ('age', 33), - ('height', 3.237), - ('birth-date', datetime.fromisoformat( - datetime.now().isoformat(timespec='milliseconds'))), -]) -def test_fetch_attribute_from_thing(typedb_interface, attr, attr_value): - typedb_interface.insert_entity( - 'person', - [('email', 'test@email.test'), ('gender', 'male')] - ) - typedb_interface.insert_attribute_in_thing( - 'person', 'email', 'test@email.test', attr, attr_value) - - result = typedb_interface.fetch_attribute_from_thing( - 'person', [('email', 'test@email.test'), ('gender', 'male')], attr) - - assert result[0] == attr_value - - -@pytest.mark.parametrize('attr, attr_value', [ - ('nickname', 't'), - ('alive', True), - ('age', 33), - ('height', 3.237), - ('birth-date', datetime.fromisoformat( - datetime.now().isoformat(timespec='milliseconds'))), -]) -def test_delete_attribute_from_thing(typedb_interface, attr, attr_value): - typedb_interface.insert_entity('person', [('email', 'test@email.test')]) - typedb_interface.insert_attribute_in_thing( - 'person', 'email', 'test@email.test', attr, attr_value) - - typedb_interface.delete_attribute_from_thing( - 'person', 'email', 'test@email.test', attr) - - result = typedb_interface.fetch_attribute_from_thing( - 'person', [('email', 'test@email.test')], attr) - - assert len(result) == 0 - - -@pytest.mark.parametrize('attr, attr_value, new_v', [ - ('nickname', 't', 'new_t'), - ('alive', True, False), - ('age', 33, 56), - ('height', 3.237, 1.66), - ('birth-date', datetime.fromisoformat( - datetime.now().isoformat(timespec='milliseconds')), - datetime.fromisoformat( - datetime.now().isoformat(timespec='milliseconds'))), -]) -def test_update_attribute_in_thing(typedb_interface, attr, attr_value, new_v): - typedb_interface.insert_entity('person', [('email', 'test@email.test')]) - typedb_interface.insert_attribute_in_thing( - 'person', 'email', 'test@email.test', attr, attr_value) - result_update = typedb_interface.update_attribute_in_thing( - 'person', 'email', 'test@email.test', attr, new_v) - result = typedb_interface.fetch_attribute_from_thing( - 'person', [('email', 'test@email.test')], attr) - - assert result_update is not None and result[0] == new_v - - -def test_insert_relationship(typedb_interface): - typedb_interface.insert_entity('person', [('email', 'test@email.test')]) - typedb_interface.insert_entity('person', [('email', 'test2@email.test')]) - typedb_interface.insert_entity('person', [('email', 'test3@email.test')]) - typedb_interface.insert_entity('person', [('email', 'test4@email.test')]) - - related_things_dict = { - 'employee': [ - ('person', 'email', 'test@email.test'), - ('person', 'email', 'test2@email.test')], - 'employer': [ - ('person', 'email', 'test3@email.test'), - ('person', 'email', 'test4@email.test') - ] - } - attribute_list = [ - ('salary', 2333), ('role-name', 'boss'), ('role-name', 'super boss')] - typedb_interface.insert_relationship( - 'employment', related_things_dict, attribute_list) - - query = """ - match $r (employee:$ee, employer:$er)isa employment, - has salary 2333, - has role-name "boss", - has role-name "super boss"; - get $r; - count; - """ - result = typedb_interface.get_aggregate_database(query) - assert result > 0 - - -@pytest.mark.parametrize('things_dict', [ - ({ - 'person': [ - { - 'prefix': 'p1', - 'attributes': { - 'email': 'test@email.test', - 'nickname': 't', - 'height': 1.80, - 'age': 18, - 'alive': True, - 'birth-date': datetime.now() - } - }, - { - 'prefix': 'p2', - 'attributes': { - 'email': 'test2@email.test', - 'nickname': 't2', - 'height': 1.33, - 'age': 128, - 'alive': False, - 'birth-date': datetime.now() - } - }, - ], - 'robot': [ - { - 'prefix': 'r', - 'attributes': { - 'full-name': 'robot123', - 'height': 1.0, - 'age': 0, - 'alive': True, - 'birth-date': datetime.now() - } - }, - ], - 'employment': [ - { - 'prefix': 'e', - 'attributes': { - 'salary': 2333, - 'role-name': ['boss', 'super boss'], - }, - 'relationship': { - 'employee': 'p1', - 'employer': 'p2' - } - }, - ] - }), -]) -def test_dict_to_query(typedb_interface, things_dict): - query = typedb_interface.dict_to_query(things_dict) - insert_result = typedb_interface.insert_database('insert ' + query) - assert insert_result is not None - - -@pytest.mark.parametrize('match_dict, r_dict', [ - ( - { - 'person': [ - { - 'prefix': 'p1', - 'attributes': { - 'email': 'test_person@test.test', - }, - 'insert_attributes': { - 'nickname': 't', - 'height': 1.80, - 'age': 18, - 'alive': True, - 'birth-date': datetime.now() - } - }, - ], - }, - { - 'person': [ - { - 'prefix': 'p1', - 'attributes': { - 'email': 'test_person@test.test', - 'nickname': 't', - 'height': 1.80, - 'age': 18, - 'alive': True, - 'birth-date': datetime.now() - } - }, - ], - }, - ), -]) -def test_insert_attributes(typedb_interface, match_dict, r_dict): - r = typedb_interface.insert_attributes_in_thing(match_dict) - query = typedb_interface.dict_to_query(r_dict) - match_result = typedb_interface.get_aggregate_database( - 'match ' + query + 'get; count;') - assert r is not None and r is not False and match_result > 0 - - -@pytest.mark.parametrize('insert_dict, match_dict', [ - ( - { - 'person': [ - { - 'prefix': 'p1', - 'attributes': { - 'email': 'test@test.test', - 'nickname': 't', - 'height': 1.80, - 'age': 18, - 'alive': True, - 'birth-date': datetime.now() - } - }, - ], - 'robot': [ - { - 'prefix': 'r', - 'attributes': { - 'full-name': 'robot123', - 'height': 1.0, - 'age': 0, - 'alive': True, - 'birth-date': datetime.now(), - 'robot-type': 'auv' - } - }, - ], - }, - { - 'person': [ - { - 'prefix': 'p1', - 'attributes': { - 'email': 'test@test.test', - }, - 'delete_attributes': [ - 'nickname', 'height', 'age', 'alive', 'birth-date'] - }, - ], - 'robot': [ - { - 'attributes': { - 'full-name': 'robot123', - }, - 'delete_attributes': ['robot-type'] - }, - ], - }, - ) -]) -def test_delete_attributes( - typedb_interface, insert_dict, match_dict): - - query = typedb_interface.dict_to_query(insert_dict) - insert_result = typedb_interface.insert_database('insert ' + query) - assert insert_result is not None - assert len(insert_result) > 0 - - delete_result = typedb_interface.delete_attributes_from_thing(match_dict) - assert delete_result is True - - -@pytest.mark.parametrize('insert_dict, update_dict, r_dict', [ - ( - { - 'person': [ - { - 'prefix': 'p1', - 'attributes': { - 'email': 'test@test.test', - 'nickname': 't', - 'height': 1.80, - 'age': 18, - 'alive': True, - 'birth-date': datetime.now() - } - }, - ], - }, - { - 'person': [ - { - 'prefix': 'p1', - 'attributes': { - 'email': 'test@test.test', - }, - 'update_attributes': { - 'nickname': 't2', - 'height': 1.50, - 'age': 17, - 'alive': False, - } - }, - ], - }, - { - 'person': [ - { - 'prefix': 'p1', - 'attributes': { - 'email': 'test@test.test', - 'nickname': 't2', - 'height': 1.50, - 'age': 17, - 'alive': False, - } - }, - ], - }, - ), -]) -def test_update_attributes(typedb_interface, insert_dict, update_dict, r_dict): - query = typedb_interface.dict_to_query(insert_dict) - typedb_interface.insert_database('insert ' + query) - - r = typedb_interface.update_attributes_in_thing(update_dict) - query = typedb_interface.dict_to_query(r_dict) - match_result = typedb_interface.get_aggregate_database( - 'match ' + query + 'get; count;') - assert r is not None and r is not False and match_result > 0 - - -def test_get_query(typedb_interface): - query = """ - match - $p isa person, has full-name $name, has email $email; - get $name, $email; - sort $name asc; limit 3; - """ - result = typedb_interface.get_database(query) - name = result[0].get('name').as_attribute().get_value() - assert len(result) == 3 and name == 'Ahmed Frazier' - - -def test_fetch_query(typedb_interface): - query = """ - match - $company isa company, has name "TU Delft"; - fetch - employee_names: { - match - $e (employer: $company, employee: $employee) isa employment; - fetch - $employee: full-name, email; - $e: salary; - }; - """ - result = typedb_interface.fetch_database(query) - assert len(result) == 1 - assert len(result[0]['employee_names']) == 3 - - expected = [ - { - 'salary': 30000, - 'email': 'phd@tudelft.nl', - 'full-name': 'PhD candidate 1' - }, - { - 'salary': 999999, - 'email': 'boss@tudelft.nl', - 'full-name': 'Big Boss' - }, - { - 'salary': 0, - 'email': 'guest@tudelft.nl', - 'full-name': 'Random guest' - } - ] - - actual = [] - for emp in result[0]['employee_names']: - actual.append({ - 'salary': emp['e']['salary'][0]['value'], - 'email': emp['employee']['email'][0]['value'], - 'full-name': emp['employee']['full-name'][0]['value'] - }) - - # Compare as sets so order does not matter - assert {tuple(sorted(d.items())) for d in actual} == \ - {tuple(sorted(d.items())) for d in expected} - - -def test_register_method(typedb_interface): - def get_name_email(self, name): - query = f""" - match - $p isa person, has full-name '{name}'; - fetch - $p: email; - """ - return typedb_interface.fetch_database(query)[0]['p']['email'][0]['value'] - typedb_interface.register_method('get_name_email', get_name_email) - assert typedb_interface.get_name_email('Big Boss') == 'boss@tudelft.nl' - - def test_database_query_reconnects_after_failed_health_check(monkeypatch): class FakeDatabases: """Fake database collection with controllable health checks.""" From 24f66a913739ad050266d574e8539d0cb044b758 Mon Sep 17 00:00:00 2001 From: Gustavo Date: Thu, 25 Jun 2026 18:30:09 -0300 Subject: [PATCH 07/36] Add TypeDB lifecycle integration tests --- .../test/test_typedb_interface_integration.py | 129 +++++++++++++++++- 1 file changed, 125 insertions(+), 4 deletions(-) diff --git a/ros_typedb/test/test_typedb_interface_integration.py b/ros_typedb/test/test_typedb_interface_integration.py index 0adbbf2..4e1ab03 100644 --- a/ros_typedb/test/test_typedb_interface_integration.py +++ b/ros_typedb/test/test_typedb_interface_integration.py @@ -19,14 +19,18 @@ from ros_typedb.typedb_interface import TypeDBInterface +SCHEMA_PATH = 'test/typedb_test_data/schema.tql' +DATA_PATH = 'test/typedb_test_data/data.tql' + + @pytest.fixture def typedb_interface(): typedb_interface = TypeDBInterface( 'localhost:1729', 'test_database', force_database=True, - schema_path=['test/typedb_test_data/schema.tql'], - data_path=['test/typedb_test_data/data.tql'], + schema_path=[SCHEMA_PATH], + data_path=[DATA_PATH], force_data=True, sort_fetch_results=True ) @@ -49,6 +53,123 @@ def _fetch_attributes(typedb_interface): 'match $a isa attribute; get $a; count;') +def _count_person_by_email(typedb_interface, email): + return typedb_interface.get_aggregate_database( + f'match $p isa person, has email "{email}"; get $p; count;') + + +def _delete_database(database_name): + typedb_interface = TypeDBInterface('localhost:1729', database_name) + try: + typedb_interface.delete_database() + finally: + typedb_interface.driver.close() + + +def test_reload_schema_false_reuses_existing_database(): + """reload_schema=False skips schema reload while keeping existing data.""" + database_name = 'test_reload_schema_false_reuses_existing_database' + typedb_interface = None + reused_interface = None + try: + typedb_interface = TypeDBInterface( + 'localhost:1729', + database_name, + force_database=True, + schema_path=[SCHEMA_PATH], + data_path=[DATA_PATH], + force_data=True, + ) + assert _count_person_by_email(typedb_interface, 'boss@tudelft.nl') == 1 + typedb_interface.driver.close() + + reused_interface = TypeDBInterface( + 'localhost:1729', + database_name, + force_database=False, + force_data=False, + reload_schema=False, + schema_path=[SCHEMA_PATH], + ) + + assert _count_person_by_email(reused_interface, 'boss@tudelft.nl') == 1 + finally: + if reused_interface is not None: + reused_interface.driver.close() + elif typedb_interface is not None: + typedb_interface.driver.close() + _delete_database(database_name) + + +def test_missing_database_is_recreated_from_configured_files(): + """A missing configured database is recreated with schema and data files.""" + database_name = 'test_missing_database_recreate_from_files' + typedb_interface = None + try: + typedb_interface = TypeDBInterface( + 'localhost:1729', + database_name, + force_database=True, + schema_path=[SCHEMA_PATH], + data_path=[DATA_PATH], + force_data=True, + ) + assert _count_person_by_email(typedb_interface, 'boss@tudelft.nl') == 1 + + typedb_interface.delete_database() + assert not typedb_interface.driver.databases.contains(database_name) + + assert _count_person_by_email(typedb_interface, 'boss@tudelft.nl') == 1 + assert _fetch_entities(typedb_interface) > 0 + finally: + if typedb_interface is not None: + typedb_interface.delete_database() + typedb_interface.driver.close() + + +def test_force_data_refresh_removes_existing_data_before_loading_fixture(): + """force_data=True clears old data before loading configured data files.""" + database_name = 'test_force_data_refresh_removes_existing_data' + typedb_interface = None + refreshed_interface = None + try: + typedb_interface = TypeDBInterface( + 'localhost:1729', + database_name, + force_database=True, + schema_path=[SCHEMA_PATH], + data_path=[DATA_PATH], + force_data=True, + ) + typedb_interface.insert_entity( + 'person', [('email', 'extra_person@test.test')]) + assert _count_person_by_email( + typedb_interface, 'extra_person@test.test') == 1 + + refreshed_interface = TypeDBInterface( + 'localhost:1729', + database_name, + force_database=False, + force_data=True, + reload_schema=False, + schema_path=[SCHEMA_PATH], + data_path=[DATA_PATH], + ) + + assert _count_person_by_email( + refreshed_interface, 'extra_person@test.test') == 0 + assert _count_person_by_email( + refreshed_interface, 'boss@tudelft.nl') == 1 + finally: + if refreshed_interface is not None: + refreshed_interface.delete_database() + refreshed_interface.driver.close() + typedb_interface.driver.close() + elif typedb_interface is not None: + typedb_interface.delete_database() + typedb_interface.driver.close() + + def test_delete_all_data_removes_entities_relations_and_attributes( typedb_interface): """delete_all_data removes all live fixture data.""" @@ -98,8 +219,8 @@ def test_create_and_delete_database(): 'localhost:1729', 'test_database', force_database=True, - schema_path=['test/typedb_test_data/schema.tql'], - data_path=['test/typedb_test_data/data.tql'], + schema_path=[SCHEMA_PATH], + data_path=[DATA_PATH], force_data=True, ) From 61bbe80c8d60718c18508d1ddf7f79f108ec2c47 Mon Sep 17 00:00:00 2001 From: Gustavo Date: Thu, 25 Jun 2026 18:48:25 -0300 Subject: [PATCH 08/36] Split ROS TypeDB interface tests --- .../test_ros_typedb_interface_integration.py | 746 ++++++++++++++++++ ...e.py => test_ros_typedb_interface_unit.py} | 710 ----------------- 2 files changed, 746 insertions(+), 710 deletions(-) create mode 100644 ros_typedb/test/test_ros_typedb_interface_integration.py rename ros_typedb/test/{test_ros_typedb_interface.py => test_ros_typedb_interface_unit.py} (51%) diff --git a/ros_typedb/test/test_ros_typedb_interface_integration.py b/ros_typedb/test/test_ros_typedb_interface_integration.py new file mode 100644 index 0000000..1097423 --- /dev/null +++ b/ros_typedb/test/test_ros_typedb_interface_integration.py @@ -0,0 +1,746 @@ +# Copyright 2023 Gustavo Rezende Silva +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from pathlib import Path +import sys +from threading import Event +from threading import Thread + +import launch +import launch_pytest +import launch_ros + +from lifecycle_msgs.srv import ChangeState +from lifecycle_msgs.srv import GetState + +import pytest + +from rcl_interfaces.msg import ParameterType +import rclpy +from rclpy.node import Node + +from ros_typedb_msgs.msg import Attribute +from ros_typedb_msgs.msg import IndexList +from ros_typedb_msgs.msg import QueryResult +from ros_typedb_msgs.msg import ResultTree +from ros_typedb_msgs.msg import Thing +from ros_typedb_msgs.srv import Query + +from std_msgs.msg import String +from std_srvs.srv import Empty + + +@pytest.fixture +def insert_query(): + query_req = Query.Request() + query_req.query_type = query_req.INSERT + query_req.query = """ + insert + $person isa person, + has email 'test@test.com', + has nickname 'test', + has age 33, + has height 1.00, + has alive true, + has birth-date 1990-06-01; + $person2 isa person, + has email 'test2@test.com', + has nickname 'employer'; + $person3 isa person, + has email 'test3@test.com', + has nickname 'employer2'; + (employee:$person, employer:$person2) isa employment, + has salary 1000; + (employee:$person, employer:$person3) isa employment, + has salary 500; + """ + return query_req + + +@launch_pytest.fixture +def generate_test_description(): + path_to_test = Path(__file__).parents[1] + path_tql = path_to_test / 'test' / 'typedb_test_data' + + ros_typedb_node = launch_ros.actions.Node( + executable=sys.executable, + arguments=[ + str(path_to_test / 'ros_typedb' / 'ros_typedb_node.py')], + additional_env={'PYTHONUNBUFFERED': '1'}, + name='ros_typedb', + output='screen', + parameters=[{ + 'schema_path': [str(path_tql / 'schema.tql')], + 'data_path': [str(path_tql / 'data.tql')], + 'sort_fetch_results': True + }] + ) + + return launch.LaunchDescription([ + ros_typedb_node, + ]) + + +@pytest.fixture(scope='session', autouse=True) +def rclpy_runtime(): + if not rclpy.ok(): + rclpy.init() + try: + yield + finally: + if rclpy.ok(): + rclpy.shutdown() + + +@pytest.fixture +def test_node(): + node = MakeTestNode() + try: + yield node + finally: + node.delete_dabase() + node.destroy_node() + + +@launch_pytest.fixture +def generate_test_description_bad_data(): + path_to_test = Path(__file__).parents[1] + path_tql = path_to_test / 'test' / 'typedb_test_data' + + ros_typedb_node = launch_ros.actions.Node( + executable=sys.executable, + arguments=[ + str(path_to_test / 'ros_typedb' / 'ros_typedb_node.py')], + additional_env={'PYTHONUNBUFFERED': '1'}, + name='ros_typedb', + output='screen', + parameters=[{ + 'schema_path': [str(path_tql / 'schema.tql')], + 'data_path': [str(path_tql / 'bad_data.tql')], + 'database_name': 'ros_typedb_bad_data_test', + }] + ) + + return launch.LaunchDescription([ + ros_typedb_node, + ]) + + +@pytest.mark.launch(fixture=generate_test_description_bad_data) +def test_ros_typedb_configure_bad_data(test_node): + """Regression: configure must fail when data_path contains invalid TypeQL.""" + configure_res = test_node.change_ros_typedb_state(1) + assert configure_res.success is False + + +@pytest.mark.launch(fixture=generate_test_description) +def test_ros_typedb_lc_states(test_node): + configure_res = test_node.change_ros_typedb_state(1) + get_inactive_state_res = test_node.get_ros_typedb_state() + + activate_res = test_node.change_ros_typedb_state(3) + get_active_state_res = test_node.get_ros_typedb_state() + + assert configure_res.success is True and \ + get_inactive_state_res.current_state.id == 2 and \ + activate_res.success is True and \ + get_active_state_res.current_state.id == 3 + + +@pytest.mark.launch(fixture=generate_test_description) +def test_ros_typedb_cleanup_transition(test_node): + configure_res = test_node.change_ros_typedb_state(1) + activate_res = test_node.change_ros_typedb_state(3) + deactivate_res = test_node.change_ros_typedb_state(4) + cleanup_res = test_node.change_ros_typedb_state(2) + get_unconfigured_state_res = test_node.get_ros_typedb_state() + + assert configure_res.success is True + assert activate_res.success is True + assert deactivate_res.success is True + assert cleanup_res.success is True + assert get_unconfigured_state_res.current_state.id == 1 + + +@pytest.mark.launch(fixture=generate_test_description) +def test_ros_typedb_insert_query(test_node, insert_query): + test_node.activate_ros_typedb() + + query_res = test_node.call_service(test_node.query_cli, insert_query) + assert query_res.success is True + + +@pytest.mark.skip( + reason='Events deactivated') +@pytest.mark.launch(fixture=generate_test_description) +def test_ros_typedb_insert_event(insert_query): + node = MakeTestNode() + node.start_node() + node.activate_ros_typedb() + + node.call_service(node.query_cli, insert_query) + event_flag = node.typedb_event.wait(timeout=5.0) + assert event_flag and node.typedb_event_data == 'insert' + + +@pytest.mark.launch(fixture=generate_test_description) +def test_ros_typedb_delete_query(test_node, insert_query): + test_node.activate_ros_typedb() + + test_node.call_service(test_node.query_cli, insert_query) + + query_req = Query.Request() + query_req.query_type = query_req.DELETE + query_req.query = """ + match $entity isa person, has email 'test@test.com'; + delete $entity isa person; + """ + query_res = test_node.call_service(test_node.query_cli, query_req) + + match_query_req = Query.Request() + match_query_req.query_type = match_query_req.GET_AGGREGATE + match_query_req.query = """ + match $entity isa person, has email 'test@test.com'; + get $entity; + count; + """ + match_query_res = test_node.call_service( + test_node.query_cli, match_query_req) + + assert query_res.success + assert match_query_res.success + assert match_query_res.results[0].results[0].attribute.value.type == \ + ParameterType.PARAMETER_INTEGER + assert match_query_res.results[0].results[0].attribute.value.integer_value == 0 + + +@pytest.mark.skip( + reason='Events deactivated') +@pytest.mark.launch(fixture=generate_test_description) +def test_ros_typedb_delete_event(test_node, insert_query): + test_node.activate_ros_typedb() + + test_node.call_service(test_node.query_cli, insert_query) + + query_req = Query.Request() + query_req.query_type = query_req.DELETE + query_req.query = """ + match $entity isa person, has email 'test@test.com'; + delete $entity isa person; + """ + query_res = test_node.call_service(test_node.query_cli, query_req) + event_flag = test_node.typedb_event.wait(timeout=5.0) + assert event_flag and test_node.typedb_event_data == 'delete' + + assert query_res.success is True + + +@pytest.mark.launch(fixture=generate_test_description) +def test_ros_typedb_wrong_query(test_node, insert_query): + test_node.activate_ros_typedb() + + insert_query_req = insert_query + insert_query_req.query_type = 100 + query_res = test_node.call_service(test_node.query_cli, insert_query_req) + assert query_res.success is False + + +@pytest.mark.launch(fixture=generate_test_description) +def test_ros_typedb_fetch_query_attribute(test_node, insert_query): + test_node.activate_ros_typedb() + + test_node.call_service(test_node.query_cli, insert_query) + + query_req = Query.Request() + query_req.query_type = query_req.FETCH + query_req.query = """ + match $entity isa person, + has email 'test@test.com', + has nickname $nick, + has age $age, + has height $height, + has alive $alive, + has birth-date $date; + fetch $age; $alive; $date; $height; $nick; + """ + query_res = test_node.call_service(test_node.query_cli, query_req) + + expected_age_attr = Attribute() + expected_age_attr.variable_name = 'age' + expected_age_attr.label = 'age' + expected_age_attr.value.type = ParameterType.PARAMETER_INTEGER + expected_age_attr.value.integer_value = 33 + + expected_alive_attr = Attribute() + expected_alive_attr.variable_name = 'alive' + expected_alive_attr.label = 'alive' + expected_alive_attr.value.type = ParameterType.PARAMETER_BOOL + expected_alive_attr.value.bool_value = True + + expected_date_attr = Attribute() + expected_date_attr.variable_name = 'date' + expected_date_attr.label = 'birth-date' + expected_date_attr.value.type = ParameterType.PARAMETER_STRING + expected_date_attr.value.string_value = '1990-06-01T00:00:00.000' + + expected_height_attr = Attribute() + expected_height_attr.variable_name = 'height' + expected_height_attr.label = 'height' + expected_height_attr.value.type = ParameterType.PARAMETER_DOUBLE + expected_height_attr.value.double_value = 1.0 + + expected_nick_attr = Attribute() + expected_nick_attr.variable_name = 'nick' + expected_nick_attr.label = 'nickname' + expected_nick_attr.value.type = ParameterType.PARAMETER_STRING + expected_nick_attr.value.string_value = 'test' + + query_result_age = QueryResult() + query_result_age.type = QueryResult.ATTRIBUTE + query_result_age.result_index = 0 + query_result_age.attribute = expected_age_attr + + query_result_alive = QueryResult() + query_result_alive.type = QueryResult.ATTRIBUTE + query_result_alive.result_index = 1 + query_result_alive.attribute = expected_alive_attr + + query_result_date = QueryResult() + query_result_date.type = QueryResult.ATTRIBUTE + query_result_date.result_index = 2 + query_result_date.attribute = expected_date_attr + + query_result_height = QueryResult() + query_result_height.type = QueryResult.ATTRIBUTE + query_result_height.result_index = 3 + query_result_height.attribute = expected_height_attr + + query_result_nick = QueryResult() + query_result_nick.type = QueryResult.ATTRIBUTE + query_result_nick.result_index = 4 + query_result_nick.attribute = expected_nick_attr + + expected_tree = ResultTree() + expected_tree.results.append(query_result_age) + expected_tree.results.append(query_result_alive) + expected_tree.results.append(query_result_date) + expected_tree.results.append(query_result_height) + expected_tree.results.append(query_result_nick) + + assert query_res.success is True + assert len(query_res.results) == 1 + + assert query_result_age in query_res.results[0].results + assert query_result_alive in query_res.results[0].results + assert query_result_date in query_res.results[0].results + assert query_result_height in query_res.results[0].results + assert query_result_nick in query_res.results[0].results + + query_req = Query.Request() + query_req.query_type = query_req.FETCH + query_req.query = """ + match + $company_var isa company, has name 'TU Delft'; + fetch + $company_var: name, address; + employee_names: { + match + $employment_var (employer: $company_var, employee: $employee_var) + isa employment; + fetch + $employee_var: full-name, email; + $employment_var: salary; + }; + """ + + expected_tree = ResultTree() + + company_var_thing = Thing() + company_var_thing.type = Thing.ENTITY + company_var_thing.variable_name = 'company_var' + company_var_thing.type_name = 'company' + + address_attr = Attribute() + address_attr.label = 'address' + address_attr.variable_name = 'address' + address_attr.value.type = ParameterType.PARAMETER_STRING_ARRAY + address_attr.value.string_array_value = ['street 1', 'street 2'] + company_var_thing.attributes.append(address_attr) + + name_attr = Attribute() + name_attr.label = 'name' + name_attr.variable_name = 'name' + name_attr.value.type = ParameterType.PARAMETER_STRING_ARRAY + name_attr.value.string_array_value = ['TU Delft'] + company_var_thing.attributes.append(name_attr) + + company_var_query_result = QueryResult() + company_var_query_result.type = QueryResult.THING + company_var_query_result.result_index = 0 + company_var_query_result.thing = company_var_thing + + employee_names_subquery = QueryResult() + employee_names_subquery.type = QueryResult.SUB_QUERY + employee_names_subquery.result_index = 1 + employee_names_subquery.sub_query_name = 'employee_names' + + employee_var_thing1 = Thing() + employee_var_thing1.type = Thing.ENTITY + employee_var_thing1.variable_name = 'employee_var' + employee_var_thing1.type_name = 'person' + + email_attr = Attribute() + email_attr.label = 'email' + email_attr.variable_name = 'email' + email_attr.value.type = ParameterType.PARAMETER_STRING_ARRAY + email_attr.value.string_array_value = ['phd@tudelft.nl'] + employee_var_thing1.attributes.append(email_attr) + + full_name_attr = Attribute() + full_name_attr.label = 'full-name' + full_name_attr.variable_name = 'full-name' + full_name_attr.value.type = ParameterType.PARAMETER_STRING_ARRAY + full_name_attr.value.string_array_value = ['PhD candidate 1'] + employee_var_thing1.attributes.append(full_name_attr) + + employee_var_thing1_result = QueryResult() + employee_var_thing1_result.type = QueryResult.THING + employee_var_thing1_result.result_index = 2 + employee_var_thing1_result.thing = employee_var_thing1 + + employment_var_thing1 = Thing() + employment_var_thing1.type = Thing.RELATION + employment_var_thing1.variable_name = 'employment_var' + employment_var_thing1.type_name = 'employment' + + salary_attr = Attribute() + salary_attr.label = 'salary' + salary_attr.variable_name = 'salary' + salary_attr.value.type = ParameterType.PARAMETER_INTEGER_ARRAY + salary_attr.value.integer_array_value = [30000] + employment_var_thing1.attributes.append(salary_attr) + + employment_var_thing1_result = QueryResult() + employment_var_thing1_result.type = QueryResult.THING + employment_var_thing1_result.result_index = 3 + employment_var_thing1_result.thing = employment_var_thing1 + + list_index = IndexList() + list_index.index.append(2) + list_index.index.append(3) + employee_names_subquery.children_index.append(list_index) + + # Subtree 2 + employee_var_thing2 = Thing() + employee_var_thing2.type = Thing.ENTITY + employee_var_thing2.variable_name = 'employee_var' + employee_var_thing2.type_name = 'person' + + email_attr = Attribute() + email_attr.label = 'email' + email_attr.variable_name = 'email' + email_attr.value.type = ParameterType.PARAMETER_STRING_ARRAY + email_attr.value.string_array_value = ['boss@tudelft.nl'] + employee_var_thing2.attributes.append(email_attr) + + full_name_attr = Attribute() + full_name_attr.label = 'full-name' + full_name_attr.variable_name = 'full-name' + full_name_attr.value.type = ParameterType.PARAMETER_STRING_ARRAY + full_name_attr.value.string_array_value = ['Big Boss'] + employee_var_thing2.attributes.append(full_name_attr) + + employee_var_thing2_result = QueryResult() + employee_var_thing2_result.type = QueryResult.THING + employee_var_thing2_result.result_index = 4 + employee_var_thing2_result.thing = employee_var_thing2 + + employment_var_thing2 = Thing() + employment_var_thing2.type = Thing.RELATION + employment_var_thing2.variable_name = 'employment_var' + employment_var_thing2.type_name = 'employment' + + salary_attr = Attribute() + salary_attr.label = 'salary' + salary_attr.variable_name = 'salary' + salary_attr.value.type = ParameterType.PARAMETER_INTEGER_ARRAY + salary_attr.value.integer_array_value = [999999] + employment_var_thing2.attributes.append(salary_attr) + + employment_var_thing2_result = QueryResult() + employment_var_thing2_result.type = QueryResult.THING + employment_var_thing2_result.result_index = 5 + employment_var_thing2_result.thing = employment_var_thing2 + + list_index = IndexList() + list_index.index.append(4) + list_index.index.append(5) + employee_names_subquery.children_index.append(list_index) + + # Subtree 3 + employee_var_thing3 = Thing() + employee_var_thing3.type = Thing.ENTITY + employee_var_thing3.variable_name = 'employee_var' + employee_var_thing3.type_name = 'person' + + email_attr = Attribute() + email_attr.label = 'email' + email_attr.variable_name = 'email' + email_attr.value.type = ParameterType.PARAMETER_STRING_ARRAY + email_attr.value.string_array_value = ['guest@tudelft.nl'] + employee_var_thing3.attributes.append(email_attr) + + full_name_attr = Attribute() + full_name_attr.label = 'full-name' + full_name_attr.variable_name = 'full-name' + full_name_attr.value.type = ParameterType.PARAMETER_STRING_ARRAY + full_name_attr.value.string_array_value = ['Random guest'] + employee_var_thing3.attributes.append(full_name_attr) + + employee_var_thing3_result = QueryResult() + employee_var_thing3_result.type = QueryResult.THING + employee_var_thing3_result.result_index = 6 + employee_var_thing3_result.thing = employee_var_thing3 + + employment_var_thing3 = Thing() + employment_var_thing3.type = Thing.RELATION + employment_var_thing3.variable_name = 'employment_var' + employment_var_thing3.type_name = 'employment' + + salary_attr = Attribute() + salary_attr.label = 'salary' + salary_attr.variable_name = 'salary' + salary_attr.value.type = ParameterType.PARAMETER_INTEGER_ARRAY + salary_attr.value.integer_array_value = [0] + employment_var_thing3.attributes.append(salary_attr) + + employment_var_thing3_result = QueryResult() + employment_var_thing3_result.type = QueryResult.THING + employment_var_thing3_result.result_index = 7 + employment_var_thing3_result.thing = employment_var_thing3 + + list_index = IndexList() + list_index.index.append(6) + list_index.index.append(7) + employee_names_subquery.children_index.append(list_index) + + expected_tree.results.append(company_var_query_result) + + expected_tree.results.append(employee_names_subquery) + + expected_tree.results.append(employee_var_thing1_result) + expected_tree.results.append(employment_var_thing1_result) + + expected_tree.results.append(employee_var_thing2_result) + expected_tree.results.append(employment_var_thing2_result) + + expected_tree.results.append(employee_var_thing3_result) + expected_tree.results.append(employment_var_thing3_result) + + query_res = test_node.call_service(test_node.query_cli, query_req) + + assert query_res.success is True + assert len(query_res.results) == 1 + + expected = [company_var_query_result, employee_names_subquery, + employee_var_thing1_result, employment_var_thing1_result, + employee_var_thing2_result, employment_var_thing2_result, + employee_var_thing3_result, employment_var_thing3_result] + + actual = [] + for result in query_res.results[0].results: + actual.append(result) + + import copy + + def strip_indices(obj): + # Make a shallow copy to avoid mutating the original + obj_copy = copy.copy(obj) + if hasattr(obj_copy, 'result_index'): + obj_copy.result_index = 0 + if hasattr(obj_copy, 'children_index'): + obj_copy.children_index = [IndexList()] + return obj_copy + + # Compare expected and actual ignoring result_index and children_index + for item in expected: + assert any(strip_indices(item) == strip_indices(a) for a in actual) + + +@pytest.mark.launch(fixture=generate_test_description) +def test_ros_typedb_get_query(test_node, insert_query): + test_node.activate_ros_typedb() + + test_node.call_service(test_node.query_cli, insert_query) + + query_req = Query.Request() + query_req.query_type = query_req.GET + query_req.query = """ + match + $p isa person, has full-name $name, has email $email; + get $name, $email; + sort $name asc; limit 3; + """ + query_res = test_node.call_service(test_node.query_cli, query_req) + + assert query_res.success is True + assert len(query_res.results) == 3 + assert len(query_res.results[0].results) == 2 + assert len(query_res.results[1].results) == 2 + assert len(query_res.results[2].results) == 2 + + correct_name = False + correct_email = False + for r in query_res.results[0].results: + if r.attribute.variable_name == 'name' and r.attribute.label == 'full-name' \ + and r.attribute.value.string_value == 'Ahmed Frazier': + correct_name = True + if r.attribute.variable_name == 'email' and r.attribute.label == 'email' and \ + r.attribute.value.string_value == 'ahmed.frazier@gmail.com': + correct_email = True + assert correct_name and correct_email + + correct_name = False + correct_email = False + for r in query_res.results[1].results: + if r.attribute.variable_name == 'name' and r.attribute.label == 'full-name' \ + and r.attribute.value.string_value == 'Big Boss': + correct_name = True + if r.attribute.variable_name == 'email' and r.attribute.label == 'email' and \ + r.attribute.value.string_value == 'boss@tudelft.nl': + correct_email = True + assert correct_name and correct_email + + correct_name = False + correct_email = False + for r in query_res.results[2].results: + if r.attribute.variable_name == 'name' and r.attribute.label == 'full-name' \ + and r.attribute.value.string_value == 'Dominic Lyons': + correct_name = True + if r.attribute.variable_name == 'email' and r.attribute.label == 'email' and \ + r.attribute.value.string_value == 'dominic.lyons@gmail.com': + correct_email = True + assert correct_name and correct_email + + +@pytest.mark.launch(fixture=generate_test_description) +def test_ros_typedb_get_aggregate_query(test_node, insert_query): + test_node.activate_ros_typedb() + + test_node.call_service(test_node.query_cli, insert_query) + + query_req = Query.Request() + query_req.query_type = query_req.GET_AGGREGATE + query_req.query = """ + match + $person isa person, + has email 'test@test.com'; + (employee:$person) isa employment, has salary $s; + get $s; sum $s; + """ + query_res = test_node.call_service(test_node.query_cli, query_req) + + assert query_res.success is True + assert query_res.results[0].results[0].type == QueryResult.ATTRIBUTE + assert query_res.results[0].results[0].attribute.value.integer_value == 1500 + + +@pytest.mark.launch(fixture=generate_test_description) +def test_ros_typedb_update_query(test_node, insert_query): + test_node.activate_ros_typedb() + + test_node.call_service(test_node.query_cli, insert_query) + + query_req = Query.Request() + query_req.query_type = query_req.UPDATE + query_req.query = """ + match + $person isa person, has email $email; + $email = 'test@test.com'; + delete + $person has $email; + insert + $person has email 'updatedemail@test.com'; + """ + query_res = test_node.call_service(test_node.query_cli, query_req) + + assert query_res.success is True + # Potential TODO: add get to make sure the email was updated + + +class MakeTestNode(Node): + + def __init__(self, name='test_node'): + super().__init__(name) + self.typedb_event = Event() + self.typedb_event_data = None + + self.change_state_cli = self.create_client( + ChangeState, '/ros_typedb/change_state' + ) + + self.get_state_cli = self.create_client( + GetState, '/ros_typedb/get_state' + ) + + self.query_cli = self.create_client( + Query, '/ros_typedb/query' + ) + + self.delete_db_cli = self.create_client( + Empty, '/ros_typedb/delete_database' + ) + + self.event_sub = self.create_subscription( + String, '/ros_typedb/events', self.event_sub_cb, 10) + + def start_node(self): + self.ros_spin_thread = Thread( + target=lambda node: rclpy.spin(node), + args=(self,)) + self.ros_spin_thread.start() + + def change_ros_typedb_state(self, transition_id): + change_state_req = ChangeState.Request() + change_state_req.transition.id = transition_id + return self.call_service(self.change_state_cli, change_state_req) + + def get_ros_typedb_state(self): + get_state_req = GetState.Request() + return self.call_service(self.get_state_cli, get_state_req) + + def call_service(self, cli, request): + if cli.wait_for_service(timeout_sec=5.0) is False: + self.get_logger().error( + 'service not available {}'.format(cli.srv_name)) + return None + future = cli.call_async(request) + rclpy.spin_until_future_complete(self, future, timeout_sec=5.0) + if future.done() is False: + self.get_logger().error( + 'Future not completed {}'.format(cli.srv_name)) + return None + + return future.result() + + def activate_ros_typedb(self): + self.change_ros_typedb_state(1) + self.change_ros_typedb_state(3) + + def event_sub_cb(self, msg): + self.typedb_event_data = msg.data + self.typedb_event.set() + + def delete_dabase(self): + self.call_service(self.delete_db_cli, Empty.Request()) diff --git a/ros_typedb/test/test_ros_typedb_interface.py b/ros_typedb/test/test_ros_typedb_interface_unit.py similarity index 51% rename from ros_typedb/test/test_ros_typedb_interface.py rename to ros_typedb/test/test_ros_typedb_interface_unit.py index d561361..4b40727 100644 --- a/ros_typedb/test/test_ros_typedb_interface.py +++ b/ros_typedb/test/test_ros_typedb_interface_unit.py @@ -12,26 +12,11 @@ # See the License for the specific language governing permissions and # limitations under the License. -from pathlib import Path -import sys -from threading import Event -from threading import Thread from types import SimpleNamespace from unittest.mock import MagicMock -import launch -import launch_pytest -import launch_ros - -from lifecycle_msgs.srv import ChangeState -from lifecycle_msgs.srv import GetState - -import pytest - from rcl_interfaces.msg import ParameterType -import rclpy from rclpy.lifecycle import TransitionCallbackReturn -from rclpy.node import Node from ros_typedb.ros_typedb_interface import convert_attribute_dict_to_ros_msg from ros_typedb.ros_typedb_interface import fetch_result_to_ros_result_tree @@ -44,36 +29,6 @@ from ros_typedb_msgs.msg import Thing from ros_typedb_msgs.srv import Query -from std_msgs.msg import String -from std_srvs.srv import Empty - - -@pytest.fixture -def insert_query(): - query_req = Query.Request() - query_req.query_type = query_req.INSERT - query_req.query = """ - insert - $person isa person, - has email 'test@test.com', - has nickname 'test', - has age 33, - has height 1.00, - has alive true, - has birth-date 1990-06-01; - $person2 isa person, - has email 'test2@test.com', - has nickname 'employer'; - $person3 isa person, - has email 'test3@test.com', - has nickname 'employer2'; - (employee:$person, employer:$person2) isa employment, - has salary 1000; - (employee:$person, employer:$person3) isa employment, - has salary 500; - """ - return query_req - def test_query_srv_has_timeout_s_field(): """Query.Request must have a timeout_s field defaulting to 0.0.""" @@ -174,51 +129,6 @@ def fetch_database(self, query, timeout=None): assert captured.get('timeout') is None -@launch_pytest.fixture -def generate_test_description(): - path_to_test = Path(__file__).parents[1] - path_tql = path_to_test / 'test' / 'typedb_test_data' - - ros_typedb_node = launch_ros.actions.Node( - executable=sys.executable, - arguments=[ - str(path_to_test / 'ros_typedb' / 'ros_typedb_node.py')], - additional_env={'PYTHONUNBUFFERED': '1'}, - name='ros_typedb', - output='screen', - parameters=[{ - 'schema_path': [str(path_tql / 'schema.tql')], - 'data_path': [str(path_tql / 'data.tql')], - 'sort_fetch_results': True - }] - ) - - return launch.LaunchDescription([ - ros_typedb_node, - ]) - - -@pytest.fixture(scope='session', autouse=True) -def rclpy_runtime(): - if not rclpy.ok(): - rclpy.init() - try: - yield - finally: - if rclpy.ok(): - rclpy.shutdown() - - -@pytest.fixture -def test_node(): - node = MakeTestNode() - try: - yield node - finally: - node.delete_dabase() - node.destroy_node() - - class MockTypeDBDriver: def __init__(self): @@ -245,37 +155,6 @@ def test_close_typedb_interface_closes_driver_and_clears_reference(): assert ros_typedb_interface.typedb_interface is None -@launch_pytest.fixture -def generate_test_description_bad_data(): - path_to_test = Path(__file__).parents[1] - path_tql = path_to_test / 'test' / 'typedb_test_data' - - ros_typedb_node = launch_ros.actions.Node( - executable=sys.executable, - arguments=[ - str(path_to_test / 'ros_typedb' / 'ros_typedb_node.py')], - additional_env={'PYTHONUNBUFFERED': '1'}, - name='ros_typedb', - output='screen', - parameters=[{ - 'schema_path': [str(path_tql / 'schema.tql')], - 'data_path': [str(path_tql / 'bad_data.tql')], - 'database_name': 'ros_typedb_bad_data_test', - }] - ) - - return launch.LaunchDescription([ - ros_typedb_node, - ]) - - -@pytest.mark.launch(fixture=generate_test_description_bad_data) -def test_ros_typedb_configure_bad_data(test_node): - """Regression: configure must fail when data_path contains invalid TypeQL.""" - configure_res = test_node.change_ros_typedb_state(1) - assert configure_res.success is False - - def test_fetch_result_to_ros_result_tree_accepts_more_than_uint8_indices(): json_test = { f'attr_{index}': { @@ -299,103 +178,6 @@ def test_fetch_result_to_ros_result_tree_accepts_more_than_uint8_indices(): assert index_list.index[256] == 256 -@pytest.mark.launch(fixture=generate_test_description) -def test_ros_typedb_lc_states(test_node): - configure_res = test_node.change_ros_typedb_state(1) - get_inactive_state_res = test_node.get_ros_typedb_state() - - activate_res = test_node.change_ros_typedb_state(3) - get_active_state_res = test_node.get_ros_typedb_state() - - assert configure_res.success is True and \ - get_inactive_state_res.current_state.id == 2 and \ - activate_res.success is True and \ - get_active_state_res.current_state.id == 3 - - -@pytest.mark.launch(fixture=generate_test_description) -def test_ros_typedb_insert_query(test_node, insert_query): - test_node.activate_ros_typedb() - - query_res = test_node.call_service(test_node.query_cli, insert_query) - assert query_res.success is True - - -@pytest.mark.skip( - reason='Events deactivated') -@pytest.mark.launch(fixture=generate_test_description) -def test_ros_typedb_insert_event(insert_query): - node = MakeTestNode() - node.start_node() - node.activate_ros_typedb() - - node.call_service(node.query_cli, insert_query) - event_flag = node.typedb_event.wait(timeout=5.0) - assert event_flag and node.typedb_event_data == 'insert' - - -@pytest.mark.launch(fixture=generate_test_description) -def test_ros_typedb_delete_query(test_node, insert_query): - test_node.activate_ros_typedb() - - test_node.call_service(test_node.query_cli, insert_query) - - query_req = Query.Request() - query_req.query_type = query_req.DELETE - query_req.query = """ - match $entity isa person, has email 'test@test.com'; - delete $entity isa person; - """ - query_res = test_node.call_service(test_node.query_cli, query_req) - - match_query_req = Query.Request() - match_query_req.query_type = match_query_req.GET_AGGREGATE - match_query_req.query = """ - match $entity isa person, has email 'test@test.com'; - get $entity; - count; - """ - match_query_res = test_node.call_service( - test_node.query_cli, match_query_req) - - assert query_res.success - assert match_query_res.success - assert match_query_res.results[0].results[0].attribute.value.type == \ - ParameterType.PARAMETER_INTEGER - assert match_query_res.results[0].results[0].attribute.value.integer_value == 0 - - -@pytest.mark.skip( - reason='Events deactivated') -@pytest.mark.launch(fixture=generate_test_description) -def test_ros_typedb_delete_event(test_node, insert_query): - test_node.activate_ros_typedb() - - test_node.call_service(test_node.query_cli, insert_query) - - query_req = Query.Request() - query_req.query_type = query_req.DELETE - query_req.query = """ - match $entity isa person, has email 'test@test.com'; - delete $entity isa person; - """ - query_res = test_node.call_service(test_node.query_cli, query_req) - event_flag = test_node.typedb_event.wait(timeout=5.0) - assert event_flag and test_node.typedb_event_data == 'delete' - - assert query_res.success is True - - -@pytest.mark.launch(fixture=generate_test_description) -def test_ros_typedb_wrong_query(test_node, insert_query): - test_node.activate_ros_typedb() - - insert_query_req = insert_query - insert_query_req.query_type = 100 - query_res = test_node.call_service(test_node.query_cli, insert_query_req) - assert query_res.success is False - - def test_close_typedb_interface_closes_driver_and_clears_interface(): driver = MagicMock() node = SimpleNamespace(typedb_interface=SimpleNamespace(driver=driver)) @@ -667,8 +449,6 @@ def test_fetch_result_to_ros_result_tree(): assert result_tree.results[3] == query_result_height assert result_tree.results[4] == query_result_nick - assert expected_tree == result_tree - json_test = { 'company_var': { 'address': [ @@ -946,493 +726,3 @@ def test_fetch_result_to_ros_result_tree(): assert result_tree.results[6] == employee_var_thing3_result assert result_tree.results[7] == employment_var_thing3_result assert expected_tree == result_tree - - -@pytest.mark.launch(fixture=generate_test_description) -def test_ros_typedb_fetch_query_attribute(test_node, insert_query): - test_node.activate_ros_typedb() - - test_node.call_service(test_node.query_cli, insert_query) - - query_req = Query.Request() - query_req.query_type = query_req.FETCH - query_req.query = """ - match $entity isa person, - has email 'test@test.com', - has nickname $nick, - has age $age, - has height $height, - has alive $alive, - has birth-date $date; - fetch $age; $alive; $date; $height; $nick; - """ - query_res = test_node.call_service(test_node.query_cli, query_req) - - expected_age_attr = Attribute() - expected_age_attr.variable_name = 'age' - expected_age_attr.label = 'age' - expected_age_attr.value.type = ParameterType.PARAMETER_INTEGER - expected_age_attr.value.integer_value = 33 - - expected_alive_attr = Attribute() - expected_alive_attr.variable_name = 'alive' - expected_alive_attr.label = 'alive' - expected_alive_attr.value.type = ParameterType.PARAMETER_BOOL - expected_alive_attr.value.bool_value = True - - expected_date_attr = Attribute() - expected_date_attr.variable_name = 'date' - expected_date_attr.label = 'birth-date' - expected_date_attr.value.type = ParameterType.PARAMETER_STRING - expected_date_attr.value.string_value = '1990-06-01T00:00:00.000' - - expected_height_attr = Attribute() - expected_height_attr.variable_name = 'height' - expected_height_attr.label = 'height' - expected_height_attr.value.type = ParameterType.PARAMETER_DOUBLE - expected_height_attr.value.double_value = 1.0 - - expected_nick_attr = Attribute() - expected_nick_attr.variable_name = 'nick' - expected_nick_attr.label = 'nickname' - expected_nick_attr.value.type = ParameterType.PARAMETER_STRING - expected_nick_attr.value.string_value = 'test' - - query_result_age = QueryResult() - query_result_age.type = QueryResult.ATTRIBUTE - query_result_age.result_index = 0 - query_result_age.attribute = expected_age_attr - - query_result_alive = QueryResult() - query_result_alive.type = QueryResult.ATTRIBUTE - query_result_alive.result_index = 1 - query_result_alive.attribute = expected_alive_attr - - query_result_date = QueryResult() - query_result_date.type = QueryResult.ATTRIBUTE - query_result_date.result_index = 2 - query_result_date.attribute = expected_date_attr - - query_result_height = QueryResult() - query_result_height.type = QueryResult.ATTRIBUTE - query_result_height.result_index = 3 - query_result_height.attribute = expected_height_attr - - query_result_nick = QueryResult() - query_result_nick.type = QueryResult.ATTRIBUTE - query_result_nick.result_index = 4 - query_result_nick.attribute = expected_nick_attr - - expected_tree = ResultTree() - expected_tree.results.append(query_result_age) - expected_tree.results.append(query_result_alive) - expected_tree.results.append(query_result_date) - expected_tree.results.append(query_result_height) - expected_tree.results.append(query_result_nick) - - assert query_res.success is True - assert len(query_res.results) == 1 - - assert query_result_age in query_res.results[0].results - assert query_result_alive in query_res.results[0].results - assert query_result_date in query_res.results[0].results - assert query_result_height in query_res.results[0].results - assert query_result_nick in query_res.results[0].results - - query_req = Query.Request() - query_req.query_type = query_req.FETCH - query_req.query = """ - match - $company_var isa company, has name 'TU Delft'; - fetch - $company_var: name, address; - employee_names: { - match - $employment_var (employer: $company_var, employee: $employee_var) - isa employment; - fetch - $employee_var: full-name, email; - $employment_var: salary; - }; - """ - - expected_tree = ResultTree() - - company_var_thing = Thing() - company_var_thing.type = Thing.ENTITY - company_var_thing.variable_name = 'company_var' - company_var_thing.type_name = 'company' - - address_attr = Attribute() - address_attr.label = 'address' - address_attr.variable_name = 'address' - address_attr.value.type = ParameterType.PARAMETER_STRING_ARRAY - address_attr.value.string_array_value = ['street 1', 'street 2'] - company_var_thing.attributes.append(address_attr) - - name_attr = Attribute() - name_attr.label = 'name' - name_attr.variable_name = 'name' - name_attr.value.type = ParameterType.PARAMETER_STRING_ARRAY - name_attr.value.string_array_value = ['TU Delft'] - company_var_thing.attributes.append(name_attr) - - company_var_query_result = QueryResult() - company_var_query_result.type = QueryResult.THING - company_var_query_result.result_index = 0 - company_var_query_result.thing = company_var_thing - - employee_names_subquery = QueryResult() - employee_names_subquery.type = QueryResult.SUB_QUERY - employee_names_subquery.result_index = 1 - employee_names_subquery.sub_query_name = 'employee_names' - - employee_var_thing1 = Thing() - employee_var_thing1.type = Thing.ENTITY - employee_var_thing1.variable_name = 'employee_var' - employee_var_thing1.type_name = 'person' - - email_attr = Attribute() - email_attr.label = 'email' - email_attr.variable_name = 'email' - email_attr.value.type = ParameterType.PARAMETER_STRING_ARRAY - email_attr.value.string_array_value = ['phd@tudelft.nl'] - employee_var_thing1.attributes.append(email_attr) - - full_name_attr = Attribute() - full_name_attr.label = 'full-name' - full_name_attr.variable_name = 'full-name' - full_name_attr.value.type = ParameterType.PARAMETER_STRING_ARRAY - full_name_attr.value.string_array_value = ['PhD candidate 1'] - employee_var_thing1.attributes.append(full_name_attr) - - employee_var_thing1_result = QueryResult() - employee_var_thing1_result.type = QueryResult.THING - employee_var_thing1_result.result_index = 2 - employee_var_thing1_result.thing = employee_var_thing1 - - employment_var_thing1 = Thing() - employment_var_thing1.type = Thing.RELATION - employment_var_thing1.variable_name = 'employment_var' - employment_var_thing1.type_name = 'employment' - - salary_attr = Attribute() - salary_attr.label = 'salary' - salary_attr.variable_name = 'salary' - salary_attr.value.type = ParameterType.PARAMETER_INTEGER_ARRAY - salary_attr.value.integer_array_value = [30000] - employment_var_thing1.attributes.append(salary_attr) - - employment_var_thing1_result = QueryResult() - employment_var_thing1_result.type = QueryResult.THING - employment_var_thing1_result.result_index = 3 - employment_var_thing1_result.thing = employment_var_thing1 - - list_index = IndexList() - list_index.index.append(2) - list_index.index.append(3) - employee_names_subquery.children_index.append(list_index) - - # Subtree 2 - employee_var_thing2 = Thing() - employee_var_thing2.type = Thing.ENTITY - employee_var_thing2.variable_name = 'employee_var' - employee_var_thing2.type_name = 'person' - - email_attr = Attribute() - email_attr.label = 'email' - email_attr.variable_name = 'email' - email_attr.value.type = ParameterType.PARAMETER_STRING_ARRAY - email_attr.value.string_array_value = ['boss@tudelft.nl'] - employee_var_thing2.attributes.append(email_attr) - - full_name_attr = Attribute() - full_name_attr.label = 'full-name' - full_name_attr.variable_name = 'full-name' - full_name_attr.value.type = ParameterType.PARAMETER_STRING_ARRAY - full_name_attr.value.string_array_value = ['Big Boss'] - employee_var_thing2.attributes.append(full_name_attr) - - employee_var_thing2_result = QueryResult() - employee_var_thing2_result.type = QueryResult.THING - employee_var_thing2_result.result_index = 4 - employee_var_thing2_result.thing = employee_var_thing2 - - employment_var_thing2 = Thing() - employment_var_thing2.type = Thing.RELATION - employment_var_thing2.variable_name = 'employment_var' - employment_var_thing2.type_name = 'employment' - - salary_attr = Attribute() - salary_attr.label = 'salary' - salary_attr.variable_name = 'salary' - salary_attr.value.type = ParameterType.PARAMETER_INTEGER_ARRAY - salary_attr.value.integer_array_value = [999999] - employment_var_thing2.attributes.append(salary_attr) - - employment_var_thing2_result = QueryResult() - employment_var_thing2_result.type = QueryResult.THING - employment_var_thing2_result.result_index = 5 - employment_var_thing2_result.thing = employment_var_thing2 - - list_index = IndexList() - list_index.index.append(4) - list_index.index.append(5) - employee_names_subquery.children_index.append(list_index) - - # Subtree 3 - employee_var_thing3 = Thing() - employee_var_thing3.type = Thing.ENTITY - employee_var_thing3.variable_name = 'employee_var' - employee_var_thing3.type_name = 'person' - - email_attr = Attribute() - email_attr.label = 'email' - email_attr.variable_name = 'email' - email_attr.value.type = ParameterType.PARAMETER_STRING_ARRAY - email_attr.value.string_array_value = ['guest@tudelft.nl'] - employee_var_thing3.attributes.append(email_attr) - - full_name_attr = Attribute() - full_name_attr.label = 'full-name' - full_name_attr.variable_name = 'full-name' - full_name_attr.value.type = ParameterType.PARAMETER_STRING_ARRAY - full_name_attr.value.string_array_value = ['Random guest'] - employee_var_thing3.attributes.append(full_name_attr) - - employee_var_thing3_result = QueryResult() - employee_var_thing3_result.type = QueryResult.THING - employee_var_thing3_result.result_index = 6 - employee_var_thing3_result.thing = employee_var_thing3 - - employment_var_thing3 = Thing() - employment_var_thing3.type = Thing.RELATION - employment_var_thing3.variable_name = 'employment_var' - employment_var_thing3.type_name = 'employment' - - salary_attr = Attribute() - salary_attr.label = 'salary' - salary_attr.variable_name = 'salary' - salary_attr.value.type = ParameterType.PARAMETER_INTEGER_ARRAY - salary_attr.value.integer_array_value = [0] - employment_var_thing3.attributes.append(salary_attr) - - employment_var_thing3_result = QueryResult() - employment_var_thing3_result.type = QueryResult.THING - employment_var_thing3_result.result_index = 7 - employment_var_thing3_result.thing = employment_var_thing3 - - list_index = IndexList() - list_index.index.append(6) - list_index.index.append(7) - employee_names_subquery.children_index.append(list_index) - - expected_tree.results.append(company_var_query_result) - - expected_tree.results.append(employee_names_subquery) - - expected_tree.results.append(employee_var_thing1_result) - expected_tree.results.append(employment_var_thing1_result) - - expected_tree.results.append(employee_var_thing2_result) - expected_tree.results.append(employment_var_thing2_result) - - expected_tree.results.append(employee_var_thing3_result) - expected_tree.results.append(employment_var_thing3_result) - - query_res = test_node.call_service(test_node.query_cli, query_req) - - assert query_res.success is True - assert len(query_res.results) == 1 - - expected = [company_var_query_result, employee_names_subquery, - employee_var_thing1_result, employment_var_thing1_result, - employee_var_thing2_result, employment_var_thing2_result, - employee_var_thing3_result, employment_var_thing3_result] - - actual = [] - for result in query_res.results[0].results: - actual.append(result) - - import copy - - def strip_indices(obj): - # Make a shallow copy to avoid mutating the original - obj_copy = copy.copy(obj) - if hasattr(obj_copy, 'result_index'): - obj_copy.result_index = 0 - if hasattr(obj_copy, 'children_index'): - obj_copy.children_index = [IndexList()] - return obj_copy - - # Compare expected and actual ignoring result_index and children_index - for item in expected: - assert any(strip_indices(item) == strip_indices(a) for a in actual) - - -@pytest.mark.launch(fixture=generate_test_description) -def test_ros_typedb_get_query(test_node, insert_query): - test_node.activate_ros_typedb() - - test_node.call_service(test_node.query_cli, insert_query) - - query_req = Query.Request() - query_req.query_type = query_req.GET - query_req.query = """ - match - $p isa person, has full-name $name, has email $email; - get $name, $email; - sort $name asc; limit 3; - """ - query_res = test_node.call_service(test_node.query_cli, query_req) - - assert query_res.success is True - assert len(query_res.results) == 3 - assert len(query_res.results[0].results) == 2 - assert len(query_res.results[1].results) == 2 - assert len(query_res.results[2].results) == 2 - - correct_name = False - correct_email = False - for r in query_res.results[0].results: - if r.attribute.variable_name == 'name' and r.attribute.label == 'full-name' \ - and r.attribute.value.string_value == 'Ahmed Frazier': - correct_name = True - if r.attribute.variable_name == 'email' and r.attribute.label == 'email' and \ - r.attribute.value.string_value == 'ahmed.frazier@gmail.com': - correct_email = True - assert correct_name and correct_email - - correct_name = False - correct_email = False - for r in query_res.results[1].results: - if r.attribute.variable_name == 'name' and r.attribute.label == 'full-name' \ - and r.attribute.value.string_value == 'Big Boss': - correct_name = True - if r.attribute.variable_name == 'email' and r.attribute.label == 'email' and \ - r.attribute.value.string_value == 'boss@tudelft.nl': - correct_email = True - assert correct_name and correct_email - - correct_name = False - correct_email = False - for r in query_res.results[2].results: - if r.attribute.variable_name == 'name' and r.attribute.label == 'full-name' \ - and r.attribute.value.string_value == 'Dominic Lyons': - correct_name = True - if r.attribute.variable_name == 'email' and r.attribute.label == 'email' and \ - r.attribute.value.string_value == 'dominic.lyons@gmail.com': - correct_email = True - assert correct_name and correct_email - - -@pytest.mark.launch(fixture=generate_test_description) -def test_ros_typedb_get_aggregate_query(test_node, insert_query): - test_node.activate_ros_typedb() - - test_node.call_service(test_node.query_cli, insert_query) - - query_req = Query.Request() - query_req.query_type = query_req.GET_AGGREGATE - query_req.query = """ - match - $person isa person, - has email 'test@test.com'; - (employee:$person) isa employment, has salary $s; - get $s; sum $s; - """ - query_res = test_node.call_service(test_node.query_cli, query_req) - - assert query_res.success is True - assert query_res.results[0].results[0].type == QueryResult.ATTRIBUTE - assert query_res.results[0].results[0].attribute.value.integer_value == 1500 - - -@pytest.mark.launch(fixture=generate_test_description) -def test_ros_typedb_update_query(test_node, insert_query): - test_node.activate_ros_typedb() - - test_node.call_service(test_node.query_cli, insert_query) - - query_req = Query.Request() - query_req.query_type = query_req.UPDATE - query_req.query = """ - match - $person isa person, has email $email; - $email = 'test@test.com'; - delete - $person has $email; - insert - $person has email 'updatedemail@test.com'; - """ - query_res = test_node.call_service(test_node.query_cli, query_req) - - assert query_res.success is True - # Potential TODO: add get to make sure the email was updated - - -class MakeTestNode(Node): - - def __init__(self, name='test_node'): - super().__init__(name) - self.typedb_event = Event() - self.typedb_event_data = None - - self.change_state_cli = self.create_client( - ChangeState, '/ros_typedb/change_state' - ) - - self.get_state_cli = self.create_client( - GetState, '/ros_typedb/get_state' - ) - - self.query_cli = self.create_client( - Query, '/ros_typedb/query' - ) - - self.delete_db_cli = self.create_client( - Empty, '/ros_typedb/delete_database' - ) - - self.event_sub = self.create_subscription( - String, '/ros_typedb/events', self.event_sub_cb, 10) - - def start_node(self): - self.ros_spin_thread = Thread( - target=lambda node: rclpy.spin(node), - args=(self,)) - self.ros_spin_thread.start() - - def change_ros_typedb_state(self, transition_id): - change_state_req = ChangeState.Request() - change_state_req.transition.id = transition_id - return self.call_service(self.change_state_cli, change_state_req) - - def get_ros_typedb_state(self): - get_state_req = GetState.Request() - return self.call_service(self.get_state_cli, get_state_req) - - def call_service(self, cli, request): - if cli.wait_for_service(timeout_sec=5.0) is False: - self.get_logger().error( - 'service not available {}'.format(cli.srv_name)) - return None - future = cli.call_async(request) - rclpy.spin_until_future_complete(self, future, timeout_sec=5.0) - if future.done() is False: - self.get_logger().error( - 'Future not completed {}'.format(cli.srv_name)) - return None - - return future.result() - - def activate_ros_typedb(self): - self.change_ros_typedb_state(1) - self.change_ros_typedb_state(3) - - def event_sub_cb(self, msg): - self.typedb_event_data = msg.data - self.typedb_event.set() - - def delete_dabase(self): - self.call_service(self.delete_db_cli, Empty.Request()) From fad57561c0659c3e7c8c6a96adbe0faf3e6ba7e8 Mon Sep 17 00:00:00 2001 From: Gustavo Date: Fri, 26 Jun 2026 11:24:36 -0300 Subject: [PATCH 09/36] add dockerignore --- .dockerignore | 171 ++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 171 insertions(+) create mode 100644 .dockerignore diff --git a/.dockerignore b/.dockerignore new file mode 100644 index 0000000..06c423b --- /dev/null +++ b/.dockerignore @@ -0,0 +1,171 @@ +.vscode +docs/source/_autosummary/ +docs/source/generated/ +.claude +.serena +**/.pytest_cache/ +docs-agents/ +AGENTS.md +CLAUDE.md +.gitignore + +# Byte-compiled / optimized / DLL files +__pycache__/ +*.py[cod] +*$py.class + +# C extensions +*.so + +# Distribution / packaging +.Python +build/ +develop-eggs/ +dist/ +downloads/ +eggs/ +.eggs/ +lib/ +lib64/ +parts/ +sdist/ +var/ +wheels/ +share/python-wheels/ +*.egg-info/ +.installed.cfg +*.egg +MANIFEST + +# PyInstaller +# Usually these files are written by a python script from a template +# before PyInstaller builds the exe, so as to inject date/other infos into it. +*.manifest +*.spec + +# Installer logs +pip-log.txt +pip-delete-this-directory.txt + +# Unit test / coverage reports +htmlcov/ +.tox/ +.nox/ +.coverage +.coverage.* +.cache +nosetests.xml +coverage.xml +*.cover +*.py,cover +.hypothesis/ +.pytest_cache/ +cover/ + +# Translations +*.mo +*.pot + +# Django stuff: +*.log +local_settings.py +db.sqlite3 +db.sqlite3-journal + +# Flask stuff: +instance/ +.webassets-cache + +# Scrapy stuff: +.scrapy + +# Sphinx documentation +docs/_build/ + +# PyBuilder +.pybuilder/ +target/ + +# Jupyter Notebook +.ipynb_checkpoints + +# IPython +profile_default/ +ipython_config.py + +# pyenv +# For a library or package, you might want to ignore these files since the code is +# intended to run in multiple environments; otherwise, check them in: +# .python-version + +# pipenv +# According to pypa/pipenv#598, it is recommended to include Pipfile.lock in version control. +# However, in case of collaboration, if having platform-specific dependencies or dependencies +# having no cross-platform support, pipenv may install dependencies that don't work, or not +# install all needed dependencies. +#Pipfile.lock + +# poetry +# Similar to Pipfile.lock, it is generally recommended to include poetry.lock in version control. +# This is especially recommended for binary packages to ensure reproducibility, and is more +# commonly ignored for libraries. +# https://python-poetry.org/docs/basic-usage/#commit-your-poetrylock-file-to-version-control +#poetry.lock + +# pdm +# Similar to Pipfile.lock, it is generally recommended to include pdm.lock in version control. +#pdm.lock +# pdm stores project-wide configurations in .pdm.toml, but it is recommended to not include it +# in version control. +# https://pdm.fming.dev/#use-with-ide +.pdm.toml + +# PEP 582; used by e.g. github.com/David-OConnor/pyflow and github.com/pdm-project/pdm +__pypackages__/ + +# Celery stuff +celerybeat-schedule +celerybeat.pid + +# SageMath parsed files +*.sage.py + +# Environments +.env +.venv +env/ +venv/ +ENV/ +env.bak/ +venv.bak/ + +# Spyder project settings +.spyderproject +.spyproject + +# Rope project settings +.ropeproject + +# mkdocs documentation +/site + +# mypy +.mypy_cache/ +.dmypy.json +dmypy.json + +# Pyre type checker +.pyre/ + +# pytype static type analyzer +.pytype/ + +# Cython debug symbols +cython_debug/ + +# PyCharm +# JetBrains specific template is maintained in a separate JetBrains.gitignore that can +# be found at https://github.com/github/gitignore/blob/main/Global/JetBrains.gitignore +# and can be added to the global gitignore or merged into this file. For a more nuclear +# option (not recommended) you can uncomment the following to ignore the entire idea folder. +#.idea/ From 26fe97f84af4258418132a4737b4ade882314131 Mon Sep 17 00:00:00 2001 From: Gustavo Date: Fri, 26 Jun 2026 11:25:20 -0300 Subject: [PATCH 10/36] Make TypeDB batch writes atomic --- ros_typedb/ros_typedb/typedb_interface.py | 252 +++++++++--- .../test/test_typedb_interface_integration.py | 13 + ros_typedb/test/test_typedb_interface_unit.py | 373 +++++++++++++++++- 3 files changed, 578 insertions(+), 60 deletions(-) diff --git a/ros_typedb/ros_typedb/typedb_interface.py b/ros_typedb/ros_typedb/typedb_interface.py index 1fb8d13..11836b3 100644 --- a/ros_typedb/ros_typedb/typedb_interface.py +++ b/ros_typedb/ros_typedb/typedb_interface.py @@ -21,9 +21,11 @@ import threading from threading import Lock from types import MethodType +from typing import cast from typing import Iterator from typing import Literal from typing import Optional +from typing import Sequence from typing import Tuple from typing import TypedDict @@ -158,6 +160,95 @@ class MatchResultDict(TypedDict): value: str #: value +DatabaseQueryType = Literal[ + 'define', 'insert', 'delete', 'fetch', 'get', 'get_aggregate', 'update'] +DatabaseQueryResult = Literal[True] | Iterator[ConceptMap] | \ + list[dict[str, MatchResultDict]] | None | int | float +NormalizedDatabaseQueryBatch = Tuple[ + list[DatabaseQueryType], list[str], bool] + +_DATABASE_QUERY_TYPES = { + 'define', 'insert', 'delete', 'fetch', 'get', 'get_aggregate', 'update'} +_DATA_READ_QUERY_TYPES = {'fetch', 'get', 'get_aggregate'} +_DATA_WRITE_QUERY_TYPES = {'insert', 'delete', 'update'} +_SCHEMA_WRITE_QUERY_TYPES = {'define'} + + +def normalize_database_query_batch( + query_type: DatabaseQueryType | Sequence[DatabaseQueryType], + query: str | Sequence[str]) -> NormalizedDatabaseQueryBatch: + """ + Normalize scalar-or-batch database query inputs. + + :param query_type: query type or query types. + :param query: query or queries. + :return: normalized query types, normalized queries, single-query flag. + """ + single_query = isinstance(query, str) and isinstance(query_type, str) + queries = [query] if isinstance(query, str) else list(query) + query_types = ( + [query_type] * len(queries) + if isinstance(query_type, str) + else list(query_type)) + if len(query_types) != len(queries): + raise ValueError( + 'query_type and query lists must have the same length') + for query_value in queries: + if not isinstance(query_value, str): + raise TypeError('query values must be strings') + for query_type_value in query_types: + if query_type_value not in _DATABASE_QUERY_TYPES: + raise ValueError( + 'Unsupported TypeDB query type: {}'.format( + query_type_value)) + return cast(list[DatabaseQueryType], query_types), queries, single_query + + +def validate_database_query_batch( + session_type: SessionType, + transaction_type: TransactionType, + query_types: Sequence[DatabaseQueryType]) -> None: + """ + Validate session and transaction compatibility for query types. + + :param session_type: TypeDB session type. + :param transaction_type: TypeDB transaction type. + :param query_types: normalized query types. + """ + if transaction_type == TransactionType.READ: + invalid_query_types = [ + query_type for query_type in query_types + if query_type not in _DATA_READ_QUERY_TYPES] + if invalid_query_types: + raise ValueError( + 'Read transactions only support query types: {}'.format( + ', '.join(sorted(_DATA_READ_QUERY_TYPES)))) + if session_type != SessionType.DATA: + raise ValueError('Read queries require a DATA session') + return + + if transaction_type == TransactionType.WRITE: + if session_type == SessionType.DATA: + allowed_query_types = _DATA_WRITE_QUERY_TYPES + elif session_type == SessionType.SCHEMA: + allowed_query_types = _SCHEMA_WRITE_QUERY_TYPES + else: + raise ValueError('Unsupported TypeDB session type') + invalid_query_types = [ + query_type for query_type in query_types + if query_type not in allowed_query_types] + if invalid_query_types: + raise ValueError( + '{} transactions with {} sessions only support query types: ' + '{}'.format( + transaction_type, + session_type, + ', '.join(sorted(allowed_query_types)))) + return + + raise ValueError('Unsupported TypeDB transaction type') + + class ThingPrefixAttrDict(TypedDict): """TypedDict for dict with thing prefix and attributes.""" @@ -385,13 +476,13 @@ def ensure_server_alive(self) -> None: else: self.ensure_database_exists() - def delete_database(self, database_name: str = None) -> None: + def delete_database(self, database_name: str = '') -> None: """ Delete database. :param database_name: database name. """ - if database_name is None: + if database_name == '': database_name = self.database_name if self.driver.databases.contains(database_name): self.driver.databases.get(database_name).delete() @@ -438,14 +529,12 @@ def _database_query_unlocked( self, session_type: SessionType, transaction_type: TransactionType, - query_type: Literal[ - 'define', 'insert', 'delete', 'fetch', 'get', - 'get_aggregate', 'update'], - query: str, + query_type: DatabaseQueryType | Sequence[DatabaseQueryType], + query: str | Sequence[str], options: TypeDBOptions, effective_timeout: Optional[float] = None - ) -> Literal[True] | Iterator[ConceptMap] | \ - list[dict[str, MatchResultDict]] | None | int | float: + ) -> DatabaseQueryResult | \ + list[DatabaseQueryResult]: """ Query database without acquiring the database query lock. @@ -461,6 +550,11 @@ def _database_query_unlocked( transaction_timeout_millis when positive. :return: Query result, type depends on the query_type. """ + query_types, queries, single_query = normalize_database_query_batch( + query_type, query) + validate_database_query_batch( + session_type, transaction_type, query_types) + results = [] with self.create_session( self.database_name, session_type) as session: if effective_timeout and effective_timeout > 0: @@ -470,27 +564,30 @@ def _database_query_unlocked( options.parallel = True with session.transaction( transaction_type, options) as transaction: - transaction_query_function = getattr( - transaction.query, query_type) - query_answer = transaction_query_function(query) + for batch_query_type, batch_query in zip( + query_types, queries): + transaction_query_function = getattr( + transaction.query, batch_query_type) + query_answer = transaction_query_function(batch_query) + if transaction_type == TransactionType.WRITE: + if batch_query_type in ('insert', 'update'): + results.append(list(query_answer)) + else: + results.append(True) + elif transaction_type == TransactionType.READ: + if batch_query_type == 'get_aggregate': + answer = query_answer.resolve() + if answer.is_long(): + results.append(answer.as_long()) + elif answer.is_float(): + results.append(answer.as_float()) + else: + results.append(None) + else: + results.append(list(query_answer)) if transaction_type == TransactionType.WRITE: - if query_type == 'insert': - query_answer = list(query_answer) transaction.commit() - - if query_type == 'delete' or query_type == 'define': - return True # delete and define always return None - - return query_answer - elif transaction_type == TransactionType.READ: - if query_type == 'get_aggregate': - answer = query_answer.resolve() - if answer.is_long(): - return answer.as_long() - if answer.is_float(): - return answer.as_float() - return None - return list(query_answer) + return results[0] if single_query else results # Read/write database # Generic query method @@ -498,14 +595,12 @@ def _database_query_impl( self, session_type: SessionType, transaction_type: TransactionType, - query_type: Literal[ - 'define', 'insert', 'delete', 'fetch', 'get', - 'get_aggregate', 'update'], - query: str, + query_type: DatabaseQueryType | Sequence[DatabaseQueryType], + query: str | Sequence[str], options: TypeDBOptions, effective_timeout: Optional[float] = None - ) -> Literal[True] | Iterator[ConceptMap] | \ - list[dict[str, MatchResultDict]] | None | int | float: + ) -> DatabaseQueryResult | \ + list[DatabaseQueryResult]: """ Query database (internal implementation). @@ -535,14 +630,12 @@ def database_query( self, session_type: SessionType, transaction_type: TransactionType, - query_type: Literal[ - 'define', 'insert', 'delete', 'fetch', 'get', - 'get_aggregate', 'update'], - query: str, + query_type: DatabaseQueryType | Sequence[DatabaseQueryType], + query: str | Sequence[str], options: Optional[TypeDBOptions] = None, timeout: Optional[float] = None - ) -> Literal[True] | Iterator[ConceptMap] | \ - list[dict[str, MatchResultDict]] | None | int | float: + ) -> DatabaseQueryResult | \ + list[DatabaseQueryResult]: """ Query database. @@ -553,8 +646,9 @@ def database_query( :param session_type: TypeDB session type. :param transaction_type: TypeDB transaction type. - :param query_type: TypeDB query type. - :param query: Query to be performed. + :param query_type: TypeDB query type, or one type per query. + :param query: Query to be performed, or queries to run in one + transaction. :param options: TypeDB options; defaults to a fresh TypeDBOptions(). :param timeout: per-call timeout in seconds. Falls back to self._query_timeout_s. If both are None or non-positive, no @@ -684,11 +778,23 @@ def delete_all_data(self) -> None: 'match $r isa relation; delete $r isa relation;', 'match $a isa attribute; delete $a isa attribute;', ] - for query in delete_queries: - if self.delete_from_database(query) is None: - raise RuntimeError( - 'Failed to delete all TypeDB data: {}'.format( - self.last_error)) + delete_query_types: list[DatabaseQueryType] = [ + 'delete' for _ in delete_queries] + self.last_error = '' + try: + self.database_query( + SessionType.DATA, + TransactionType.WRITE, + delete_query_types, + delete_queries) + except Exception as err: + self.logger.warning( + 'Error deleting all TypeDB data! Exception retrieved: %s', + err) + self.last_error = str(err) + raise RuntimeError( + 'Failed to delete all TypeDB data: {}'.format( + self.last_error)) from err def load_data(self, data_path: str, force: bool = False) -> None: """ @@ -1624,10 +1730,34 @@ def update_attribute_in_thing( :param attr_value: attribute value to be inserted :return: Insert query result. """ - self.delete_attribute_from_thing( - thing, key, key_value, attr) - return self.insert_attribute_in_thing( - thing, key, key_value, attr, attr_value) + key_value = convert_py_type_to_query_type(key_value) + attr_value = convert_py_type_to_query_type(attr_value) + delete_query = f""" + match $thing isa {thing}, + has {key} {key_value}, + has {attr} $attribute; + delete $thing has $attribute; + """ + insert_query = f""" + match $thing isa {thing}, + has {key} {key_value}; + insert $thing has {attr} {attr_value}; + """ + query_types: list[DatabaseQueryType] = ['delete', 'insert'] + self.last_error = '' + try: + result = self.database_query( + SessionType.DATA, + TransactionType.WRITE, + query_types, + [delete_query, insert_query]) + return result[-1] + except Exception as err: + self.logger.warning( + 'Error updating attribute in thing! Exception retrieved: %s', + err) + self.last_error = str(err) + return None def update_attributes_in_thing( self, @@ -1676,5 +1806,23 @@ def update_attributes_in_thing( insert $p1 isa person, has height 1.50, has age 17; """ - self.delete_attributes_from_thing(match_dict, 'update_attributes') - return self.insert_attributes_in_thing(match_dict, 'update_attributes') + delete_query = 'match ' + self.dict_to_query( + match_dict, delete_attribute_str='update_attributes') + insert_query = 'match ' + self.dict_to_query(match_dict) + insert_query += 'insert ' + self.dict_to_query( + match_dict, 'update_attributes') + query_types: list[DatabaseQueryType] = ['delete', 'insert'] + self.last_error = '' + try: + result = self.database_query( + SessionType.DATA, + TransactionType.WRITE, + query_types, + [delete_query, insert_query]) + return result[-1] + except Exception as err: + self.logger.warning( + 'Error updating attributes in thing! Exception retrieved: %s', + err) + self.last_error = str(err) + return None diff --git a/ros_typedb/test/test_typedb_interface_integration.py b/ros_typedb/test/test_typedb_interface_integration.py index 4e1ab03..2b449f0 100644 --- a/ros_typedb/test/test_typedb_interface_integration.py +++ b/ros_typedb/test/test_typedb_interface_integration.py @@ -352,6 +352,19 @@ def test_update_attribute_in_thing(typedb_interface, attr, attr_value, new_v): assert result_update is not None and result[0] == new_v +def test_update_attribute_in_thing_inserts_when_attribute_absent( + typedb_interface): + """update_attribute_in_thing preserves existing insert-if-absent behavior.""" + typedb_interface.insert_entity('person', [('email', 'test@email.test')]) + + result_update = typedb_interface.update_attribute_in_thing( + 'person', 'email', 'test@email.test', 'age', 56) + result = typedb_interface.fetch_attribute_from_thing( + 'person', [('email', 'test@email.test')], 'age') + + assert result_update is not None and result == [56] + + def test_insert_relationship(typedb_interface): typedb_interface.insert_entity('person', [('email', 'test@email.test')]) typedb_interface.insert_entity('person', [('email', 'test2@email.test')]) diff --git a/ros_typedb/test/test_typedb_interface_unit.py b/ros_typedb/test/test_typedb_interface_unit.py index d2c8add..d64e133 100644 --- a/ros_typedb/test/test_typedb_interface_unit.py +++ b/ros_typedb/test/test_typedb_interface_unit.py @@ -17,10 +17,12 @@ import threading from threading import Lock import time +from typing import cast import pytest from ros_typedb.typedb_interface import convert_py_type_to_query_type +from ros_typedb.typedb_interface import DatabaseQueryType from ros_typedb.typedb_interface import string_to_string_array from ros_typedb.typedb_interface import TypeDBInterface @@ -372,6 +374,190 @@ def session(self, *args, **kwargs): return CapturingSession() assert captured_options[0].transaction_timeout_millis == 2500 +def test_database_query_batch_uses_one_transaction(): + """database_query can run an ordered query batch in one transaction.""" + calls = [] + + class CapturingQuery: + + @staticmethod + def delete(query): + calls.append(('delete', query)) + + @staticmethod + def insert(query): + calls.append(('insert', query)) + return iter(['inserted']) + + class CapturingTransaction: + + query = CapturingQuery() + + def __enter__(self): return self + def __exit__(self, *args): return False + def commit(self): calls.append(('commit', None)) + + class CapturingSession: + + def __enter__(self): return self + def __exit__(self, *args): return False + + def transaction(self, txn_type, options): + calls.append(('transaction', txn_type)) + return CapturingTransaction() + + class FakeDriver: + + class databases: + + @staticmethod + def contains(name): return True + + def session(self, *args, **kwargs): return CapturingSession() + + tdb = TypeDBInterface.__new__(TypeDBInterface) + tdb._query_timeout_s = None + tdb._database_query_lock = Lock() + tdb._infer = False + tdb.database_name = 'test_database' + tdb.last_error = '' + tdb.logger = logging.getLogger() + tdb.driver = FakeDriver() + + result = tdb.database_query( + SessionType.DATA, + TransactionType.WRITE, + ['delete', 'insert'], + ['delete query', 'insert query']) + + assert result == [True, ['inserted']] + assert calls == [ + ('transaction', TransactionType.WRITE), + ('delete', 'delete query'), + ('insert', 'insert query'), + ('commit', None), + ] + + +def test_database_query_batch_rejects_mismatched_query_lists(): + """database_query rejects batches with unmatched query types and queries.""" + tdb = TypeDBInterface.__new__(TypeDBInterface) + tdb._query_timeout_s = None + tdb._database_query_lock = Lock() + tdb._infer = False + tdb.database_name = 'test_database' + tdb.last_error = '' + tdb.logger = logging.getLogger() + + class FakeDriver: + + class databases: + + @staticmethod + def contains(name): return True + + tdb.driver = FakeDriver() + + with pytest.raises(ValueError, match='same length'): + tdb.database_query( + SessionType.DATA, + TransactionType.WRITE, + ['delete', 'insert'], + ['delete query']) + + +def _make_validation_tdb(): + """Create a TypeDBInterface that fails if validation opens a session.""" + tdb = TypeDBInterface.__new__(TypeDBInterface) + tdb._query_timeout_s = None + tdb._database_query_lock = Lock() + tdb._infer = False + tdb.database_name = 'test_database' + tdb.last_error = '' + tdb.logger = logging.getLogger() + + class FakeDriver: + + class databases: + + @staticmethod + def contains(name): return True + + def session(self, *args, **kwargs): + raise AssertionError('validation should fail before session open') + + tdb.driver = FakeDriver() + return tdb + + +def test_database_query_rejects_unsupported_query_type(): + """database_query rejects unknown query types before opening a session.""" + tdb = _make_validation_tdb() + invalid_query_type = cast(DatabaseQueryType, 'bad_query') + + with pytest.raises(ValueError, match='Unsupported TypeDB query type'): + tdb.database_query( + SessionType.DATA, + TransactionType.WRITE, + invalid_query_type, + 'insert $x isa thing;') + + +def test_database_query_rejects_write_query_in_read_transaction(): + """database_query rejects write query types in read transactions.""" + tdb = _make_validation_tdb() + + with pytest.raises(ValueError, match='Read transactions only support'): + tdb.database_query( + SessionType.DATA, + TransactionType.READ, + 'insert', + 'insert $x isa thing;') + + +def test_database_query_rejects_read_query_in_write_transaction(): + """database_query rejects read query types in write transactions.""" + tdb = _make_validation_tdb() + + with pytest.raises(ValueError, match='only support query types'): + tdb.database_query( + SessionType.DATA, + TransactionType.WRITE, + 'fetch', + 'match $x isa thing; fetch $x;') + + +def test_database_query_rejects_define_query_in_data_session(): + """database_query requires schema write queries to use schema sessions.""" + tdb = _make_validation_tdb() + + with pytest.raises(ValueError, match='only support query types'): + tdb.database_query( + SessionType.DATA, + TransactionType.WRITE, + 'define', + 'define person sub entity;') + + +def test_raw_wrapper_logs_validation_error(caplog, monkeypatch): + """Public wrappers catch validation errors and log them.""" + tdb = TypeDBInterface.__new__(TypeDBInterface) + tdb.last_error = '' + tdb.logger = logging.getLogger() + + def raise_validation_error(*args, **kwargs): + raise ValueError('Unsupported TypeDB query type: bad_query') + + monkeypatch.setattr(tdb, 'database_query', raise_validation_error) + + with caplog.at_level(logging.WARNING): + result = tdb.insert_database('insert $x isa thing;') + + assert result is None + assert tdb.last_error == 'Unsupported TypeDB query type: bad_query' + assert 'Unsupported TypeDB query type: bad_query' in caplog.text + + def test_string_to_string_array_preserves_raw_path_with_comma(): path = '/tmp/schema,with-comma.tql' @@ -573,23 +759,194 @@ def test_init_loads_schema_for_new_database_when_reload_disabled(monkeypatch): assert loaded_schema_paths == ['schema.tql'] -def test_delete_all_data_raises_when_delete_step_fails(monkeypatch): +def test_delete_all_data_uses_single_atomic_database_query(monkeypatch): + """delete_all_data runs all deletes in one write transaction.""" + typedb_interface = TypeDBInterface.__new__(TypeDBInterface) + typedb_interface.last_error = '' + typedb_interface.logger = logging.getLogger() + captured = {} + + def fake_database_query(session_type, transaction_type, query_type, query): + captured['session_type'] = session_type + captured['transaction_type'] = transaction_type + captured['query_type'] = query_type + captured['query'] = query + return True + + monkeypatch.setattr( + typedb_interface, + 'database_query', + fake_database_query) + monkeypatch.setattr( + typedb_interface, + 'delete_from_database', + lambda query: pytest.fail('delete_all_data must use one batch')) + + typedb_interface.delete_all_data() + + assert captured['session_type'] == SessionType.DATA + assert captured['transaction_type'] == TransactionType.WRITE + assert captured['query_type'] == ['delete', 'delete', 'delete'] + assert captured['query'] == [ + 'match $e isa entity; delete $e isa entity;', + 'match $r isa relation; delete $r isa relation;', + 'match $a isa attribute; delete $a isa attribute;', + ] + + +def test_delete_all_data_raises_when_delete_batch_fails(monkeypatch): """delete_all_data stops instead of loading data over residual data.""" typedb_interface = TypeDBInterface.__new__(TypeDBInterface) - typedb_interface.last_error = 'delete failed' - queries = [] + typedb_interface.last_error = '' + typedb_interface.logger = logging.getLogger() - def fake_delete_from_database(query): - queries.append(query) - return None + def fake_database_query(*args): + raise RuntimeError('delete failed') monkeypatch.setattr( - typedb_interface, 'delete_from_database', fake_delete_from_database) + typedb_interface, + 'database_query', + fake_database_query) with pytest.raises(RuntimeError, match='delete failed'): typedb_interface.delete_all_data() - assert queries == ['match $e isa entity; delete $e isa entity;'] + assert typedb_interface.last_error == 'delete failed' + + +def test_update_attribute_in_thing_uses_atomic_database_query(monkeypatch): + """Single-attribute update deletes and inserts in one transaction.""" + typedb_interface = TypeDBInterface.__new__(TypeDBInterface) + typedb_interface.last_error = '' + typedb_interface.logger = logging.getLogger() + captured = {} + + def fake_database_query(session_type, transaction_type, query_type, query): + captured['session_type'] = session_type + captured['transaction_type'] = transaction_type + captured['query_type'] = query_type + captured['query'] = query + return [True, ['insert result']] + + monkeypatch.setattr( + typedb_interface, + 'database_query', + fake_database_query) + monkeypatch.setattr( + typedb_interface, + 'delete_attribute_from_thing', + lambda *args: pytest.fail('update must not call public delete')) + monkeypatch.setattr( + typedb_interface, + 'insert_attribute_in_thing', + lambda *args: pytest.fail('update must not call public insert')) + + result = typedb_interface.update_attribute_in_thing( + 'person', 'email', "o'brien@test.test", 'age', 42) + + assert result == ['insert result'] + assert captured['session_type'] == SessionType.DATA + assert captured['transaction_type'] == TransactionType.WRITE + assert captured['query_type'] == ['delete', 'insert'] + assert "has email 'o\\'brien@test.test'" in captured['query'][0] + assert 'has age $attribute' in captured['query'][0] + assert 'delete $thing has $attribute;' in captured['query'][0] + assert 'insert $thing has age 42;' in captured['query'][1] + + +def test_update_attributes_in_thing_uses_atomic_database_query(monkeypatch): + """Multi-attribute update deletes and inserts in one transaction.""" + typedb_interface = TypeDBInterface.__new__(TypeDBInterface) + typedb_interface.last_error = '' + typedb_interface.logger = logging.getLogger() + captured = {} + + def fake_database_query(session_type, transaction_type, query_type, query): + captured['session_type'] = session_type + captured['transaction_type'] = transaction_type + captured['query_type'] = query_type + captured['query'] = query + return [True, ['insert result']] + + monkeypatch.setattr( + typedb_interface, + 'database_query', + fake_database_query) + monkeypatch.setattr( + typedb_interface, + 'delete_attributes_from_thing', + lambda *args: pytest.fail('update must not call public delete')) + monkeypatch.setattr( + typedb_interface, + 'insert_attributes_in_thing', + lambda *args: pytest.fail('update must not call public insert')) + + result = typedb_interface.update_attributes_in_thing({ + 'person': [ + { + 'prefix': 'p1', + 'attributes': {'email': 'test@test.test'}, + 'update_attributes': {'height': 1.5, 'age': 17}, + }, + ], + }) + + assert result == ['insert result'] + assert captured['session_type'] == SessionType.DATA + assert captured['transaction_type'] == TransactionType.WRITE + assert captured['query_type'] == ['delete', 'insert'] + assert 'delete $p1 has $p1_height, has $p1_age;' in captured['query'][0] + assert 'insert $p1 isa person,' in captured['query'][1] + assert 'has height 1.5, has age 17;' in captured['query'][1] + + +def test_update_attribute_in_thing_returns_none_when_batch_fails(monkeypatch): + """Single-attribute update preserves wrapper-style failure behavior.""" + typedb_interface = TypeDBInterface.__new__(TypeDBInterface) + typedb_interface.last_error = '' + typedb_interface.logger = logging.getLogger() + + def fake_database_query(*args): + raise RuntimeError('update failed') + + monkeypatch.setattr( + typedb_interface, + 'database_query', + fake_database_query) + + result = typedb_interface.update_attribute_in_thing( + 'person', 'email', 'test@test.test', 'age', 42) + + assert result is None + assert typedb_interface.last_error == 'update failed' + + +def test_update_attributes_in_thing_returns_none_when_batch_fails(monkeypatch): + """Multi-attribute update preserves wrapper-style failure behavior.""" + typedb_interface = TypeDBInterface.__new__(TypeDBInterface) + typedb_interface.last_error = '' + typedb_interface.logger = logging.getLogger() + + def fake_database_query(*args): + raise RuntimeError('update failed') + + monkeypatch.setattr( + typedb_interface, + 'database_query', + fake_database_query) + + result = typedb_interface.update_attributes_in_thing({ + 'person': [ + { + 'prefix': 'p1', + 'attributes': {'email': 'test@test.test'}, + 'update_attributes': {'age': 42}, + }, + ], + }) + + assert result is None + assert typedb_interface.last_error == 'update failed' def test_convert_py_type_to_query_type_escapes_strings(): From 697b828475f8629b24c63e63b6dffdb4e47e343a Mon Sep 17 00:00:00 2001 From: Gustavo Date: Fri, 26 Jun 2026 11:36:05 -0300 Subject: [PATCH 11/36] Avoid shared TypeDBOptions defaults --- ros_typedb/ros_typedb/typedb_interface.py | 4 ++- ros_typedb/test/test_typedb_interface_unit.py | 25 +++++++++++++++++++ 2 files changed, 28 insertions(+), 1 deletion(-) diff --git a/ros_typedb/ros_typedb/typedb_interface.py b/ros_typedb/ros_typedb/typedb_interface.py index 11836b3..2959680 100644 --- a/ros_typedb/ros_typedb/typedb_interface.py +++ b/ros_typedb/ros_typedb/typedb_interface.py @@ -514,7 +514,7 @@ def create_session( self, database_name: str, session_type: SessionType, - options: Optional[TypeDBOptions] = TypeDBOptions() + options: Optional[TypeDBOptions] = None ) -> TypeDBSession: """ Create session with the database. @@ -523,6 +523,8 @@ def create_session( :param session_type: session type, e.g., schema or data. :param options: typedb options. """ + if options is None: + options = TypeDBOptions() return self.driver.session(database_name, session_type, options) def _database_query_unlocked( diff --git a/ros_typedb/test/test_typedb_interface_unit.py b/ros_typedb/test/test_typedb_interface_unit.py index d64e133..f1b0655 100644 --- a/ros_typedb/test/test_typedb_interface_unit.py +++ b/ros_typedb/test/test_typedb_interface_unit.py @@ -28,6 +28,7 @@ from typedb.driver import SessionType from typedb.driver import TransactionType +from typedb.driver import TypeDBOptions # --------------------------------------------------------------------------- @@ -374,6 +375,30 @@ def session(self, *args, **kwargs): return CapturingSession() assert captured_options[0].transaction_timeout_millis == 2500 +def test_create_session_uses_fresh_default_options(): + """create_session creates a fresh TypeDBOptions when none is provided.""" + captured_options = [] + + class FakeDriver: + + def session(self, database_name, session_type, options): + captured_options.append(options) + return 'session' + + tdb = TypeDBInterface.__new__(TypeDBInterface) + tdb.driver = FakeDriver() + + first_session = tdb.create_session('test_database', SessionType.DATA) + second_session = tdb.create_session('test_database', SessionType.DATA) + + assert first_session == 'session' + assert second_session == 'session' + assert len(captured_options) == 2 + assert isinstance(captured_options[0], TypeDBOptions) + assert isinstance(captured_options[1], TypeDBOptions) + assert captured_options[0] is not captured_options[1] + + def test_database_query_batch_uses_one_transaction(): """database_query can run an ordered query batch in one transaction.""" calls = [] From c79ce49ac85320fd8f2b65cfd7a5686a1cb76df0 Mon Sep 17 00:00:00 2001 From: Gustavo Date: Fri, 26 Jun 2026 11:56:30 -0300 Subject: [PATCH 12/36] Serialize TypeDB cleanup operations --- ros_typedb/ros_typedb/ros_typedb_interface.py | 85 +++++++++++-------- ros_typedb/ros_typedb/typedb_interface.py | 14 ++- .../test/test_ros_typedb_interface_unit.py | 46 +++++++++- ros_typedb/test/test_typedb_interface_unit.py | 49 +++++++++++ 4 files changed, 158 insertions(+), 36 deletions(-) diff --git a/ros_typedb/ros_typedb/ros_typedb_interface.py b/ros_typedb/ros_typedb/ros_typedb_interface.py index 1828fb4..36a13f7 100644 --- a/ros_typedb/ros_typedb/ros_typedb_interface.py +++ b/ros_typedb/ros_typedb/ros_typedb_interface.py @@ -13,6 +13,7 @@ # limitations under the License. """ros_typedb_interface - python interface to interact with typedb via ROS.""" +import threading import traceback from typing import Any @@ -332,6 +333,15 @@ def __init__(self, node_name: str, **kwargs): self.typedb_interface_class = TypeDBInterface self.query_cb_group = MutuallyExclusiveCallbackGroup() + self._service_callback_lock = threading.Lock() + + def _get_service_callback_lock(self): + """Return the lock used to serialize service callbacks with cleanup.""" + lock = getattr(self, '_service_callback_lock', None) + if lock is None: + lock = threading.Lock() + self._service_callback_lock = lock + return lock def init_typedb_interface( self, @@ -479,11 +489,13 @@ def on_cleanup(self, state: State) -> TransitionCallbackReturn: self.destroy_service(service) delattr(self, service_name) - if getattr(self, 'event_pub', None) is not None: - self.destroy_publisher(self.event_pub) - del self.event_pub + service_callback_lock = self._get_service_callback_lock() + with service_callback_lock: + if getattr(self, 'event_pub', None) is not None: + self.destroy_publisher(self.event_pub) + del self.event_pub - self.close_typedb_interface() + self.close_typedb_interface() self.get_logger().info(self.get_name() + ' :on_cleanup() is called.') return TransitionCallbackReturn.SUCCESS @@ -502,37 +514,40 @@ def query_service_cb( :param response: query result :return: query result """ - if req.query_type == Query.Request.INSERT: - query_func = self.typedb_interface.insert_database - elif req.query_type == Query.Request.DELETE: - query_func = self.typedb_interface.delete_from_database - elif req.query_type == Query.Request.FETCH: - query_func = self.typedb_interface.fetch_database - elif req.query_type == Query.Request.GET: - query_func = self.typedb_interface.get_database - elif req.query_type == Query.Request.GET_AGGREGATE: - query_func = self.typedb_interface.get_aggregate_database - elif req.query_type == Query.Request.UPDATE: - query_func = self.typedb_interface.update_database - elif req.query_type == Query.Request.DEFINE: - query_func = self.typedb_interface.define_database - else: - self.get_logger().warning( - 'Query type {} not recognized'.format(req.query_type)) - response.success = False - response.error_message = f'Unknown query type: {req.query_type}' + service_callback_lock = self._get_service_callback_lock() + with service_callback_lock: + if req.query_type == Query.Request.INSERT: + query_func = self.typedb_interface.insert_database + elif req.query_type == Query.Request.DELETE: + query_func = self.typedb_interface.delete_from_database + elif req.query_type == Query.Request.FETCH: + query_func = self.typedb_interface.fetch_database + elif req.query_type == Query.Request.GET: + query_func = self.typedb_interface.get_database + elif req.query_type == Query.Request.GET_AGGREGATE: + query_func = self.typedb_interface.get_aggregate_database + elif req.query_type == Query.Request.UPDATE: + query_func = self.typedb_interface.update_database + elif req.query_type == Query.Request.DEFINE: + query_func = self.typedb_interface.define_database + else: + self.get_logger().warning( + 'Query type {} not recognized'.format(req.query_type)) + response.success = False + response.error_message = ( + f'Unknown query type: {req.query_type}') + return response + + per_call_timeout = req.timeout_s if req.timeout_s > 0 else None + query_result = query_func(req.query, timeout=per_call_timeout) + response = query_result_to_ros_msg(req.query_type, query_result) + if query_result is None: + response.success = False + response.error_message = self.typedb_interface.last_error + else: + response.success = True return response - per_call_timeout = req.timeout_s if req.timeout_s > 0 else None - query_result = query_func(req.query, timeout=per_call_timeout) - response = query_result_to_ros_msg(req.query_type, query_result) - if query_result is None: - response.success = False - response.error_message = self.typedb_interface.last_error - else: - response.success = True - return response - def delete_db_cb( self, req: std_srvs.srv.Empty.Request, @@ -543,5 +558,7 @@ def delete_db_cb( Delete the dabase. """ - self.typedb_interface.delete_database() + service_callback_lock = self._get_service_callback_lock() + with service_callback_lock: + self.typedb_interface.delete_database() return response diff --git a/ros_typedb/ros_typedb/typedb_interface.py b/ros_typedb/ros_typedb/typedb_interface.py index 2959680..4308898 100644 --- a/ros_typedb/ros_typedb/typedb_interface.py +++ b/ros_typedb/ros_typedb/typedb_interface.py @@ -448,7 +448,7 @@ def ensure_database_exists(self) -> None: self._load_data_unlocked(path) except Exception: try: - self.delete_database() + self._delete_database_unlocked() except Exception as exc: self.logger.warning( 'Failed to remove partially initialized database %s: %s', @@ -480,6 +480,18 @@ def delete_database(self, database_name: str = '') -> None: """ Delete database. + :param database_name: database name. + """ + with self._database_query_lock: + self._delete_database_unlocked(database_name) + + def _delete_database_unlocked(self, database_name: str = '') -> None: + """ + Delete database without acquiring the database query lock. + + Callers must hold _database_query_lock or otherwise know no query can + run concurrently. + :param database_name: database name. """ if database_name == '': diff --git a/ros_typedb/test/test_ros_typedb_interface_unit.py b/ros_typedb/test/test_ros_typedb_interface_unit.py index 4b40727..3132134 100644 --- a/ros_typedb/test/test_ros_typedb_interface_unit.py +++ b/ros_typedb/test/test_ros_typedb_interface_unit.py @@ -12,6 +12,8 @@ # See the License for the specific language governing permissions and # limitations under the License. +import threading +import time from types import SimpleNamespace from unittest.mock import MagicMock @@ -207,7 +209,9 @@ def test_on_cleanup_destroys_ros_entities_and_closes_typedb_driver(): get_name=MagicMock(return_value='ros_typedb'), close_typedb_interface=MagicMock( side_effect=lambda: ROSTypeDBInterface.close_typedb_interface(node) - ) + ), + _get_service_callback_lock=( + lambda: ROSTypeDBInterface._get_service_callback_lock(node)) ) result = ROSTypeDBInterface.on_cleanup(node, None) @@ -225,6 +229,46 @@ def test_on_cleanup_destroys_ros_entities_and_closes_typedb_driver(): assert node.typedb_interface is None +def test_on_cleanup_waits_for_active_service_callback_before_closing_driver(): + driver = MagicMock() + typedb_interface = SimpleNamespace(driver=driver) + service_callback_lock = threading.Lock() + service_callback_lock.acquire() + + node = SimpleNamespace( + event_pub=MagicMock(), + query_service=MagicMock(), + delete_db_service=MagicMock(), + typedb_interface=typedb_interface, + _service_callback_lock=service_callback_lock, + destroy_publisher=MagicMock(return_value=True), + destroy_service=MagicMock(return_value=True), + get_logger=MagicMock(return_value=MagicMock()), + get_name=MagicMock(return_value='ros_typedb'), + close_typedb_interface=MagicMock( + side_effect=lambda: ROSTypeDBInterface.close_typedb_interface(node) + ), + _get_service_callback_lock=( + lambda: ROSTypeDBInterface._get_service_callback_lock(node)) + ) + + cleanup_thread = threading.Thread( + target=ROSTypeDBInterface.on_cleanup, + args=(node, None)) + cleanup_thread.start() + + time.sleep(0.05) + driver.close.assert_not_called() + assert node.typedb_interface is typedb_interface + + service_callback_lock.release() + cleanup_thread.join(timeout=1.0) + + assert not cleanup_thread.is_alive() + driver.close.assert_called_once_with() + assert node.typedb_interface is None + + def test_convert_attribute_dict_to_ros_msg(): expected_address_attr = Attribute() expected_address_attr.variable_name = 'company_var' diff --git a/ros_typedb/test/test_typedb_interface_unit.py b/ros_typedb/test/test_typedb_interface_unit.py index f1b0655..6727ec9 100644 --- a/ros_typedb/test/test_typedb_interface_unit.py +++ b/ros_typedb/test/test_typedb_interface_unit.py @@ -399,6 +399,55 @@ def session(self, database_name, session_type, options): assert captured_options[0] is not captured_options[1] +def test_delete_database_waits_for_database_query_lock(): + """delete_database does not delete while a query holds the query lock.""" + calls = [] + + class FakeDatabase: + + @staticmethod + def delete(): + calls.append('delete') + + class FakeDatabases: + + @staticmethod + def contains(database_name): + calls.append(('contains', database_name)) + return True + + @staticmethod + def get(database_name): + calls.append(('get', database_name)) + return FakeDatabase() + + class FakeDriver: + + databases = FakeDatabases() + + tdb = TypeDBInterface.__new__(TypeDBInterface) + tdb._database_query_lock = Lock() + tdb.database_name = 'test_database' + tdb.driver = FakeDriver() + + tdb._database_query_lock.acquire() + thread = threading.Thread(target=tdb.delete_database) + thread.start() + + time.sleep(0.05) + assert calls == [] + + tdb._database_query_lock.release() + thread.join(timeout=1.0) + + assert not thread.is_alive() + assert calls == [ + ('contains', 'test_database'), + ('get', 'test_database'), + 'delete', + ] + + def test_database_query_batch_uses_one_transaction(): """database_query can run an ordered query batch in one transaction.""" calls = [] From 83dc21c57d8f2ff05c14ecb996b38e61905ac98d Mon Sep 17 00:00:00 2001 From: Gustavo Date: Fri, 26 Jun 2026 15:17:02 -0300 Subject: [PATCH 13/36] Add ros_typedb examples package --- ros_typedb_examples/README.md | 2 + ros_typedb_examples/data/plan_data.tql | 71 +++++++++ ros_typedb_examples/data/plan_schema.tql | 91 +++++++++++ .../data/queries/fetch_plan_actions.tql | 4 + .../launch/plan_example.launch.py | 150 ++++++++++++++++++ ros_typedb_examples/package.xml | 24 +++ .../resource/ros_typedb_examples | 1 + .../ros_typedb_examples/__init__.py | 1 + ros_typedb_examples/setup.cfg | 4 + ros_typedb_examples/setup.py | 38 +++++ 10 files changed, 386 insertions(+) create mode 100644 ros_typedb_examples/README.md create mode 100644 ros_typedb_examples/data/plan_data.tql create mode 100644 ros_typedb_examples/data/plan_schema.tql create mode 100644 ros_typedb_examples/data/queries/fetch_plan_actions.tql create mode 100644 ros_typedb_examples/launch/plan_example.launch.py create mode 100644 ros_typedb_examples/package.xml create mode 100644 ros_typedb_examples/resource/ros_typedb_examples create mode 100644 ros_typedb_examples/ros_typedb_examples/__init__.py create mode 100644 ros_typedb_examples/setup.cfg create mode 100644 ros_typedb_examples/setup.py diff --git a/ros_typedb_examples/README.md b/ros_typedb_examples/README.md new file mode 100644 index 0000000..1c789df --- /dev/null +++ b/ros_typedb_examples/README.md @@ -0,0 +1,2 @@ +# ROS TypeDB examples + diff --git a/ros_typedb_examples/data/plan_data.tql b/ros_typedb_examples/data/plan_data.tql new file mode 100644 index 0000000..31051bc --- /dev/null +++ b/ros_typedb_examples/data/plan_data.tql @@ -0,0 +1,71 @@ +insert + +$navigation_system_ready isa Proposition, has valid true; +$camera_system_ready isa Proposition, has valid true; +$water_sampler_ready isa Proposition, has valid true; +$vehicle_at_survey_site isa Proposition, has valid false; +$hull_inspected isa Proposition, has valid false; +$water_sample_collected isa Proposition, has valid false; +$vehicle_back_at_base isa Proposition, has valid false; + +$navigate_to_survey_site isa Action, + has action_name "navigate-to-survey-site", + has action_duration 120.0, + has start_time 2026-06-26T10:00:00.000; + +$inspect_hull isa Action, + has action_name "inspect-hull", + has action_duration 180.0, + has start_time 2026-06-26T10:02:00.000; + +$collect_water_sample isa Action, + has action_name "collect-water-sample", + has action_duration 240.0, + has start_time 2026-06-26T10:05:00.000; + +$return_to_base isa Action, + has action_name "return-to-base", + has action_duration 150.0, + has start_time 2026-06-26T10:09:00.000; + +(action: $navigate_to_survey_site, precondition: $navigation_system_ready) + isa action_preconditions; +(action: $navigate_to_survey_site, effect: $vehicle_at_survey_site) + isa action_effects; + +(action: $inspect_hull, precondition: $vehicle_at_survey_site, precondition: $camera_system_ready) + isa action_preconditions; +(action: $inspect_hull, effect: $hull_inspected) + isa action_effects; + +(action: $collect_water_sample, precondition: $vehicle_at_survey_site, precondition: $water_sampler_ready) + isa action_preconditions; +(action: $collect_water_sample, effect: $water_sample_collected) + isa action_effects; + +(action: $return_to_base, precondition: $navigation_system_ready) + isa action_preconditions; +(action: $return_to_base, effect: $vehicle_back_at_base) + isa action_effects; + +$inspection_plan isa Plan, + has start_time 2026-06-26T10:00:00.000, + has cost 42.5; + +$sampling_plan isa Plan, + has start_time 2026-06-26T10:00:00.000, + has cost 58.0; + +(plan: $inspection_plan, action: $navigate_to_survey_site) + isa has_action, has index 1; +(plan: $inspection_plan, action: $inspect_hull) + isa has_action, has index 2; +(plan: $inspection_plan, action: $return_to_base) + isa has_action, has index 3; + +(plan: $sampling_plan, action: $navigate_to_survey_site) + isa has_action, has index 1; +(plan: $sampling_plan, action: $collect_water_sample) + isa has_action, has index 2; +(plan: $sampling_plan, action: $return_to_base) + isa has_action, has index 3; diff --git a/ros_typedb_examples/data/plan_schema.tql b/ros_typedb_examples/data/plan_schema.tql new file mode 100644 index 0000000..449e7fc --- /dev/null +++ b/ros_typedb_examples/data/plan_schema.tql @@ -0,0 +1,91 @@ +define + +name sub attribute, + abstract, + value string; +action_name sub name, + value string; +action_duration sub attribute, + value double; + +start_time sub attribute, + value datetime; +end_time sub attribute, + value datetime; + +action_preconditions sub relation, + relates action, + relates precondition; + +action_effects sub relation, + relates action, + relates effect; + +valid_precondition sub attribute, + value boolean; +cost sub attribute, + value double; + +result sub attribute, + value boolean; + +Action sub entity, + owns action_name, + owns valid_precondition, + owns action_duration, + owns start_time, + owns end_time, + owns cost, + owns result, + plays action_preconditions:action, + plays action_effects:action, + plays has_action:action; + +index sub attribute, + value long; +has_action sub relation, + owns index, + relates plan, + relates action; + +Plan sub entity, + owns start_time, + owns end_time, + owns result, + owns cost, + owns cost, + plays has_action:plan; + +valid sub attribute, + value boolean; +Proposition sub entity, + owns valid, + plays action_preconditions:precondition, + plays action_effects:effect; + +rule action_precondition_valid: + when { + $a isa Action; + (action: $a, precondition: $proposition) isa action_preconditions; + $proposition has valid true; + } then { + $a has valid_precondition true; + }; + +rule action_precondition_invalid: + when { + $a isa Action; + (action: $a, precondition: $proposition) isa action_preconditions; + $proposition has valid false; + } then { + $a has valid_precondition false; + }; + +rule action_precondition_invalid: + when { + $a isa Action; + (action: $a, precondition: $proposition) isa action_preconditions; + not {$proposition has valid $valid;}; + } then { + $a has valid_precondition false; + }; \ No newline at end of file diff --git a/ros_typedb_examples/data/queries/fetch_plan_actions.tql b/ros_typedb_examples/data/queries/fetch_plan_actions.tql new file mode 100644 index 0000000..cabe729 --- /dev/null +++ b/ros_typedb_examples/data/queries/fetch_plan_actions.tql @@ -0,0 +1,4 @@ +match + $p isa Plan; (plan: $p, action: $a) isa has_action; +fetch + $a: action_name; diff --git a/ros_typedb_examples/launch/plan_example.launch.py b/ros_typedb_examples/launch/plan_example.launch.py new file mode 100644 index 0000000..0a6569f --- /dev/null +++ b/ros_typedb_examples/launch/plan_example.launch.py @@ -0,0 +1,150 @@ +"""Launch ros_typedb with a small stress-test example model.""" + +# Copyright 2026 Gustavo Rezende Silva +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +import os + +from ament_index_python.packages import get_package_share_directory + +from launch import LaunchDescription +from launch.actions import DeclareLaunchArgument +from launch.actions import EmitEvent +from launch.actions import RegisterEventHandler +from launch.events import matches_action +from launch.substitutions import LaunchConfiguration + +from launch_ros.actions import LifecycleNode +from launch_ros.event_handlers import OnStateTransition +from launch_ros.events.lifecycle import ChangeState + +import lifecycle_msgs + + +def generate_launch_description(): + """Launch ros_typedb with the example stress schema and data.""" + package_share = get_package_share_directory('ros_typedb_examples') + default_schema_path = os.path.join( + package_share, + 'data', + 'plan_schema.tql', + ) + default_data_path = os.path.join( + package_share, + 'data', + 'plan_data.tql', + ) + + database_name = LaunchConfiguration('database_name') + address = LaunchConfiguration('address') + force_data = LaunchConfiguration('force_data') + force_database = LaunchConfiguration('force_database') + reload_schema = LaunchConfiguration('reload_schema') + infer = LaunchConfiguration('infer') + query_timeout_s = LaunchConfiguration('query_timeout_s') + + database_name_arg = DeclareLaunchArgument( + 'database_name', + default_value='ros_typedb_stress_example', + description='database name', + ) + + address_arg = DeclareLaunchArgument( + 'address', + default_value='localhost:1729', + description='TypeDB server address', + ) + + force_data_arg = DeclareLaunchArgument( + 'force_data', + default_value='True', + description='reload example data on startup', + ) + + force_database_arg = DeclareLaunchArgument( + 'force_database', + default_value='True', + description='recreate the example database on startup', + ) + + reload_schema_arg = DeclareLaunchArgument( + 'reload_schema', + default_value='True', + description='reload example schema on startup', + ) + + infer_arg = DeclareLaunchArgument( + 'infer', + default_value='True', + description='use inference engine', + ) + + query_timeout_s_arg = DeclareLaunchArgument( + 'query_timeout_s', + default_value='5.0', + description='default TypeDB query timeout in seconds', + ) + + ros_typedb_node = LifecycleNode( + package='ros_typedb', + executable='ros_typedb', + name='ros_typedb_interface', + namespace='', + output='screen', + parameters=[{ + 'schema_path': [default_schema_path], + 'data_path': [default_data_path], + 'database_name': database_name, + 'address': address, + 'force_data': force_data, + 'force_database': force_database, + 'reload_schema': reload_schema, + 'infer': infer, + 'query_timeout_s': query_timeout_s, + }], + ) + + configure_event = EmitEvent( + event=ChangeState( + lifecycle_node_matcher=matches_action(ros_typedb_node), + transition_id=lifecycle_msgs.msg.Transition.TRANSITION_CONFIGURE, + ), + ) + + activate_event = RegisterEventHandler( + OnStateTransition( + target_lifecycle_node=ros_typedb_node, + goal_state='inactive', + entities=[ + EmitEvent(event=ChangeState( + lifecycle_node_matcher=matches_action(ros_typedb_node), + transition_id=( + lifecycle_msgs.msg.Transition.TRANSITION_ACTIVATE + ), + )), + ], + ), + ) + + return LaunchDescription([ + database_name_arg, + address_arg, + force_data_arg, + force_database_arg, + reload_schema_arg, + infer_arg, + query_timeout_s_arg, + ros_typedb_node, + configure_event, + activate_event, + ]) diff --git a/ros_typedb_examples/package.xml b/ros_typedb_examples/package.xml new file mode 100644 index 0000000..38a2e30 --- /dev/null +++ b/ros_typedb_examples/package.xml @@ -0,0 +1,24 @@ + + + + ros_typedb_examples + 0.0.0 + Example launch files and TypeDB models for ros_typedb. + Gustavo Rezende + Apache-2.0 + + ament_index_python + launch + launch_ros + lifecycle_msgs + ros_typedb + + ament_copyright + ament_flake8 + ament_pep257 + python3-pytest + + + ament_python + + diff --git a/ros_typedb_examples/resource/ros_typedb_examples b/ros_typedb_examples/resource/ros_typedb_examples new file mode 100644 index 0000000..8b13789 --- /dev/null +++ b/ros_typedb_examples/resource/ros_typedb_examples @@ -0,0 +1 @@ + diff --git a/ros_typedb_examples/ros_typedb_examples/__init__.py b/ros_typedb_examples/ros_typedb_examples/__init__.py new file mode 100644 index 0000000..c7d013d --- /dev/null +++ b/ros_typedb_examples/ros_typedb_examples/__init__.py @@ -0,0 +1 @@ +"""Example resources for ros_typedb.""" diff --git a/ros_typedb_examples/setup.cfg b/ros_typedb_examples/setup.cfg new file mode 100644 index 0000000..cf66830 --- /dev/null +++ b/ros_typedb_examples/setup.cfg @@ -0,0 +1,4 @@ +[develop] +script_dir=$base/lib/ros_typedb_examples +[install] +install_scripts=$base/lib/ros_typedb_examples diff --git a/ros_typedb_examples/setup.py b/ros_typedb_examples/setup.py new file mode 100644 index 0000000..85279c9 --- /dev/null +++ b/ros_typedb_examples/setup.py @@ -0,0 +1,38 @@ +"""Setup for the ros_typedb_examples package.""" + +import os +from glob import glob + +from setuptools import find_packages +from setuptools import setup + +package_name = 'ros_typedb_examples' + +setup( + name=package_name, + version='0.0.0', + packages=find_packages(exclude=['test']), + data_files=[ + ('share/ament_index/resource_index/packages', + ['resource/' + package_name]), + ('share/' + package_name, ['package.xml']), + (os.path.join('share', package_name, 'launch'), + glob('launch/*launch.[pxy][yma]*')), + (os.path.join('share', package_name, 'data'), + glob('data/*.tql')), + ], + install_requires=['setuptools'], + zip_safe=True, + maintainer='Gustavo Rezende', + maintainer_email='g.rezendesilva@tudelft.nl', + description='Example launch files and TypeDB models for ros_typedb.', + license='Apache-2.0', + extras_require={ + 'test': [ + 'pytest', + ], + }, + entry_points={ + 'console_scripts': [], + }, +) From 2e3d9fd6211df17bacaf924e4fc78b4a3d2aa919 Mon Sep 17 00:00:00 2001 From: Gustavo Date: Fri, 26 Jun 2026 15:17:52 -0300 Subject: [PATCH 14/36] Add ros_typedb stress experiment tool --- ros_typedb_tools/README.md | 38 ++- ros_typedb_tools/package.xml | 2 + .../ros_typedb_stress_experiment.py | 295 ++++++++++++++++++ ros_typedb_tools/setup.py | 1 + .../test/test_ros_typedb_stress_experiment.py | 125 ++++++++ 5 files changed, 450 insertions(+), 11 deletions(-) create mode 100644 ros_typedb_tools/ros_typedb_tools/ros_typedb_stress_experiment.py create mode 100644 ros_typedb_tools/test/test_ros_typedb_stress_experiment.py diff --git a/ros_typedb_tools/README.md b/ros_typedb_tools/README.md index c14ec2f..4f2a9af 100644 --- a/ros_typedb_tools/README.md +++ b/ros_typedb_tools/README.md @@ -4,10 +4,11 @@ Reusable TypeDB 2 helper scripts for generating schema and rule diagrams. ## Overview -This package currently provides two console scripts: +This package currently provides three console scripts: - `typedb_schema_diagram`: generate a diagram for one schema file or a merged diagram for multiple schema files. - `typedb_rule_diagram`: generate a rule dependency/read-write diagram for one or more schema files that contain rules. +- `ros_typedb_stress_experiment`: send repeated requests to the real `ros_typedb` query service and report request metrics. Both tools can write `.svg` or `.dot` output. @@ -96,19 +97,34 @@ Useful options: - `--orientation`: `vertical` or `horizontal` - `--no-rule-dependencies`: suppress rule-to-rule dependency edges -## Running Without ros2 run +## ros_typedb_stress_experiment -If needed, you can also invoke the modules directly from the workspace root: +Runs a fixed-count stress experiment against the real +`/ros_typedb_interface/query` service. -```bash -PYTHONPATH=src/ros_typedb/ros_typedb_tools python3 -m ros_typedb_tools.typedb_schema_diagram --help -PYTHONPATH=src/ros_typedb/ros_typedb_tools python3 -m ros_typedb_tools.typedb_rule_diagram --help -``` +Prerequisites: -## Installed README Location +- TypeDB is running. +- `ros_typedb_interface` is running. +- The lifecycle node has been configured so the query service exists. -After rebuilding the package, this README is installed to: +Example: -```text -install/ros_typedb_tools/share/ros_typedb_tools/README.md +```bash +ros2 run ros_typedb_tools ros_typedb_stress_experiment \ + --query 'match $p isa Plan; (plan: $p, action: $a) isa has_action; fetch $a: action_name;' \ + --query-type fetch \ + --requests 2000 \ + --timeout-s 5 \ + --output ~/results/ros_typedb_stress_results.json ``` + +Useful options: + +- `--service-name`: query service name. Defaults to `/ros_typedb_interface/query` +- `--query`: TypeDB query to send on each request +- `--query-type`: one of `define`, `delete`, `fetch`, `get`, + `get_aggregate`, `insert`, or `update` +- `--requests`: number of requests to send +- `--timeout-s`: per-request client and server timeout +- `--output`: optional JSON results path diff --git a/ros_typedb_tools/package.xml b/ros_typedb_tools/package.xml index 6aae9c1..392819f 100644 --- a/ros_typedb_tools/package.xml +++ b/ros_typedb_tools/package.xml @@ -9,6 +9,8 @@ python3-pydot graphviz + rclpy + ros_typedb_msgs ament_copyright ament_flake8 diff --git a/ros_typedb_tools/ros_typedb_tools/ros_typedb_stress_experiment.py b/ros_typedb_tools/ros_typedb_tools/ros_typedb_stress_experiment.py new file mode 100644 index 0000000..a6f9574 --- /dev/null +++ b/ros_typedb_tools/ros_typedb_tools/ros_typedb_stress_experiment.py @@ -0,0 +1,295 @@ +# Copyright 2026 Gustavo Rezende Silva +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +"""Stress experiment CLI for the ros_typedb query service.""" + +from __future__ import annotations + +import argparse +import json +import statistics +import sys +import time +from dataclasses import asdict +from dataclasses import dataclass +from pathlib import Path +from typing import Any + +import rclpy + +from ros_typedb_msgs.srv import Query + + +QUERY_TYPE_BY_NAME = { + 'insert': Query.Request.INSERT, + 'delete': Query.Request.DELETE, + 'fetch': Query.Request.FETCH, + 'get': Query.Request.GET, + 'get_aggregate': Query.Request.GET_AGGREGATE, + 'define': Query.Request.DEFINE, + 'update': Query.Request.UPDATE, +} + + +@dataclass(frozen=True) +class RequestRecord: + """Result of one service request.""" + + index: int + started_at_s: float + ended_at_s: float + latency_s: float + success: bool + timed_out: bool + error_message: str + result_count: int + + +def build_argument_parser() -> argparse.ArgumentParser: + """Build command-line arguments for the stress experiment.""" + parser = argparse.ArgumentParser( + prog='ros_typedb_stress_experiment', + description='Run a fixed-count stress experiment against ros_typedb.', + ) + parser.add_argument( + '--service-name', + default='/ros_typedb_interface/query', + help='Query service name. Defaults to /ros_typedb_interface/query.', + ) + parser.add_argument( + '--query', + required=True, + help='TypeDB query to send on each request.', + ) + parser.add_argument( + '--query-type', + required=True, + choices=sorted(QUERY_TYPE_BY_NAME), + help='TypeDB query type.', + ) + parser.add_argument( + '--requests', + type=int, + default=1, + help='Number of requests to send. Defaults to 1.', + ) + parser.add_argument( + '--timeout-s', + type=float, + default=10.0, + help=( + 'Client and server timeout per request in seconds. Defaults to 10.' + ), + ) + parser.add_argument( + '--wait-service-timeout-s', + type=float, + default=10.0, + help='Seconds to wait for the query service. Defaults to 10.', + ) + parser.add_argument( + '--output', + help='Optional JSON output path.', + ) + return parser + + +def summarize_records(records: list[RequestRecord]) -> dict[str, Any]: + """Build summary metrics from request records.""" + latencies = [record.latency_s for record in records] + successes = sum(1 for record in records if record.success) + timeouts = sum(1 for record in records if record.timed_out) + failures = len(records) - successes + + summary: dict[str, Any] = { + 'total_requests': len(records), + 'successes': successes, + 'failures': failures, + 'timeouts': timeouts, + 'latency_s': {}, + } + if latencies: + summary['latency_s'] = { + 'min': min(latencies), + 'mean': statistics.fmean(latencies), + 'max': max(latencies), + } + return summary + + +def write_results( + output_path: Path, + *, + service_name: str, + query_type: str, + query: str, + timeout_s: float, + records: list[RequestRecord], +) -> None: + """Write experiment results to a JSON file.""" + payload = { + 'service_name': service_name, + 'query_type': query_type, + 'query': query, + 'timeout_s': timeout_s, + 'summary': summarize_records(records), + 'requests': [asdict(record) for record in records], + } + output_path.parent.mkdir(parents=True, exist_ok=True) + output_path.write_text( + json.dumps(payload, indent=2, sort_keys=True) + '\n', + encoding='utf-8', + ) + + +def print_summary(summary: dict[str, Any]) -> None: + """Print compact experiment summary.""" + latency = summary['latency_s'] + print('ros_typedb stress experiment summary') + print(f" total_requests: {summary['total_requests']}") + print(f" successes: {summary['successes']}") + print(f" failures: {summary['failures']}") + print(f" timeouts: {summary['timeouts']}") + if latency: + print(f" latency_min_s: {latency['min']:.6f}") + print(f" latency_mean_s: {latency['mean']:.6f}") + print(f" latency_max_s: {latency['max']:.6f}") + + +def _build_query_request( + query: str, + query_type: str, + timeout_s: float, +) -> Query.Request: + request = Query.Request() + request.query = query + request.query_type = QUERY_TYPE_BY_NAME[query_type] + request.timeout_s = timeout_s + return request + + +def run_experiment(args: argparse.Namespace) -> list[RequestRecord]: + """Run the fixed-count stress experiment.""" + if args.requests < 1: + raise ValueError('--requests must be greater than zero') + if args.timeout_s <= 0: + raise ValueError('--timeout-s must be greater than zero') + if args.wait_service_timeout_s <= 0: + raise ValueError('--wait-service-timeout-s must be greater than zero') + + node = rclpy.create_node('ros_typedb_stress_experiment') + client = node.create_client(Query, args.service_name) + records: list[RequestRecord] = [] + try: + service_ready = client.wait_for_service( + timeout_sec=args.wait_service_timeout_s + ) + if not service_ready: + raise RuntimeError( + f'Query service is not available: {args.service_name}' + ) + + for index in range(args.requests): + request = _build_query_request( + args.query, + args.query_type, + args.timeout_s, + ) + started_at_s = time.monotonic() + future = client.call_async(request) + rclpy.spin_until_future_complete( + node, + future, + timeout_sec=args.timeout_s, + ) + ended_at_s = time.monotonic() + latency_s = ended_at_s - started_at_s + + if not future.done(): + records.append( + RequestRecord( + index=index, + started_at_s=started_at_s, + ended_at_s=ended_at_s, + latency_s=latency_s, + success=False, + timed_out=True, + error_message='client request timed out', + result_count=0, + ) + ) + continue + + exception = future.exception() + if exception is not None: + records.append( + RequestRecord( + index=index, + started_at_s=started_at_s, + ended_at_s=ended_at_s, + latency_s=latency_s, + success=False, + timed_out=False, + error_message=str(exception), + result_count=0, + ) + ) + continue + + response = future.result() + records.append( + RequestRecord( + index=index, + started_at_s=started_at_s, + ended_at_s=ended_at_s, + latency_s=latency_s, + success=response.success, + timed_out=False, + error_message=response.error_message, + result_count=len(response.results), + ) + ) + finally: + node.destroy_node() + return records + + +def main(argv: list[str] | None = None) -> int: + """Run the stress experiment CLI.""" + parser = build_argument_parser() + args = parser.parse_args(argv) + + rclpy.init(args=None) + try: + records = run_experiment(args) + summary = summarize_records(records) + print_summary(summary) + if args.output: + write_results( + Path(args.output), + service_name=args.service_name, + query_type=args.query_type, + query=args.query, + timeout_s=args.timeout_s, + records=records, + ) + return 0 if summary['failures'] == 0 else 1 + except (RuntimeError, ValueError, OSError) as exc: + print(f'ros_typedb stress experiment failed: {exc}', file=sys.stderr) + return 1 + finally: + rclpy.shutdown() + + +if __name__ == '__main__': + raise SystemExit(main()) diff --git a/ros_typedb_tools/setup.py b/ros_typedb_tools/setup.py index 64649a9..d3952c0 100644 --- a/ros_typedb_tools/setup.py +++ b/ros_typedb_tools/setup.py @@ -26,6 +26,7 @@ 'console_scripts': [ 'typedb_schema_diagram = ros_typedb_tools.typedb_schema_diagram:main', 'typedb_rule_diagram = ros_typedb_tools.typedb_rule_diagram:main', + 'ros_typedb_stress_experiment = ros_typedb_tools.ros_typedb_stress_experiment:main', ], }, ) diff --git a/ros_typedb_tools/test/test_ros_typedb_stress_experiment.py b/ros_typedb_tools/test/test_ros_typedb_stress_experiment.py new file mode 100644 index 0000000..11e85f9 --- /dev/null +++ b/ros_typedb_tools/test/test_ros_typedb_stress_experiment.py @@ -0,0 +1,125 @@ +"""Tests for the ros_typedb stress experiment CLI helpers.""" + +import json +from pathlib import Path +from types import SimpleNamespace + +import pytest + +from ros_typedb_tools.ros_typedb_stress_experiment import QUERY_TYPE_BY_NAME +from ros_typedb_tools.ros_typedb_stress_experiment import RequestRecord +from ros_typedb_tools.ros_typedb_stress_experiment import build_argument_parser +from ros_typedb_tools.ros_typedb_stress_experiment import run_experiment +from ros_typedb_tools.ros_typedb_stress_experiment import summarize_records +from ros_typedb_tools.ros_typedb_stress_experiment import write_results + + +def _record( + index: int, + *, + latency_s: float, + success: bool, + timed_out: bool = False, +) -> RequestRecord: + return RequestRecord( + index=index, + started_at_s=10.0 + index, + ended_at_s=10.0 + index + latency_s, + latency_s=latency_s, + success=success, + timed_out=timed_out, + error_message='' if success else 'failed', + result_count=1 if success else 0, + ) + + +def test_parser_accepts_stage_one_arguments(): + """Check the Stage 1 command-line arguments.""" + parser = build_argument_parser() + + args = parser.parse_args( + [ + '--query', + 'match $x isa entity; fetch $x;', + '--query-type', + 'fetch', + '--requests', + '3', + '--timeout-s', + '2.5', + '--output', + 'results.json', + ] + ) + + assert args.service_name == '/ros_typedb_interface/query' + assert args.query_type == 'fetch' + assert args.requests == 3 + assert args.timeout_s == 2.5 + assert args.output == 'results.json' + + +def test_query_type_names_cover_query_service_constants(): + """Check all query service types are accepted by name.""" + assert set(QUERY_TYPE_BY_NAME) == { + 'define', + 'delete', + 'fetch', + 'get', + 'get_aggregate', + 'insert', + 'update', + } + + +def test_summarize_records_counts_success_failure_and_timeouts(): + """Check summary counters and latency metrics.""" + records = [ + _record(0, latency_s=0.1, success=True), + _record(1, latency_s=0.4, success=False), + _record(2, latency_s=0.2, success=False, timed_out=True), + ] + + summary = summarize_records(records) + + assert summary['total_requests'] == 3 + assert summary['successes'] == 1 + assert summary['failures'] == 2 + assert summary['timeouts'] == 1 + assert summary['latency_s']['min'] == pytest.approx(0.1) + assert summary['latency_s']['mean'] == pytest.approx(0.2333333333) + assert summary['latency_s']['max'] == pytest.approx(0.4) + + +def test_write_results_writes_summary_and_request_records(tmp_path: Path): + """Check JSON output includes metadata, summary, and records.""" + output_path = tmp_path / 'stress' / 'results.json' + records = [_record(0, latency_s=0.1, success=True)] + + write_results( + output_path, + service_name='/ros_typedb_interface/query', + query_type='fetch', + query='match $x isa entity; fetch $x;', + timeout_s=5.0, + records=records, + ) + + payload = json.loads(output_path.read_text(encoding='utf-8')) + assert payload['service_name'] == '/ros_typedb_interface/query' + assert payload['query_type'] == 'fetch' + assert payload['summary']['successes'] == 1 + assert payload['requests'][0]['index'] == 0 + assert payload['requests'][0]['success'] is True + + +def test_run_experiment_rejects_invalid_request_count(): + """Check request count validation before ROS service use.""" + args = SimpleNamespace( + requests=0, + timeout_s=5.0, + wait_service_timeout_s=1.0, + ) + + with pytest.raises(ValueError, match='--requests'): + run_experiment(args) From 17e9fd265a130eaeee7cf78381f0a0eac7648e63 Mon Sep 17 00:00:00 2001 From: Gustavo Date: Fri, 26 Jun 2026 16:24:47 -0300 Subject: [PATCH 15/36] Add concurrent TypeDB stress mode --- AGENTS.md | 132 ++++++++ README.md | 33 ++ ros_typedb_tools/README.md | 25 +- .../ros_typedb_stress_experiment.py | 302 ++++++++++++++---- .../test/test_ros_typedb_stress_experiment.py | 104 +++++- 5 files changed, 524 insertions(+), 72 deletions(-) create mode 100644 AGENTS.md diff --git a/AGENTS.md b/AGENTS.md new file mode 100644 index 0000000..18c5b6b --- /dev/null +++ b/AGENTS.md @@ -0,0 +1,132 @@ +# Repository Guidelines + +## Project Structure & Module Organization + +This repository contains three ROS 2 packages. `ros_typedb/` is the core +`ament_python` package, with runtime code in `ros_typedb/ros_typedb/`, launch +files in `ros_typedb/launch/`, and tests plus TypeDB fixtures in +`ros_typedb/test/`. `ros_typedb_msgs/` is the `ament_cmake` interface package; +message definitions live in `msg/` and services in `srv/`. `ros_typedb_tools/` +contains Python command-line tools for TypeDB schema and rule diagrams, with +tests and fixtures under `ros_typedb_tools/test/`. Sphinx documentation is in +`docs/source/`. + +## Build, Test, and Development Commands + +Run commands from the ROS workspace root unless noted: + +```bash +source /opt/ros/humble/setup.bash +rosdep install --from-paths src --ignore-src -r -y +colcon build --symlink-install +source install/setup.bash +colcon test --event-handlers console_cohesion+ --packages-up-to ros_typedb +colcon test --event-handlers console_cohesion+ --packages-select ros_typedb_tools +``` + +Use `ros2 run ros_typedb ros_typedb_interface -p schema_path:= -p data_path:=` +to run the node after building. From `docs/`, run `make html` to build docs. + +## Coding Style & Naming Conventions + +Python code follows ROS 2 `ament_flake8` and `ament_pep257` checks. Use 4-space +indentation, snake_case for modules/functions, PascalCase for classes, and +descriptive ROS parameter names. Keep console entry point names aligned with +`setup.py`, for example `typedb_schema_diagram`. Preserve existing package +layout and avoid mixing generated ROS interfaces with Python runtime modules. + +Use Serena for code navigation before and after non-trivial Python changes. +Prefer symbol overviews, symbol lookup, and reference searches to broad file +reads, and re-check Serena's symbol view after adding or renaming methods. + +## Testing Guidelines + +Tests use `pytest` through `colcon test`, with ROS lint wrappers in +`test/test_flake8.py`, `test/test_pep257.py`, and `test/test_copyright.py`. +Name new tests `test_*.py` and keep TypeDB sample data in the relevant +`test/fixtures/` or `test/typedb_test_data/` directory. Add focused tests for +query behavior, lifecycle behavior, and diagram output when those areas change. + +When the `ros_typedb` Docker container is running, prefer it for tests that need +the correct ROS Humble and TypeDB driver/server environment. The package test +fixtures use paths relative to the `ros_typedb/` package directory, so run +direct pytest from there, for example: + +```bash +docker exec ros_typedb bash -lc ' +cd /home/ubuntu-user/typedb_ws/src/ros_typedb/ros_typedb && +source /opt/ros/humble/setup.bash && +source /home/ubuntu-user/typedb_ws/install/setup.bash && +PYTHONPATH=/home/ubuntu-user/typedb_ws/src/ros_typedb/ros_typedb \ + python3 -m pytest -q test/test_typedb_interface_unit.py' +``` + +For ROS-facing direct pytest runs, preserve the ROS-provided `PYTHONPATH` and +prepend the package path instead of replacing it: + +```bash +PYTHONPATH=/home/ubuntu-user/typedb_ws/src/ros_typedb/ros_typedb:$PYTHONPATH \ + python3 -m pytest -q test/test_ros_typedb_interface_unit.py +``` + +For package-level verification, run from the container workspace root: + +```bash +docker exec ros_typedb bash -lc ' +cd /home/ubuntu-user/typedb_ws && +source /opt/ros/humble/setup.bash && +source install/setup.bash && +colcon test --event-handlers console_cohesion+ --packages-select ros_typedb' +``` + +Keep TypeDB interface tests split by dependency style: + +- `test_typedb_interface_unit.py`: fake drivers/sessions, timeout control flow, + formatting helpers, and Python-only failure paths. +- `test_typedb_interface_integration.py`: live TypeDB server tests using real + schema/data files. +- `test_ros_typedb_interface_unit.py`: ROS message conversion, direct callback + plumbing, and mocked lifecycle cleanup helpers. +- `test_ros_typedb_interface_integration.py`: launch tests, lifecycle service + transitions, and live ROS query service behavior. + +## TypeDB Runtime Notes + +Do not let `ensure_database_exists()` silently create an empty database. If the +configured database is missing, recreate it only by replaying stored schema/data +paths, or fail loudly when no initialization files are configured. Recovery code +that runs while `_database_query_lock` is held should use internal unlocked +helpers instead of calling the public `database_query()` path again; this avoids +recursive health checks and avoids needing `RLock`. + +`reload_schema=False` should skip schema redefinition only when reusing an +existing database. Newly created, force-recreated, or missing-and-recreated +databases must still load the configured schema before data is loaded. + +`delete_all_data()` must fail loudly if any delete step fails. Do not load new +data on top of potentially residual data after a failed cleanup. The current +live fixture confirms entity-first deletion works for the bundled schema/data, +but that does not prove the order is universal for every TypeDB schema. + +For TypeDB `fetch` queries, do not fetch a bare concept variable directly +(`fetch $x;`). Fetch attributes explicitly; to fetch all attributes owned by a +matched concept, use `fetch $x: attribute;`. + +For PR support notes under `docs-agents/`, keep changelogs brief: summary, +compact bullet list of changes, and verification commands. Avoid copying the +more verbose style from planning documents. + +## Commit & Pull Request Guidelines + +Recent history uses short imperative commits such as `fix lint` and +`Serialize TypeDB database queries`. Keep subjects concise and action-oriented; +include scope when it helps, such as `ros_typedb_tools: fix rule parsing`. +Pull requests should describe behavior changes, list the `colcon test` commands +run, link related issues, and include generated diagram screenshots or samples +when tool output changes. + +## Security & Configuration Tips + +Do not commit local TypeDB databases, credentials, or generated build/install/log +directories. This project targets Ubuntu 22.04, ROS Humble, and TypeDB 2.x; note +version changes in PRs because driver compatibility affects runtime behavior. diff --git a/README.md b/README.md index 94d24e5..c0e229d 100644 --- a/README.md +++ b/README.md @@ -134,6 +134,39 @@ ros2 run ros_typedb ros_typedb_interface -p schema_path:= -p data_p **Note:** Make sure to replace and with the real path for your schema and data file **Note 2:** Remember that ros_typedb_interface is a [LifeCycle](https://design.ros2.org/articles/node_lifecycle.html) node, so you need to change its state to active before using it. Check the [lifecycle tutorial](https://github.com/ros2/demos/tree/rolling/lifecycle). +### Run the stress experiment + +`ros_typedb_tools` provides a stress experiment CLI that sends repeated or +duration-based concurrent requests to the real `ros_typedb_interface/query` +service and reports latency, successes, failures, timeouts, and optional JSON +results. + +First, start TypeDB, run `ros_typedb_interface`, and configure the lifecycle +node so the query service is available. Then run: + +```bash +ros2 run ros_typedb_tools ros_typedb_stress_experiment \ + --query 'match $x isa entity; fetch $x: attribute;' \ + --query-type fetch \ + --requests 20 \ + --timeout-s 5 \ + --output /tmp/ros_typedb_stress_results.json +``` + +For concurrent read stress, omit `--query` and use the built-in read query mix: + +```bash +ros2 run ros_typedb_tools ros_typedb_stress_experiment \ + --clients 20 \ + --duration-s 60 \ + --timeout-s 10 \ + --mode read \ + --output /tmp/ros_typedb_read_stress_results.json +``` + +Use `--service-name` if the node name is different from the default +`/ros_typedb_interface/query`. + ## Extend the package To extend this package with custom functionalities, you can create a new ROS Node inheriting from ROSTypeDBInterface and a new typedb interface inheriting from TypeDBInterface. Then you simply need to add the new functionalities you need into your class. diff --git a/ros_typedb_tools/README.md b/ros_typedb_tools/README.md index 4f2a9af..18fddc4 100644 --- a/ros_typedb_tools/README.md +++ b/ros_typedb_tools/README.md @@ -99,7 +99,7 @@ Useful options: ## ros_typedb_stress_experiment -Runs a fixed-count stress experiment against the real +Runs fixed-count or duration-based stress experiments against the real `/ros_typedb_interface/query` service. Prerequisites: @@ -112,19 +112,34 @@ Example: ```bash ros2 run ros_typedb_tools ros_typedb_stress_experiment \ - --query 'match $p isa Plan; (plan: $p, action: $a) isa has_action; fetch $a: action_name;' \ + --query 'match $p isa Plan; (plan: $p, action: $a) isa has_action; fetch $a: attribute;' \ --query-type fetch \ --requests 2000 \ --timeout-s 5 \ --output ~/results/ros_typedb_stress_results.json ``` +Concurrent read stress example: + +```bash +ros2 run ros_typedb_tools ros_typedb_stress_experiment \ + --clients 20 \ + --duration-s 60 \ + --timeout-s 10 \ + --mode read \ + --output ~/results/ros_typedb_read_stress_results.json +``` + Useful options: - `--service-name`: query service name. Defaults to `/ros_typedb_interface/query` -- `--query`: TypeDB query to send on each request +- `--query`: optional TypeDB query to send on each request - `--query-type`: one of `define`, `delete`, `fetch`, `get`, - `get_aggregate`, `insert`, or `update` -- `--requests`: number of requests to send + `get_aggregate`, `insert`, or `update`; required when `--query` is used +- `--requests`: number of requests to send when `--duration-s` is omitted +- `--clients`: number of concurrent service clients +- `--duration-s`: run for this many seconds instead of a fixed request count +- `--mode`: built-in query mix to use when `--query` is omitted; currently + supports `read` - `--timeout-s`: per-request client and server timeout - `--output`: optional JSON results path diff --git a/ros_typedb_tools/ros_typedb_tools/ros_typedb_stress_experiment.py b/ros_typedb_tools/ros_typedb_tools/ros_typedb_stress_experiment.py index a6f9574..6cc2f0c 100644 --- a/ros_typedb_tools/ros_typedb_tools/ros_typedb_stress_experiment.py +++ b/ros_typedb_tools/ros_typedb_tools/ros_typedb_stress_experiment.py @@ -20,6 +20,7 @@ import statistics import sys import time +from collections.abc import Sequence from dataclasses import asdict from dataclasses import dataclass from pathlib import Path @@ -40,12 +41,36 @@ 'update': Query.Request.UPDATE, } +DEFAULT_READ_QUERY_SPECS = ( + ( + 'match $x isa entity; get $x;', + 'get', + ), + ( + 'match $x isa relation; get $x;', + 'get', + ), + ( + 'match $x isa attribute; get $x;', + 'get', + ), +) + + +@dataclass(frozen=True) +class QuerySpec: + """Query and service query type used by one request.""" + + query: str + query_type: str + @dataclass(frozen=True) class RequestRecord: """Result of one service request.""" index: int + client_id: int started_at_s: float ended_at_s: float latency_s: float @@ -59,7 +84,7 @@ def build_argument_parser() -> argparse.ArgumentParser: """Build command-line arguments for the stress experiment.""" parser = argparse.ArgumentParser( prog='ros_typedb_stress_experiment', - description='Run a fixed-count stress experiment against ros_typedb.', + description='Run a stress experiment against ros_typedb.', ) parser.add_argument( '--service-name', @@ -68,20 +93,40 @@ def build_argument_parser() -> argparse.ArgumentParser: ) parser.add_argument( '--query', - required=True, help='TypeDB query to send on each request.', ) parser.add_argument( '--query-type', - required=True, choices=sorted(QUERY_TYPE_BY_NAME), - help='TypeDB query type.', + help='TypeDB query type. Required when --query is used.', ) parser.add_argument( '--requests', type=int, default=1, - help='Number of requests to send. Defaults to 1.', + help=( + 'Number of requests to send when --duration-s is omitted. ' + 'Defaults to 1.' + ), + ) + parser.add_argument( + '--clients', + type=int, + default=1, + help='Number of concurrent service clients. Defaults to 1.', + ) + parser.add_argument( + '--duration-s', + type=float, + help=( + 'Run load for this many seconds instead of a fixed request count.' + ), + ) + parser.add_argument( + '--mode', + choices=('read',), + default='read', + help='Built-in query mix to use when --query is omitted.', ) parser.add_argument( '--timeout-s', @@ -104,9 +149,27 @@ def build_argument_parser() -> argparse.ArgumentParser: return parser +def _latency_percentile( + sorted_latencies: Sequence[float], + rank: float, +) -> float: + if not sorted_latencies: + raise ValueError('cannot compute percentile for empty sequence') + if len(sorted_latencies) == 1: + return sorted_latencies[0] + + position = (len(sorted_latencies) - 1) * rank + lower_index = int(position) + upper_index = min(lower_index + 1, len(sorted_latencies) - 1) + fraction = position - lower_index + lower_value = sorted_latencies[lower_index] + upper_value = sorted_latencies[upper_index] + return lower_value + ((upper_value - lower_value) * fraction) + + def summarize_records(records: list[RequestRecord]) -> dict[str, Any]: """Build summary metrics from request records.""" - latencies = [record.latency_s for record in records] + latencies = sorted(record.latency_s for record in records) successes = sum(1 for record in records if record.success) timeouts = sum(1 for record in records if record.timed_out) failures = len(records) - successes @@ -122,6 +185,10 @@ def summarize_records(records: list[RequestRecord]) -> dict[str, Any]: summary['latency_s'] = { 'min': min(latencies), 'mean': statistics.fmean(latencies), + 'p50': _latency_percentile(latencies, 0.50), + 'p90': _latency_percentile(latencies, 0.90), + 'p95': _latency_percentile(latencies, 0.95), + 'p99': _latency_percentile(latencies, 0.99), 'max': max(latencies), } return summary @@ -131,16 +198,26 @@ def write_results( output_path: Path, *, service_name: str, - query_type: str, - query: str, + query_type: str | None, + query: str | None, timeout_s: float, records: list[RequestRecord], + clients: int = 1, + duration_s: float | None = None, + mode: str = 'read', + query_mix: Sequence[QuerySpec] | None = None, ) -> None: """Write experiment results to a JSON file.""" payload = { 'service_name': service_name, + 'mode': mode, + 'clients': clients, + 'duration_s': duration_s, 'query_type': query_type, 'query': query, + 'query_mix': [ + asdict(query_spec) for query_spec in query_mix or () + ], 'timeout_s': timeout_s, 'summary': summarize_records(records), 'requests': [asdict(record) for record in records], @@ -163,6 +240,10 @@ def print_summary(summary: dict[str, Any]) -> None: if latency: print(f" latency_min_s: {latency['min']:.6f}") print(f" latency_mean_s: {latency['mean']:.6f}") + print(f" latency_p50_s: {latency['p50']:.6f}") + print(f" latency_p90_s: {latency['p90']:.6f}") + print(f" latency_p95_s: {latency['p95']:.6f}") + print(f" latency_p99_s: {latency['p99']:.6f}") print(f" latency_max_s: {latency['max']:.6f}") @@ -178,20 +259,107 @@ def _build_query_request( return request +def _build_query_specs(args: argparse.Namespace) -> list[QuerySpec]: + if args.query and not args.query_type: + raise ValueError('--query-type is required when --query is used') + if args.query_type and not args.query: + raise ValueError('--query-type requires --query') + + if args.query: + return [QuerySpec(query=args.query, query_type=args.query_type)] + + if args.mode == 'read': + return [ + QuerySpec(query=query, query_type=query_type) + for query, query_type in DEFAULT_READ_QUERY_SPECS + ] + + raise ValueError(f'unsupported mode: {args.mode}') + + +@dataclass +class _PendingRequest: + client_id: int + index: int + query_spec: QuerySpec + future: Any + started_at_s: float + + +def _record_from_pending( + pending: _PendingRequest, + *, + ended_at_s: float, + success: bool, + timed_out: bool, + error_message: str, + result_count: int, +) -> RequestRecord: + return RequestRecord( + index=pending.index, + client_id=pending.client_id, + started_at_s=pending.started_at_s, + ended_at_s=ended_at_s, + latency_s=ended_at_s - pending.started_at_s, + success=success, + timed_out=timed_out, + error_message=error_message, + result_count=result_count, + ) + + def run_experiment(args: argparse.Namespace) -> list[RequestRecord]: - """Run the fixed-count stress experiment.""" + """Run the stress experiment.""" if args.requests < 1: raise ValueError('--requests must be greater than zero') + if args.clients < 1: + raise ValueError('--clients must be greater than zero') + if args.duration_s is not None and args.duration_s <= 0: + raise ValueError('--duration-s must be greater than zero') if args.timeout_s <= 0: raise ValueError('--timeout-s must be greater than zero') if args.wait_service_timeout_s <= 0: raise ValueError('--wait-service-timeout-s must be greater than zero') + query_specs = _build_query_specs(args) node = rclpy.create_node('ros_typedb_stress_experiment') - client = node.create_client(Query, args.service_name) + clients = [ + node.create_client(Query, args.service_name) + for _ in range(args.clients) + ] records: list[RequestRecord] = [] + pending_by_client: dict[int, _PendingRequest] = {} + next_index = 0 + deadline_s = None + if args.duration_s is not None: + deadline_s = time.monotonic() + args.duration_s + + def should_send_request() -> bool: + if deadline_s is not None: + return time.monotonic() < deadline_s + return next_index < args.requests + + def start_request(client_id: int) -> None: + nonlocal next_index + query_spec = query_specs[next_index % len(query_specs)] + request = _build_query_request( + query_spec.query, + query_spec.query_type, + args.timeout_s, + ) + started_at_s = time.monotonic() + future = clients[client_id].call_async(request) + pending_by_client[client_id] = _PendingRequest( + client_id=client_id, + index=next_index, + query_spec=query_spec, + future=future, + started_at_s=started_at_s, + ) + next_index += 1 + try: - service_ready = client.wait_for_service( + service_ready = clients[0].wait_for_service( timeout_sec=args.wait_service_timeout_s ) if not service_ready: @@ -199,66 +367,65 @@ def run_experiment(args: argparse.Namespace) -> list[RequestRecord]: f'Query service is not available: {args.service_name}' ) - for index in range(args.requests): - request = _build_query_request( - args.query, - args.query_type, - args.timeout_s, - ) - started_at_s = time.monotonic() - future = client.call_async(request) - rclpy.spin_until_future_complete( - node, - future, - timeout_sec=args.timeout_s, - ) - ended_at_s = time.monotonic() - latency_s = ended_at_s - started_at_s - - if not future.done(): - records.append( - RequestRecord( - index=index, - started_at_s=started_at_s, - ended_at_s=ended_at_s, - latency_s=latency_s, - success=False, - timed_out=True, - error_message='client request timed out', - result_count=0, + while should_send_request() or pending_by_client: + for client_id in range(args.clients): + if client_id in pending_by_client: + continue + if should_send_request(): + start_request(client_id) + + rclpy.spin_once(node, timeout_sec=0.01) + now_s = time.monotonic() + completed_client_ids = [] + + for client_id, pending in pending_by_client.items(): + future = pending.future + if not future.done(): + if now_s - pending.started_at_s < args.timeout_s: + continue + records.append( + _record_from_pending( + pending, + ended_at_s=now_s, + success=False, + timed_out=True, + error_message='client request timed out', + result_count=0, + ) ) - ) - continue + completed_client_ids.append(client_id) + continue + + exception = future.exception() + if exception is not None: + records.append( + _record_from_pending( + pending, + ended_at_s=now_s, + success=False, + timed_out=False, + error_message=str(exception), + result_count=0, + ) + ) + completed_client_ids.append(client_id) + continue - exception = future.exception() - if exception is not None: + response = future.result() records.append( - RequestRecord( - index=index, - started_at_s=started_at_s, - ended_at_s=ended_at_s, - latency_s=latency_s, - success=False, + _record_from_pending( + pending, + ended_at_s=now_s, + success=response.success, timed_out=False, - error_message=str(exception), - result_count=0, + error_message=response.error_message, + result_count=len(response.results), ) ) - continue - - response = future.result() - records.append( - RequestRecord( - index=index, - started_at_s=started_at_s, - ended_at_s=ended_at_s, - latency_s=latency_s, - success=response.success, - timed_out=False, - error_message=response.error_message, - result_count=len(response.results), - ) - ) + completed_client_ids.append(client_id) + + for client_id in completed_client_ids: + del pending_by_client[client_id] finally: node.destroy_node() return records @@ -275,6 +442,7 @@ def main(argv: list[str] | None = None) -> int: summary = summarize_records(records) print_summary(summary) if args.output: + query_specs = _build_query_specs(args) write_results( Path(args.output), service_name=args.service_name, @@ -282,6 +450,10 @@ def main(argv: list[str] | None = None) -> int: query=args.query, timeout_s=args.timeout_s, records=records, + clients=args.clients, + duration_s=args.duration_s, + mode=args.mode, + query_mix=query_specs, ) return 0 if summary['failures'] == 0 else 1 except (RuntimeError, ValueError, OSError) as exc: diff --git a/ros_typedb_tools/test/test_ros_typedb_stress_experiment.py b/ros_typedb_tools/test/test_ros_typedb_stress_experiment.py index 11e85f9..a94dd4f 100644 --- a/ros_typedb_tools/test/test_ros_typedb_stress_experiment.py +++ b/ros_typedb_tools/test/test_ros_typedb_stress_experiment.py @@ -8,6 +8,7 @@ from ros_typedb_tools.ros_typedb_stress_experiment import QUERY_TYPE_BY_NAME from ros_typedb_tools.ros_typedb_stress_experiment import RequestRecord +from ros_typedb_tools.ros_typedb_stress_experiment import _build_query_specs from ros_typedb_tools.ros_typedb_stress_experiment import build_argument_parser from ros_typedb_tools.ros_typedb_stress_experiment import run_experiment from ros_typedb_tools.ros_typedb_stress_experiment import summarize_records @@ -23,6 +24,7 @@ def _record( ) -> RequestRecord: return RequestRecord( index=index, + client_id=0, started_at_s=10.0 + index, ended_at_s=10.0 + index + latency_s, latency_s=latency_s, @@ -40,7 +42,7 @@ def test_parser_accepts_stage_one_arguments(): args = parser.parse_args( [ '--query', - 'match $x isa entity; fetch $x;', + 'match $x isa entity; fetch $x: attribute;', '--query-type', 'fetch', '--requests', @@ -59,6 +61,30 @@ def test_parser_accepts_stage_one_arguments(): assert args.output == 'results.json' +def test_parser_accepts_stage_two_arguments(): + """Check Stage 2 duration and concurrency arguments.""" + parser = build_argument_parser() + + args = parser.parse_args( + [ + '--clients', + '20', + '--duration-s', + '60', + '--timeout-s', + '10', + '--mode', + 'read', + ] + ) + + assert args.clients == 20 + assert args.duration_s == 60 + assert args.mode == 'read' + assert args.query is None + assert args.query_type is None + + def test_query_type_names_cover_query_service_constants(): """Check all query service types are accepted by name.""" assert set(QUERY_TYPE_BY_NAME) == { @@ -88,6 +114,10 @@ def test_summarize_records_counts_success_failure_and_timeouts(): assert summary['timeouts'] == 1 assert summary['latency_s']['min'] == pytest.approx(0.1) assert summary['latency_s']['mean'] == pytest.approx(0.2333333333) + assert summary['latency_s']['p50'] == pytest.approx(0.2) + assert summary['latency_s']['p90'] == pytest.approx(0.36) + assert summary['latency_s']['p95'] == pytest.approx(0.38) + assert summary['latency_s']['p99'] == pytest.approx(0.396) assert summary['latency_s']['max'] == pytest.approx(0.4) @@ -100,26 +130,96 @@ def test_write_results_writes_summary_and_request_records(tmp_path: Path): output_path, service_name='/ros_typedb_interface/query', query_type='fetch', - query='match $x isa entity; fetch $x;', + query='match $x isa entity; fetch $x: attribute;', timeout_s=5.0, records=records, + clients=2, + duration_s=10.0, + mode='read', + query_mix=_build_query_specs( + SimpleNamespace(query=None, query_type=None, mode='read') + ), ) payload = json.loads(output_path.read_text(encoding='utf-8')) assert payload['service_name'] == '/ros_typedb_interface/query' + assert payload['clients'] == 2 + assert payload['duration_s'] == 10.0 + assert payload['mode'] == 'read' assert payload['query_type'] == 'fetch' + assert payload['query_mix'][0]['query_type'] in QUERY_TYPE_BY_NAME assert payload['summary']['successes'] == 1 assert payload['requests'][0]['index'] == 0 + assert payload['requests'][0]['client_id'] == 0 assert payload['requests'][0]['success'] is True +def test_build_query_specs_uses_single_explicit_query(): + """Check explicit query arguments override the built-in mix.""" + specs = _build_query_specs( + SimpleNamespace( + query='match $x isa entity; fetch $x: attribute;', + query_type='fetch', + mode='read', + ) + ) + + assert len(specs) == 1 + assert specs[0].query == 'match $x isa entity; fetch $x: attribute;' + assert specs[0].query_type == 'fetch' + + +def test_build_query_specs_uses_read_query_mix_by_default(): + """Check read mode provides runnable queries without explicit input.""" + specs = _build_query_specs( + SimpleNamespace(query=None, query_type=None, mode='read') + ) + + assert len(specs) > 1 + assert all(spec.query_type in QUERY_TYPE_BY_NAME for spec in specs) + + +def test_build_query_specs_rejects_query_without_query_type(): + """Check explicit queries still require an explicit query type.""" + args = SimpleNamespace( + query='match $x isa entity; fetch $x: attribute;', + query_type=None, + mode='read', + ) + + with pytest.raises(ValueError, match='--query-type'): + _build_query_specs(args) + + def test_run_experiment_rejects_invalid_request_count(): """Check request count validation before ROS service use.""" args = SimpleNamespace( requests=0, + clients=1, + duration_s=None, timeout_s=5.0, wait_service_timeout_s=1.0, + query='match $x isa entity; fetch $x: attribute;', + query_type='fetch', + mode='read', ) with pytest.raises(ValueError, match='--requests'): run_experiment(args) + + +def test_run_experiment_rejects_invalid_client_count(): + """Check client count validation before ROS service use.""" + args = SimpleNamespace( + requests=1, + clients=0, + duration_s=None, + timeout_s=5.0, + wait_service_timeout_s=1.0, + query='match $x isa entity; fetch $x: attribute;', + query_type='fetch', + mode='read', + ) + + with pytest.raises(ValueError, match='--clients'): + run_experiment(args) From ccce260958c05b3a794f4a99dd7faf2eb6745dc2 Mon Sep 17 00:00:00 2001 From: Gustavo Date: Fri, 26 Jun 2026 17:57:39 -0300 Subject: [PATCH 16/36] fix: support wildcard attribute fetch conversion --- ros_typedb/ros_typedb/ros_typedb_interface.py | 280 +++++++++++++++--- .../test_ros_typedb_interface_integration.py | 144 +++++++++ .../test/test_ros_typedb_interface_unit.py | 66 +++++ 3 files changed, 447 insertions(+), 43 deletions(-) diff --git a/ros_typedb/ros_typedb/ros_typedb_interface.py b/ros_typedb/ros_typedb/ros_typedb_interface.py index 36a13f7..d17401c 100644 --- a/ros_typedb/ros_typedb/ros_typedb_interface.py +++ b/ros_typedb/ros_typedb/ros_typedb_interface.py @@ -66,6 +66,10 @@ 'long_array': (ParameterType.PARAMETER_INTEGER_ARRAY, 'integer_array_value', 'long_array'), 'double_array': (ParameterType.PARAMETER_DOUBLE_ARRAY, 'double_array_value', 'double_array'), 'string_array': (ParameterType.PARAMETER_STRING_ARRAY, 'string_array_value', 'string_array'), + 'datetime_array': ( + ParameterType.PARAMETER_STRING_ARRAY, + 'string_array_value', + 'string_array'), } _TYPEDB_ROOT_TYPE_TO_QUERY_RESULT_TYPE = { @@ -111,7 +115,33 @@ def convert_attribute_dict_to_ros_msg( attr_name: str, attribute_value: List[Dict[str, Any]] | Dict[str, Any] ) -> Attribute: - + """ + Convert one concrete TypeDB attribute fetch result to a ROS attribute. + + ``attribute_value`` may be a single attribute dict or a homogeneous list of + attribute dicts. Homogeneous means every item has the same concrete + attribute label and value type. Wildcard fetch results such as + ``fetch $x: attribute;`` must be grouped by concrete attribute type before + calling this function. + + Example single attribute shape from ``fetch $age;``:: + + {'value': 33, + 'type': {'label': 'age', + 'root': 'attribute', + 'value_type': 'long'}} + + Example homogeneous list shape from ``fetch $x: phone-number;``:: + + [{'value': '+1-202-555-0106', + 'type': {'label': 'phone-number', + 'root': 'attribute', + 'value_type': 'string'}}, + {'value': '+44-1632-960007', + 'type': {'label': 'phone-number', + 'root': 'attribute', + 'value_type': 'string'}}] + """ attr = Attribute() attr.variable_name = attr_name @@ -127,9 +157,17 @@ def convert_attribute_dict_to_ros_msg( for value in attribute_value: current_type = value['type'] - if current_type['value_type'] != value_type or current_type['label'] != attr_label: + if ( + current_type['value_type'] != value_type or + current_type['label'] != attr_label + ): raise ValueError( - 'Inconsistent types or labels in attribute list') + f"""Inconsistent types or labels in attribute list: + attribute name: {attr_name} + attribute value: {attribute_value} + current_type: {current_type} + value type: {value_type} + attribute label: {attr_label}""") value_list.append(value['value']) @@ -147,53 +185,209 @@ def convert_attribute_dict_to_ros_msg( return attr -def fetch_result_to_ros_result_tree(json_obj, start_index=0): - result_tree = ResultTree() - index = start_index +def _attribute_type_key( + attribute_result: Dict[str, Any] +) -> tuple[str, str]: + """Return the concrete attribute label and value type for grouping.""" + attr_type = attribute_result['type'] + return attr_type['label'], attr_type['value_type'] - for key, values in json_obj.items(): - query_result = QueryResult() - query_result.result_index = index - index += 1 - if isinstance(values, dict) and 'type' in values: - result_type_info = values['type'] - result_type = result_type_info['root'] - result_label = result_type_info['label'] +def _group_wildcard_attribute_results( + attribute_results: List[Dict[str, Any]] +) -> Dict[tuple[str, str], List[Dict[str, Any]]]: + """ + Group ``fetch $x: attribute;`` results by concrete attribute type. - query_result.type = _TYPEDB_ROOT_TYPE_TO_QUERY_RESULT_TYPE[result_type] + TypeDB places all wildcard-owned attributes under an ``attribute`` key, so + one list can contain ``age`` longs, ``nickname`` strings, datetimes, etc. + The ordinary attribute converter expects homogeneous lists, so this groups + the wildcard list into homogeneous lists keyed by ``(label, value_type)``. + """ + grouped_attribute_values = {} + for attr_result in attribute_results: + attr_key = _attribute_type_key(attr_result) + grouped_attribute_values.setdefault(attr_key, []).append(attr_result) + return grouped_attribute_values - if result_type == 'attribute': - attr = convert_attribute_dict_to_ros_msg(key, values) - query_result.attribute = attr - else: - thing = Thing() - thing.type = _TYPEDB_ROOT_TYPE_TO_THING_TYPE[result_type] - thing.variable_name = key - thing.type_name = result_label - thing.attributes = [ - convert_attribute_dict_to_ros_msg(attr_name, attr_result_list) - for attr_name, attr_result_list in values.items() - if attr_name != 'type' - ] - query_result.thing = thing +def _iter_thing_attribute_groups( + thing_result: Dict[str, Any] +): + """ + Yield normalized ``(variable_name, attribute_result)`` pairs for a thing. + + TypeDB fetch returns two different shapes for thing attributes: + + * explicit fetches, e.g. ``fetch $x: email;``, are keyed by the concrete + attribute label (``email``) and already contain homogeneous lists; + * wildcard fetches, e.g. ``fetch $x: attribute;``, are keyed as + ``attribute`` and may contain mixed concrete labels and value types. + + This iterator hides that difference from the ROS message converter. + + Example explicit fetch shape:: + + {'type': {'label': 'person', 'root': 'entity'}, + 'email': [{'value': 'test@test.com', + 'type': {'label': 'email', + 'root': 'attribute', + 'value_type': 'string'}}]} + + Example wildcard fetch shape:: + + {'type': {'label': 'person', 'root': 'entity'}, + 'attribute': [ + {'value': 'test@test.com', + 'type': {'label': 'email', + 'root': 'attribute', + 'value_type': 'string'}}, + {'value': 33, + 'type': {'label': 'age', + 'root': 'attribute', + 'value_type': 'long'}}]} + """ + for attr_name, attr_result in thing_result.items(): + if attr_name == 'type': + continue + + if attr_name != 'attribute': + yield attr_name, attr_result + continue + + # A wildcard attribute may still be represented as one concrete + # attribute dict. Normalize its variable name to the concrete label. + if isinstance(attr_result, dict): + attr_label, _ = _attribute_type_key(attr_result) + yield attr_label, attr_result + continue + + # The common wildcard shape is a mixed list. Split it before conversion + # so each ROS Attribute contains only one concrete TypeDB attribute. + for attr_key, attr_group in _group_wildcard_attribute_results( + attr_result).items(): + attr_label, _ = attr_key + yield attr_label, attr_group + + +def convert_thing_attribute_dicts_to_ros_msgs( + attribute_values: Dict[str, Any] +) -> List[Attribute]: + """ + Convert TypeDB fetch thing attributes to ROS attributes. + + This function accepts a full entity/relation fetch result dict, including + its ``type`` key. Attribute shape normalization happens in + ``_iter_thing_attribute_groups``; this function only maps normalized groups + into ROS messages. + """ + return [ + convert_attribute_dict_to_ros_msg(attr_name, attr_result) + for attr_name, attr_result in _iter_thing_attribute_groups( + attribute_values) + ] + + +def _is_typedb_concept_fetch_result(values: Any) -> bool: + """Return true for fetch results that describe a TypeDB concept.""" + return isinstance(values, dict) and 'type' in values + + +def _is_subquery_fetch_result(values: Any) -> bool: + """Return true for fetch results that contain nested fetch results.""" + return isinstance(values, list) + + +def _concept_fetch_result_to_ros_msg( + variable_name: str, + values: Dict[str, Any], + result_index: int +) -> QueryResult: + """ + Convert an attribute/entity/relation fetch result to a QueryResult. + + TypeDB fetch result dicts use a ``type.root`` field to distinguish + top-level attributes from things. Top-level attributes become + ``QueryResult.ATTRIBUTE``. Entities and relations become + ``QueryResult.THING`` with their owned attributes attached. + """ + result_type_info = values['type'] + result_type = result_type_info['root'] + result_label = result_type_info['label'] + + query_result = QueryResult() + query_result.result_index = result_index + query_result.type = _TYPEDB_ROOT_TYPE_TO_QUERY_RESULT_TYPE[result_type] + + if result_type == 'attribute': + query_result.attribute = convert_attribute_dict_to_ros_msg( + variable_name, values) + return query_result + + thing = Thing() + thing.type = _TYPEDB_ROOT_TYPE_TO_THING_TYPE[result_type] + thing.variable_name = variable_name + thing.type_name = result_label + thing.attributes = convert_thing_attribute_dicts_to_ros_msgs(values) + query_result.thing = thing + return query_result + + +def _subquery_fetch_result_to_ros_msg( + subquery_name: str, + nested_results: List[Dict[str, Any]], + result_index: int +) -> tuple[QueryResult, List[QueryResult], int]: + """ + Convert a nested fetch result and return its flattened children. + + ROS result trees store subquery children in a flat ``results`` list. + ``children_index`` stores the index ranges for each nested match returned by + TypeDB. The returned integer is the next unused result index. + """ + query_result = QueryResult() + query_result.result_index = result_index + query_result.type = QueryResult.SUB_QUERY + query_result.sub_query_name = subquery_name + + next_index = result_index + 1 + children_results = [] + for value_dict in nested_results: + child_result_tree, child_last_index = fetch_result_to_ros_result_tree( + value_dict, next_index) + sub_tree_index_list = IndexList() + # The child tree uses a contiguous index range starting at next_index. + sub_tree_index_list.index = list(range(next_index, child_last_index)) + next_index = child_last_index + query_result.children_index.append(sub_tree_index_list) + children_results.extend(child_result_tree.results) + + return query_result, children_results, next_index + + +def fetch_result_to_ros_result_tree(json_obj, start_index=0): + """ + Convert one TypeDB fetch result dict to a flattened ROS result tree. + + ``json_obj`` is one item from the TypeDB fetch response list. Each key is + either a fetched variable/thing name or a subquery name. Concept results are + appended directly; subquery results are appended with their descendants + flattened immediately after the subquery node. + """ + result_tree = ResultTree() + index = start_index + + for key, values in json_obj.items(): + if _is_typedb_concept_fetch_result(values): + query_result = _concept_fetch_result_to_ros_msg( + key, values, index) result_tree.results.append(query_result) - elif isinstance(values, list): - query_result.type = QueryResult.SUB_QUERY - query_result.sub_query_name = key - - children_results = [] - for value_dict in values: - child_result_tree, child_last_index = fetch_result_to_ros_result_tree( - value_dict, index) - sub_tree_index_list = IndexList() - sub_tree_index_list.index = list( - range(index, child_last_index)) - index = child_last_index - query_result.children_index.append(sub_tree_index_list) - children_results.extend(child_result_tree.results) + index += 1 + continue + if _is_subquery_fetch_result(values): + query_result, children_results, index = ( + _subquery_fetch_result_to_ros_msg(key, values, index)) result_tree.results.append(query_result) result_tree.results.extend(children_results) diff --git a/ros_typedb/test/test_ros_typedb_interface_integration.py b/ros_typedb/test/test_ros_typedb_interface_integration.py index 1097423..9b706e4 100644 --- a/ros_typedb/test/test_ros_typedb_interface_integration.py +++ b/ros_typedb/test/test_ros_typedb_interface_integration.py @@ -578,6 +578,150 @@ def strip_indices(obj): assert any(strip_indices(item) == strip_indices(a) for a in actual) +@pytest.mark.launch(fixture=generate_test_description) +def test_ros_typedb_fetch_query_wildcard_attributes(test_node, insert_query): + """Fetch all owned attributes without crashing on mixed attribute types.""" + test_node.activate_ros_typedb() + + test_node.call_service(test_node.query_cli, insert_query) + + query_req = Query.Request() + query_req.query_type = query_req.FETCH + query_req.query = """ + match $entity isa person, has email 'test@test.com'; + fetch $entity: attribute; + """ + query_res = test_node.call_service(test_node.query_cli, query_req) + + assert query_res.success is True + assert len(query_res.results) == 1 + assert len(query_res.results[0].results) == 1 + + thing = query_res.results[0].results[0].thing + attributes_by_label = { + attr.label: attr + for attr in thing.attributes + } + + assert thing.variable_name == 'entity' + assert thing.type_name == 'person' + + email_attr = attributes_by_label['email'] + assert email_attr.variable_name == 'email' + assert email_attr.value.type == ParameterType.PARAMETER_STRING_ARRAY + assert email_attr.value.string_array_value == ['test@test.com'] + + age_attr = attributes_by_label['age'] + assert age_attr.variable_name == 'age' + assert age_attr.value.type == ParameterType.PARAMETER_INTEGER_ARRAY + assert list(age_attr.value.integer_array_value) == [33] + + alive_attr = attributes_by_label['alive'] + assert alive_attr.variable_name == 'alive' + assert alive_attr.value.type == ParameterType.PARAMETER_BOOL_ARRAY + assert list(alive_attr.value.bool_array_value) == [True] + + birth_date_attr = attributes_by_label['birth-date'] + assert birth_date_attr.variable_name == 'birth-date' + assert birth_date_attr.value.type == ParameterType.PARAMETER_STRING_ARRAY + assert birth_date_attr.value.string_array_value == [ + '1990-06-01T00:00:00.000' + ] + + height_attr = attributes_by_label['height'] + assert height_attr.variable_name == 'height' + assert height_attr.value.type == ParameterType.PARAMETER_DOUBLE_ARRAY + assert list(height_attr.value.double_array_value) == [1.0] + + nickname_attr = attributes_by_label['nickname'] + assert nickname_attr.variable_name == 'nickname' + assert nickname_attr.value.type == ParameterType.PARAMETER_STRING_ARRAY + assert nickname_attr.value.string_array_value == ['test'] + + +@pytest.mark.launch(fixture=generate_test_description) +def test_ros_typedb_fetch_query_service_survives_wildcard_fetch( + test_node, insert_query): + """Regression: wildcard fetch conversion must not kill future service use.""" + test_node.activate_ros_typedb() + + test_node.call_service(test_node.query_cli, insert_query) + + query_req = Query.Request() + query_req.query_type = query_req.FETCH + query_req.query = """ + match $entity isa person, has email 'test@test.com'; + fetch $entity: attribute; + """ + query_res = test_node.call_service(test_node.query_cli, query_req) + + followup_req = Query.Request() + followup_req.query_type = followup_req.GET_AGGREGATE + followup_req.query = """ + match $entity isa person, has email 'test@test.com'; + get $entity; + count; + """ + followup_res = test_node.call_service(test_node.query_cli, followup_req) + + assert query_res.success is True + assert followup_res.success is True + count_value = followup_res.results[0].results[0].attribute.value + assert count_value.integer_value == 1 + + +@pytest.mark.launch(fixture=generate_test_description) +def test_ros_typedb_fetch_query_nested_attribute_wildcard( + test_node, insert_query): + """Fetch nested wildcard attributes without mixing concrete labels.""" + test_node.activate_ros_typedb() + + test_node.call_service(test_node.query_cli, insert_query) + + query_req = Query.Request() + query_req.query_type = query_req.FETCH + query_req.query = """ + match + $company_var isa company, has name 'TU Delft'; + fetch + $company_var: attribute; + employee_data: { + match + $employment_var (employer: $company_var, + employee: $employee_var) isa employment; + $employee_var has email 'phd@tudelft.nl'; + fetch + $employee_var: attribute; + $employment_var: attribute; + }; + """ + query_res = test_node.call_service(test_node.query_cli, query_req) + + assert query_res.success is True + assert len(query_res.results) == 1 + + results = query_res.results[0].results + company = results[0].thing + assert company.variable_name == 'company_var' + assert sorted(attr.label for attr in company.attributes) == [ + 'address', 'name' + ] + + child_things = [ + result.thing + for result in results + if result.type == QueryResult.THING + and result.thing.variable_name in ('employee_var', 'employment_var') + ] + labels_by_variable = { + thing.variable_name: sorted(attr.label for attr in thing.attributes) + for thing in child_things + } + + assert labels_by_variable['employee_var'] == ['email', 'full-name'] + assert labels_by_variable['employment_var'] == ['role-name', 'salary'] + + @pytest.mark.launch(fixture=generate_test_description) def test_ros_typedb_get_query(test_node, insert_query): test_node.activate_ros_typedb() diff --git a/ros_typedb/test/test_ros_typedb_interface_unit.py b/ros_typedb/test/test_ros_typedb_interface_unit.py index 3132134..fa66d39 100644 --- a/ros_typedb/test/test_ros_typedb_interface_unit.py +++ b/ros_typedb/test/test_ros_typedb_interface_unit.py @@ -387,6 +387,72 @@ def test_convert_attribute_dict_to_ros_msg(): assert expected_nick_attr == nick_attr +def test_fetch_result_to_ros_result_tree_groups_wildcard_attributes(): + """Wildcard attribute fetches group values by concrete attribute label.""" + json_test = { + 'entity': { + 'attribute': [ + {'value': 'test@test.com', + 'type': {'label': 'email', + 'root': 'attribute', + 'value_type': 'string'}}, + {'value': 'test', + 'type': {'label': 'nickname', + 'root': 'attribute', + 'value_type': 'string'}}, + {'value': 33, + 'type': {'label': 'age', + 'root': 'attribute', + 'value_type': 'long'}}, + {'value': True, + 'type': {'label': 'alive', + 'root': 'attribute', + 'value_type': 'boolean'}}, + {'value': '1990-06-01T00:00:00.000', + 'type': {'label': 'birth-date', + 'root': 'attribute', + 'value_type': 'datetime'}}, + {'value': 'alias', + 'type': {'label': 'nickname', + 'root': 'attribute', + 'value_type': 'string'}}, + ], + 'type': {'label': 'person', 'root': 'entity'} + } + } + + result_tree, _ = fetch_result_to_ros_result_tree(json_test) + + assert len(result_tree.results) == 1 + thing = result_tree.results[0].thing + attributes_by_label = { + attr.label: attr + for attr in thing.attributes + } + + assert thing.variable_name == 'entity' + assert thing.type_name == 'person' + assert sorted(attributes_by_label) == [ + 'age', 'alive', 'birth-date', 'email', 'nickname' + ] + assert attributes_by_label['email'].variable_name == 'email' + assert attributes_by_label['email'].value.string_array_value == [ + 'test@test.com' + ] + assert attributes_by_label['nickname'].variable_name == 'nickname' + assert attributes_by_label['nickname'].value.string_array_value == [ + 'test', 'alias' + ] + assert attributes_by_label['age'].variable_name == 'age' + assert list(attributes_by_label['age'].value.integer_array_value) == [33] + assert attributes_by_label['alive'].variable_name == 'alive' + assert list(attributes_by_label['alive'].value.bool_array_value) == [True] + assert attributes_by_label['birth-date'].variable_name == 'birth-date' + assert attributes_by_label['birth-date'].value.string_array_value == [ + '1990-06-01T00:00:00.000' + ] + + def test_fetch_result_to_ros_result_tree(): json_test = { 'age': { From 90dc99d865d2ca3c5e4bf58664423c78bfec0817 Mon Sep 17 00:00:00 2001 From: Gustavo Date: Fri, 26 Jun 2026 17:58:00 -0300 Subject: [PATCH 17/36] fix: catch query service conversion errors --- ros_typedb/ros_typedb/ros_typedb_interface.py | 26 ++++++++++---- .../test/test_ros_typedb_interface_unit.py | 34 +++++++++++++++++++ 2 files changed, 53 insertions(+), 7 deletions(-) diff --git a/ros_typedb/ros_typedb/ros_typedb_interface.py b/ros_typedb/ros_typedb/ros_typedb_interface.py index d17401c..413d871 100644 --- a/ros_typedb/ros_typedb/ros_typedb_interface.py +++ b/ros_typedb/ros_typedb/ros_typedb_interface.py @@ -733,14 +733,26 @@ def query_service_cb( return response per_call_timeout = req.timeout_s if req.timeout_s > 0 else None - query_result = query_func(req.query, timeout=per_call_timeout) - response = query_result_to_ros_msg(req.query_type, query_result) - if query_result is None: + try: + query_result = query_func(req.query, timeout=per_call_timeout) + response = query_result_to_ros_msg( + req.query_type, query_result) + if query_result is None: + response.success = False + response.error_message = self.typedb_interface.last_error + else: + response.success = True + return response + except Exception as exc: + try: + self.get_logger().error( + 'Query service failed: {}\n{}'.format( + exc, traceback.format_exc())) + except Exception: + pass response.success = False - response.error_message = self.typedb_interface.last_error - else: - response.success = True - return response + response.error_message = str(exc) + return response def delete_db_cb( self, diff --git a/ros_typedb/test/test_ros_typedb_interface_unit.py b/ros_typedb/test/test_ros_typedb_interface_unit.py index fa66d39..cae0c8a 100644 --- a/ros_typedb/test/test_ros_typedb_interface_unit.py +++ b/ros_typedb/test/test_ros_typedb_interface_unit.py @@ -20,6 +20,7 @@ from rcl_interfaces.msg import ParameterType from rclpy.lifecycle import TransitionCallbackReturn +import ros_typedb.ros_typedb_interface as ros_typedb_interface from ros_typedb.ros_typedb_interface import convert_attribute_dict_to_ros_msg from ros_typedb.ros_typedb_interface import fetch_result_to_ros_result_tree from ros_typedb.ros_typedb_interface import ROSTypeDBInterface @@ -131,6 +132,39 @@ def fetch_database(self, query, timeout=None): assert captured.get('timeout') is None +def test_query_service_cb_catches_query_result_conversion_error(monkeypatch): + """query_service_cb returns an error response when conversion fails.""" + + class FakeTypeDBInterface: + last_error = '' + + def fetch_database(self, query, timeout=None): + return [{'unexpected': 'shape'}] + + def raise_conversion_error(query_type, query_result): + raise ValueError('conversion failed') + + node = ROSTypeDBInterface.__new__(ROSTypeDBInterface) + node.typedb_interface = FakeTypeDBInterface() + logger = MagicMock() + node.get_logger = MagicMock(return_value=logger) + monkeypatch.setattr( + ros_typedb_interface, + 'query_result_to_ros_msg', + raise_conversion_error) + + req = Query.Request() + req.query_type = Query.Request.FETCH + req.query = 'match $x isa thing; fetch $x: attribute;' + + response = Query.Response() + result = node.query_service_cb(req, response) + + assert result.success is False + assert result.error_message == 'conversion failed' + logger.error.assert_called_once() + + class MockTypeDBDriver: def __init__(self): From 55b457f0ff59ae74728f744dd9120d335828516c Mon Sep 17 00:00:00 2001 From: Gustavo Date: Sat, 27 Jun 2026 13:15:22 -0300 Subject: [PATCH 18/36] Add stress timeout diagnostics --- ros_typedb_tools/README.md | 80 +++ .../ros_typedb_stress_experiment.py | 501 ++++++++++++++++-- .../scripts/run_stage2_timeout_scenarios.sh | 84 +++ ros_typedb_tools/setup.py | 27 +- .../test/test_ros_typedb_stress_experiment.py | 140 ++++- 5 files changed, 765 insertions(+), 67 deletions(-) create mode 100755 ros_typedb_tools/scripts/run_stage2_timeout_scenarios.sh diff --git a/ros_typedb_tools/README.md b/ros_typedb_tools/README.md index 18fddc4..857cd51 100644 --- a/ros_typedb_tools/README.md +++ b/ros_typedb_tools/README.md @@ -127,9 +127,57 @@ ros2 run ros_typedb_tools ros_typedb_stress_experiment \ --duration-s 60 \ --timeout-s 10 \ --mode read \ + --debug-events-output ~/results/ros_typedb_read_stress_events.jsonl \ --output ~/results/ros_typedb_read_stress_results.json ``` +```bash +ros2 run ros_typedb_tools ros_typedb_stress_experiment \ + --clients 20 \ + --duration-s 60 \ + --timeout-s 10 \ + --query 'match $x isa entity; get $x;' \ + --query-type get \ + --debug-events-output ~/results/ros_typedb_read_stress_events.jsonl \ + --output ~/results/ros_typedb_get_stress_results.json +``` + +When `--output` is set, timeout-only records are also written next to the main +result file using the suffix `_timeouts.json`. Use `--timeout-output` to choose +a different path. Timeout records include request index, client id, query index, +query type, query text, latency, and whether future cancellation was requested. +The command exits successfully when the experiment runs to completion, even if +some requests fail. Use `--fail-on-failure` when a nonzero exit code is useful +for CI or scripted thresholds. + +For ROS client/executor debugging, `--debug-events-output` writes JSONL events +for request sends, `spin_once()` calls, completed futures, timeouts, and periodic +pending-request snapshots. Use `--executor single` or `--executor multi` to +compare explicit executors against the default `rclpy.spin_once(node)` behavior. +Use `--request-gap-s` to add a small per-client delay between requests, or +`--max-in-flight` to cap total outstanding service requests independently of the +number of clients. + +To isolate ROS service/client behavior from TypeDB work, run a fake Query +service in one terminal: + +```bash +ros2 run ros_typedb_tools ros_typedb_fake_query_service +``` + +Then run the same stress command against it from another terminal. + +To run the common timeout-debug scenarios in sequence, use: + +```bash +bash src/ros_typedb/ros_typedb_tools/scripts/run_stage2_timeout_scenarios.sh +``` + +The script runs real read/global, real read/multi, fake read/global, fake +read/multi, and real explicit entity-get/global scenarios. Override defaults +with environment variables such as `CLIENTS`, `DURATION_S`, `TIMEOUT_S`, and +`OUTPUT_DIR`. + Useful options: - `--service-name`: query service name. Defaults to `/ros_typedb_interface/query` @@ -141,5 +189,37 @@ Useful options: - `--duration-s`: run for this many seconds instead of a fixed request count - `--mode`: built-in query mix to use when `--query` is omitted; currently supports `read` +- `--request-gap-s`: minimum delay between requests from the same client +- `--max-in-flight`: maximum number of outstanding requests across all clients +- `--executor`: client executor mode, one of `global`, `single`, or `multi` +- `--debug-events-output`: optional JSONL debug event path - `--timeout-s`: per-request client and server timeout - `--output`: optional JSON results path +- `--timeout-output`: optional timeout-only JSON results path +- `--fail-on-failure`: return exit code 1 when any request fails + +## Tests + +```Bash +colcon test --event-handlers console_cohesion+ --packages-select ros_typedb_tools +``` + +## Docker + +The commands below assume you are running them from the workspace root: + +```Bash +cd +``` + +Start dev container **without** display and the `ros_typedb` directory mounted: + +```Bash +docker run -it --rm --name ros_typedb -v /etc/localtime:/etc/localtime:ro -v $PWD/src/ros_typedb:/home/ubuntu-user/typedb_ws/src/ros_typedb -v $PWD/ros_typedb_results/:/home/ubuntu-user/results/ ros_typedb +``` + +Start new terminal in the container: + +```Bash +docker exec -it ros_typedb bash +``` diff --git a/ros_typedb_tools/ros_typedb_tools/ros_typedb_stress_experiment.py b/ros_typedb_tools/ros_typedb_tools/ros_typedb_stress_experiment.py index 6cc2f0c..91742bf 100644 --- a/ros_typedb_tools/ros_typedb_tools/ros_typedb_stress_experiment.py +++ b/ros_typedb_tools/ros_typedb_tools/ros_typedb_stress_experiment.py @@ -27,6 +27,8 @@ from typing import Any import rclpy +from rclpy.executors import MultiThreadedExecutor +from rclpy.executors import SingleThreadedExecutor from ros_typedb_msgs.srv import Query @@ -71,11 +73,15 @@ class RequestRecord: index: int client_id: int + query_index: int + query_type: str + query: str started_at_s: float ended_at_s: float latency_s: float success: bool timed_out: bool + cancel_requested: bool error_message: str result_count: int @@ -122,6 +128,38 @@ def build_argument_parser() -> argparse.ArgumentParser: 'Run load for this many seconds instead of a fixed request count.' ), ) + parser.add_argument( + '--request-gap-s', + type=float, + default=0.0, + help=( + 'Minimum delay between requests from the same client. Defaults ' + 'to 0.' + ), + ) + parser.add_argument( + '--max-in-flight', + type=int, + help=( + 'Maximum number of outstanding requests across all clients. ' + 'Defaults to the number of clients.' + ), + ) + parser.add_argument( + '--executor', + choices=('global', 'single', 'multi'), + default='global', + help=( + 'Client-side executor used for spinning. Defaults to global, ' + 'matching rclpy.spin_once(node).' + ), + ) + parser.add_argument( + '--executor-threads', + type=int, + default=2, + help='Thread count for --executor multi. Defaults to 2.', + ) parser.add_argument( '--mode', choices=('read',), @@ -146,6 +184,58 @@ def build_argument_parser() -> argparse.ArgumentParser: '--output', help='Optional JSON output path.', ) + parser.add_argument( + '--timeout-output', + help=( + 'Optional JSON path for timeout-only records. Defaults to ' + '_timeouts.json when --output is set.' + ), + ) + parser.add_argument( + '--debug-events-output', + help='Optional JSONL path for request/future/executor debug events.', + ) + parser.add_argument( + '--fail-on-failure', + action='store_true', + help=( + 'Exit with status 1 when any request fails. By default, request ' + 'failures are reported in the summary and JSON output without ' + 'making the tool command fail.' + ), + ) + return parser + + +def build_fake_service_argument_parser() -> argparse.ArgumentParser: + """Build command-line arguments for the fake Query service.""" + parser = argparse.ArgumentParser( + prog='ros_typedb_fake_query_service', + description='Serve the ros_typedb Query service without TypeDB work.', + ) + parser.add_argument( + '--service-name', + default='/ros_typedb_interface/query', + help='Query service name. Defaults to /ros_typedb_interface/query.', + ) + parser.add_argument( + '--response-delay-s', + type=float, + default=0.0, + help='Optional delay before each response. Defaults to 0.', + ) + parser.add_argument( + '--executor', + choices=('single', 'multi'), + default='single', + help='Server executor. Defaults to single.', + ) + parser.add_argument( + '--executor-threads', + type=int, + default=2, + help='Thread count for --executor multi. Defaults to 2.', + ) return parser @@ -206,6 +296,9 @@ def write_results( duration_s: float | None = None, mode: str = 'read', query_mix: Sequence[QuerySpec] | None = None, + request_gap_s: float = 0.0, + max_in_flight: int | None = None, + executor: str = 'global', ) -> None: """Write experiment results to a JSON file.""" payload = { @@ -213,6 +306,9 @@ def write_results( 'mode': mode, 'clients': clients, 'duration_s': duration_s, + 'request_gap_s': request_gap_s, + 'max_in_flight': max_in_flight, + 'executor': executor, 'query_type': query_type, 'query': query, 'query_mix': [ @@ -229,6 +325,74 @@ def write_results( ) +def default_timeout_output_path(output_path: Path) -> Path: + """Build the default timeout-only output path for a result path.""" + suffix = output_path.suffix or '.json' + stem = output_path.stem if output_path.suffix else output_path.name + return output_path.with_name(f'{stem}_timeouts{suffix}') + + +def write_timeout_results( + output_path: Path, + *, + source_output_path: Path | None, + records: list[RequestRecord], +) -> None: + """Write timeout-only request records to a JSON file.""" + timeout_records = [ + record + for record in records + if record.timed_out + ] + payload = { + 'source_output_path': ( + str(source_output_path) if source_output_path is not None else None + ), + 'summary': { + 'timeouts': len(timeout_records), + 'total_requests': len(records), + }, + 'timeouts': [asdict(record) for record in timeout_records], + } + output_path.parent.mkdir(parents=True, exist_ok=True) + output_path.write_text( + json.dumps(payload, indent=2, sort_keys=True) + '\n', + encoding='utf-8', + ) + + +class DebugEventWriter: + """Write compact JSONL events for diagnosing client-side scheduling.""" + + def __init__(self, output_path: Path | None) -> None: + """Create an event writer for the optional output path.""" + self._output_path = output_path + self._stream = None + + def __enter__(self) -> 'DebugEventWriter': + """Open the event stream when debug output is enabled.""" + if self._output_path is not None: + self._output_path.parent.mkdir(parents=True, exist_ok=True) + self._stream = self._output_path.open('w', encoding='utf-8') + return self + + def __exit__(self, *args: object) -> None: + """Close the event stream when it was opened.""" + if self._stream is not None: + self._stream.close() + + def log(self, event: str, **fields: Any) -> None: + """Append one debug event if debug output is enabled.""" + if self._stream is None: + return + payload = { + 'event': event, + 'monotonic_s': time.monotonic(), + **fields, + } + self._stream.write(json.dumps(payload, sort_keys=True) + '\n') + + def print_summary(summary: dict[str, Any]) -> None: """Print compact experiment summary.""" latency = summary['latency_s'] @@ -281,28 +445,51 @@ def _build_query_specs(args: argparse.Namespace) -> list[QuerySpec]: class _PendingRequest: client_id: int index: int + query_index: int query_spec: QuerySpec future: Any started_at_s: float +def _build_executor(args: argparse.Namespace) -> Any | None: + if args.executor == 'global': + return None + if args.executor == 'single': + return SingleThreadedExecutor() + if args.executor == 'multi': + return MultiThreadedExecutor(num_threads=args.executor_threads) + raise ValueError(f'unsupported executor: {args.executor}') + + +def _spin_once(node: Any, executor: Any | None) -> None: + if executor is None: + rclpy.spin_once(node, timeout_sec=0.01) + else: + executor.spin_once(timeout_sec=0.01) + + def _record_from_pending( pending: _PendingRequest, *, ended_at_s: float, success: bool, timed_out: bool, + cancel_requested: bool, error_message: str, result_count: int, ) -> RequestRecord: return RequestRecord( index=pending.index, client_id=pending.client_id, + query_index=pending.query_index, + query_type=pending.query_spec.query_type, + query=pending.query_spec.query, started_at_s=pending.started_at_s, ended_at_s=ended_at_s, latency_s=ended_at_s - pending.started_at_s, success=success, timed_out=timed_out, + cancel_requested=cancel_requested, error_message=error_message, result_count=result_count, ) @@ -320,117 +507,281 @@ def run_experiment(args: argparse.Namespace) -> list[RequestRecord]: raise ValueError('--timeout-s must be greater than zero') if args.wait_service_timeout_s <= 0: raise ValueError('--wait-service-timeout-s must be greater than zero') + if args.request_gap_s < 0: + raise ValueError( + '--request-gap-s must be greater than or equal to zero' + ) + if args.max_in_flight is not None and args.max_in_flight < 1: + raise ValueError('--max-in-flight must be greater than zero') + if args.executor_threads < 1: + raise ValueError('--executor-threads must be greater than zero') query_specs = _build_query_specs(args) + max_in_flight = args.max_in_flight or args.clients + debug_events_output = ( + Path(args.debug_events_output).expanduser() + if args.debug_events_output + else None + ) node = rclpy.create_node('ros_typedb_stress_experiment') + executor = _build_executor(args) + if executor is not None: + executor.add_node(node) clients = [ node.create_client(Query, args.service_name) for _ in range(args.clients) ] records: list[RequestRecord] = [] pending_by_client: dict[int, _PendingRequest] = {} + last_started_by_client = [float('-inf') for _ in range(args.clients)] next_index = 0 deadline_s = None if args.duration_s is not None: deadline_s = time.monotonic() + args.duration_s + next_snapshot_s = time.monotonic() + 1.0 def should_send_request() -> bool: if deadline_s is not None: return time.monotonic() < deadline_s return next_index < args.requests - def start_request(client_id: int) -> None: + def start_request(client_id: int, started_at_s: float) -> None: nonlocal next_index - query_spec = query_specs[next_index % len(query_specs)] + query_index = next_index % len(query_specs) + query_spec = query_specs[query_index] request = _build_query_request( query_spec.query, query_spec.query_type, args.timeout_s, ) - started_at_s = time.monotonic() future = clients[client_id].call_async(request) pending_by_client[client_id] = _PendingRequest( client_id=client_id, index=next_index, + query_index=query_index, query_spec=query_spec, future=future, started_at_s=started_at_s, ) + last_started_by_client[client_id] = started_at_s next_index += 1 try: - service_ready = clients[0].wait_for_service( - timeout_sec=args.wait_service_timeout_s - ) - if not service_ready: - raise RuntimeError( - f'Query service is not available: {args.service_name}' + with DebugEventWriter(debug_events_output) as debug_events: + debug_events.log( + 'experiment_started', + clients=args.clients, + executor=args.executor, + max_in_flight=max_in_flight, + request_gap_s=args.request_gap_s, ) + wait_deadline_s = time.monotonic() + args.wait_service_timeout_s + for client_id, client in enumerate(clients): + wait_timeout_s = max(0.0, wait_deadline_s - time.monotonic()) + service_ready = client.wait_for_service( + timeout_sec=wait_timeout_s + ) + debug_events.log( + 'service_wait_finished', + client_id=client_id, + ready=service_ready, + ) + if not service_ready: + raise RuntimeError( + f'Query service is not available: {args.service_name}' + ) - while should_send_request() or pending_by_client: - for client_id in range(args.clients): - if client_id in pending_by_client: - continue - if should_send_request(): - start_request(client_id) - - rclpy.spin_once(node, timeout_sec=0.01) - now_s = time.monotonic() - completed_client_ids = [] - - for client_id, pending in pending_by_client.items(): - future = pending.future - if not future.done(): - if now_s - pending.started_at_s < args.timeout_s: - continue - records.append( - _record_from_pending( - pending, - ended_at_s=now_s, - success=False, - timed_out=True, - error_message='client request timed out', - result_count=0, + while should_send_request() or pending_by_client: + spin_started_s = time.monotonic() + _spin_once(node, executor) + now_s = time.monotonic() + debug_events.log( + 'spin_once', + duration_s=now_s - spin_started_s, + pending_count=len(pending_by_client), + ) + if now_s >= next_snapshot_s: + debug_events.log( + 'pending_snapshot', + next_index=next_index, + pending_count=len(pending_by_client), + pending_indices=[ + pending.index + for pending in pending_by_client.values() + ], + ) + next_snapshot_s = now_s + 1.0 + + completed_client_ids = [] + timed_out_this_cycle = False + + for client_id, pending in pending_by_client.items(): + future = pending.future + if not future.done(): + if now_s - pending.started_at_s < args.timeout_s: + continue + timed_out_this_cycle = True + future.cancel() + records.append( + _record_from_pending( + pending, + ended_at_s=now_s, + success=False, + timed_out=True, + cancel_requested=True, + error_message='client request timed out', + result_count=0, + ) ) + debug_events.log( + 'request_timeout', + client_id=client_id, + index=pending.index, + latency_s=now_s - pending.started_at_s, + pending_count=len(pending_by_client), + query_index=pending.query_index, + ) + completed_client_ids.append(client_id) + continue + + debug_events.log( + 'future_done', + client_id=client_id, + index=pending.index, + latency_s=now_s - pending.started_at_s, + pending_count=len(pending_by_client), + query_index=pending.query_index, ) - completed_client_ids.append(client_id) - continue + exception = future.exception() + if exception is not None: + records.append( + _record_from_pending( + pending, + ended_at_s=now_s, + success=False, + timed_out=False, + cancel_requested=False, + error_message=str(exception), + result_count=0, + ) + ) + debug_events.log( + 'request_exception', + client_id=client_id, + error_message=str(exception), + index=pending.index, + ) + completed_client_ids.append(client_id) + continue - exception = future.exception() - if exception is not None: + response = future.result() records.append( _record_from_pending( pending, ended_at_s=now_s, - success=False, + success=response.success, timed_out=False, - error_message=str(exception), - result_count=0, + cancel_requested=False, + error_message=response.error_message, + result_count=len(response.results), ) ) + debug_events.log( + 'request_complete', + client_id=client_id, + index=pending.index, + result_count=len(response.results), + success=response.success, + ) completed_client_ids.append(client_id) - continue - response = future.result() - records.append( - _record_from_pending( - pending, - ended_at_s=now_s, - success=response.success, - timed_out=False, - error_message=response.error_message, - result_count=len(response.results), + for client_id in completed_client_ids: + del pending_by_client[client_id] + + if timed_out_this_cycle: + debug_events.log( + 'timeout_barrier', + pending_count=len(pending_by_client), ) - ) - completed_client_ids.append(client_id) + continue - for client_id in completed_client_ids: - del pending_by_client[client_id] + now_s = time.monotonic() + for client_id in range(args.clients): + if len(pending_by_client) >= max_in_flight: + break + if client_id in pending_by_client: + continue + if ( + args.request_gap_s > 0 + and now_s - last_started_by_client[client_id] + < args.request_gap_s + ): + continue + if should_send_request(): + start_request(client_id, now_s) + pending = pending_by_client[client_id] + debug_events.log( + 'request_sent', + client_id=client_id, + index=pending.index, + pending_count=len(pending_by_client), + query_index=pending.query_index, + query_type=pending.query_spec.query_type, + ) + debug_events.log( + 'experiment_finished', + records=len(records), + timeouts=sum(1 for record in records if record.timed_out), + ) finally: + if executor is not None: + executor.remove_node(node) + executor.shutdown() node.destroy_node() return records +def run_fake_query_service(args: argparse.Namespace) -> None: + """Run an immediate-response Query service for ROS client diagnostics.""" + if args.response_delay_s < 0: + raise ValueError( + '--response-delay-s must be greater than or equal to zero' + ) + if args.executor_threads < 1: + raise ValueError('--executor-threads must be greater than zero') + + node = rclpy.create_node('ros_typedb_fake_query_service') + executor = ( + MultiThreadedExecutor(num_threads=args.executor_threads) + if args.executor == 'multi' + else SingleThreadedExecutor() + ) + executor.add_node(node) + + def handle_query( + request: Query.Request, + response: Query.Response, + ) -> Query.Response: + if args.response_delay_s > 0: + time.sleep(args.response_delay_s) + response.success = True + response.error_message = '' + response.results = [] + return response + + service = node.create_service(Query, args.service_name, handle_query) + node.get_logger().info( + f'Fake Query service ready on {args.service_name}' + ) + try: + executor.spin() + finally: + node.destroy_service(service) + executor.remove_node(node) + executor.shutdown() + node.destroy_node() + + def main(argv: list[str] | None = None) -> int: """Run the stress experiment CLI.""" parser = build_argument_parser() @@ -441,10 +792,12 @@ def main(argv: list[str] | None = None) -> int: records = run_experiment(args) summary = summarize_records(records) print_summary(summary) + output_path = None if args.output: query_specs = _build_query_specs(args) + output_path = Path(args.output).expanduser() write_results( - Path(args.output), + output_path, service_name=args.service_name, query_type=args.query_type, query=args.query, @@ -454,8 +807,26 @@ def main(argv: list[str] | None = None) -> int: duration_s=args.duration_s, mode=args.mode, query_mix=query_specs, + request_gap_s=args.request_gap_s, + max_in_flight=args.max_in_flight, + executor=args.executor, + ) + print(f' output: {output_path}') + timeout_output_path = None + if args.timeout_output: + timeout_output_path = Path(args.timeout_output).expanduser() + elif output_path is not None: + timeout_output_path = default_timeout_output_path(output_path) + if timeout_output_path is not None: + write_timeout_results( + timeout_output_path, + source_output_path=output_path, + records=records, ) - return 0 if summary['failures'] == 0 else 1 + print(f' timeout_output: {timeout_output_path}') + if args.fail_on_failure and summary['failures'] > 0: + return 1 + return 0 except (RuntimeError, ValueError, OSError) as exc: print(f'ros_typedb stress experiment failed: {exc}', file=sys.stderr) return 1 @@ -463,5 +834,23 @@ def main(argv: list[str] | None = None) -> int: rclpy.shutdown() +def fake_query_service_main(argv: list[str] | None = None) -> int: + """Run the fake Query service CLI.""" + parser = build_fake_service_argument_parser() + args = parser.parse_args(argv) + + rclpy.init(args=None) + try: + run_fake_query_service(args) + return 0 + except (RuntimeError, ValueError, OSError, KeyboardInterrupt) as exc: + if isinstance(exc, KeyboardInterrupt): + return 0 + print(f'ros_typedb fake query service failed: {exc}', file=sys.stderr) + return 1 + finally: + rclpy.shutdown() + + if __name__ == '__main__': raise SystemExit(main()) diff --git a/ros_typedb_tools/scripts/run_stage2_timeout_scenarios.sh b/ros_typedb_tools/scripts/run_stage2_timeout_scenarios.sh new file mode 100755 index 0000000..be8b8d9 --- /dev/null +++ b/ros_typedb_tools/scripts/run_stage2_timeout_scenarios.sh @@ -0,0 +1,84 @@ +#!/usr/bin/env bash + +set -euo pipefail + +CLIENTS="${CLIENTS:-20}" +DURATION_S="${DURATION_S:-60}" +TIMEOUT_S="${TIMEOUT_S:-10}" +EXECUTOR_THREADS="${EXECUTOR_THREADS:-4}" +REAL_SERVICE_NAME="${REAL_SERVICE_NAME:-/ros_typedb_interface/query}" +FAKE_SERVICE_NAME="${FAKE_SERVICE_NAME:-/ros_typedb_fake_query}" +OUTPUT_DIR="${OUTPUT_DIR:-$HOME/results/ros_typedb_stage2_timeout_scenarios}" + +mkdir -p "$OUTPUT_DIR" + +if ! command -v ros2 >/dev/null 2>&1; then + echo "ros2 was not found. Source ROS and the workspace before running." >&2 + exit 1 +fi + +run_stress() { + local name="$1" + local service_name="$2" + shift 2 + + echo + echo "==> Running $name" + ros2 run ros_typedb_tools ros_typedb_stress_experiment \ + --service-name "$service_name" \ + --clients "$CLIENTS" \ + --duration-s "$DURATION_S" \ + --timeout-s "$TIMEOUT_S" \ + --debug-events-output "$OUTPUT_DIR/${name}_events.jsonl" \ + --output "$OUTPUT_DIR/${name}_results.json" \ + "$@" +} + +start_fake_service() { + echo + echo "==> Starting fake Query service on $FAKE_SERVICE_NAME" + ros2 run ros_typedb_tools ros_typedb_fake_query_service \ + --service-name "$FAKE_SERVICE_NAME" \ + >"$OUTPUT_DIR/fake_query_service.log" 2>&1 & + FAKE_SERVICE_PID="$!" + sleep 1 +} + +stop_fake_service() { + if [[ -n "${FAKE_SERVICE_PID:-}" ]]; then + kill "$FAKE_SERVICE_PID" 2>/dev/null || true + wait "$FAKE_SERVICE_PID" 2>/dev/null || true + fi +} + +trap stop_fake_service EXIT + +echo "Writing results to $OUTPUT_DIR" + +run_stress "real_read_global" "$REAL_SERVICE_NAME" \ + --mode read + +run_stress "real_read_multi" "$REAL_SERVICE_NAME" \ + --mode read \ + --executor multi \ + --executor-threads "$EXECUTOR_THREADS" + +start_fake_service + +run_stress "fake_read_global" "$FAKE_SERVICE_NAME" \ + --mode read + +run_stress "fake_read_multi" "$FAKE_SERVICE_NAME" \ + --mode read \ + --executor multi \ + --executor-threads "$EXECUTOR_THREADS" + +stop_fake_service +unset FAKE_SERVICE_PID + +run_stress "real_entity_get_global" "$REAL_SERVICE_NAME" \ + --query 'match $x isa entity; get $x;' \ + --query-type get + +echo +echo "Finished. Results are in $OUTPUT_DIR" diff --git a/ros_typedb_tools/setup.py b/ros_typedb_tools/setup.py index d3952c0..a58c152 100644 --- a/ros_typedb_tools/setup.py +++ b/ros_typedb_tools/setup.py @@ -1,3 +1,5 @@ +"""Package setup for ros_typedb_tools.""" + from setuptools import find_packages, setup package_name = 'ros_typedb_tools' @@ -15,7 +17,10 @@ zip_safe=True, maintainer='Gustavo Rezende', maintainer_email='g.rezendesilva@tudelft.nl', - description='Tools for TypeDB workflows, including schema and rule diagram generation', + description=( + 'Tools for TypeDB workflows, including schema and rule diagram ' + 'generation' + ), license='Apache-2.0', extras_require={ 'test': [ @@ -24,9 +29,23 @@ }, entry_points={ 'console_scripts': [ - 'typedb_schema_diagram = ros_typedb_tools.typedb_schema_diagram:main', - 'typedb_rule_diagram = ros_typedb_tools.typedb_rule_diagram:main', - 'ros_typedb_stress_experiment = ros_typedb_tools.ros_typedb_stress_experiment:main', + ( + 'typedb_schema_diagram = ' + 'ros_typedb_tools.typedb_schema_diagram:main' + ), + ( + 'typedb_rule_diagram = ' + 'ros_typedb_tools.typedb_rule_diagram:main' + ), + ( + 'ros_typedb_stress_experiment = ' + 'ros_typedb_tools.ros_typedb_stress_experiment:main' + ), + ( + 'ros_typedb_fake_query_service = ' + 'ros_typedb_tools.ros_typedb_stress_experiment:' + 'fake_query_service_main' + ), ], }, ) diff --git a/ros_typedb_tools/test/test_ros_typedb_stress_experiment.py b/ros_typedb_tools/test/test_ros_typedb_stress_experiment.py index a94dd4f..86e83f6 100644 --- a/ros_typedb_tools/test/test_ros_typedb_stress_experiment.py +++ b/ros_typedb_tools/test/test_ros_typedb_stress_experiment.py @@ -6,13 +6,19 @@ import pytest -from ros_typedb_tools.ros_typedb_stress_experiment import QUERY_TYPE_BY_NAME -from ros_typedb_tools.ros_typedb_stress_experiment import RequestRecord -from ros_typedb_tools.ros_typedb_stress_experiment import _build_query_specs -from ros_typedb_tools.ros_typedb_stress_experiment import build_argument_parser -from ros_typedb_tools.ros_typedb_stress_experiment import run_experiment -from ros_typedb_tools.ros_typedb_stress_experiment import summarize_records -from ros_typedb_tools.ros_typedb_stress_experiment import write_results +from ros_typedb_tools.ros_typedb_stress_experiment import ( + DebugEventWriter, + QUERY_TYPE_BY_NAME, + RequestRecord, + _build_query_specs, + build_argument_parser, + build_fake_service_argument_parser, + default_timeout_output_path, + run_experiment, + summarize_records, + write_results, + write_timeout_results, +) def _record( @@ -25,11 +31,15 @@ def _record( return RequestRecord( index=index, client_id=0, + query_index=0, + query_type='fetch', + query='match $x isa entity; fetch $x: attribute;', started_at_s=10.0 + index, ended_at_s=10.0 + index + latency_s, latency_s=latency_s, success=success, timed_out=timed_out, + cancel_requested=timed_out, error_message='' if success else 'failed', result_count=1 if success else 0, ) @@ -51,6 +61,19 @@ def test_parser_accepts_stage_one_arguments(): '2.5', '--output', 'results.json', + '--timeout-output', + 'timeouts.json', + '--debug-events-output', + 'events.jsonl', + '--request-gap-s', + '0.01', + '--max-in-flight', + '2', + '--executor', + 'multi', + '--executor-threads', + '4', + '--fail-on-failure', ] ) @@ -59,6 +82,13 @@ def test_parser_accepts_stage_one_arguments(): assert args.requests == 3 assert args.timeout_s == 2.5 assert args.output == 'results.json' + assert args.timeout_output == 'timeouts.json' + assert args.debug_events_output == 'events.jsonl' + assert args.request_gap_s == 0.01 + assert args.max_in_flight == 2 + assert args.executor == 'multi' + assert args.executor_threads == 4 + assert args.fail_on_failure is True def test_parser_accepts_stage_two_arguments(): @@ -83,6 +113,32 @@ def test_parser_accepts_stage_two_arguments(): assert args.mode == 'read' assert args.query is None assert args.query_type is None + assert args.request_gap_s == 0.0 + assert args.max_in_flight is None + assert args.executor == 'global' + + +def test_fake_service_parser_accepts_diagnostic_arguments(): + """Check the fake service CLI options.""" + parser = build_fake_service_argument_parser() + + args = parser.parse_args( + [ + '--service-name', + '/debug_query', + '--response-delay-s', + '0.001', + '--executor', + 'multi', + '--executor-threads', + '3', + ] + ) + + assert args.service_name == '/debug_query' + assert args.response_delay_s == 0.001 + assert args.executor == 'multi' + assert args.executor_threads == 3 def test_query_type_names_cover_query_service_constants(): @@ -139,21 +195,86 @@ def test_write_results_writes_summary_and_request_records(tmp_path: Path): query_mix=_build_query_specs( SimpleNamespace(query=None, query_type=None, mode='read') ), + request_gap_s=0.01, + max_in_flight=1, + executor='single', ) payload = json.loads(output_path.read_text(encoding='utf-8')) assert payload['service_name'] == '/ros_typedb_interface/query' assert payload['clients'] == 2 assert payload['duration_s'] == 10.0 + assert payload['request_gap_s'] == 0.01 + assert payload['max_in_flight'] == 1 + assert payload['executor'] == 'single' assert payload['mode'] == 'read' assert payload['query_type'] == 'fetch' assert payload['query_mix'][0]['query_type'] in QUERY_TYPE_BY_NAME assert payload['summary']['successes'] == 1 assert payload['requests'][0]['index'] == 0 assert payload['requests'][0]['client_id'] == 0 + assert payload['requests'][0]['query_index'] == 0 + assert payload['requests'][0]['query_type'] == 'fetch' + assert payload['requests'][0]['query'] == ( + 'match $x isa entity; fetch $x: attribute;' + ) assert payload['requests'][0]['success'] is True +def test_default_timeout_output_path_uses_result_stem(): + """Check timeout output path is derived from the main output path.""" + assert default_timeout_output_path( + Path('/tmp/results.json') + ) == Path('/tmp/results_timeouts.json') + assert default_timeout_output_path( + Path('/tmp/results') + ) == Path('/tmp/results_timeouts.json') + + +def test_write_timeout_results_writes_only_timeout_records(tmp_path: Path): + """Check timeout output contains only timed-out request records.""" + output_path = tmp_path / 'stress' / 'timeouts.json' + records = [ + _record(0, latency_s=0.1, success=True), + _record(1, latency_s=10.0, success=False, timed_out=True), + ] + + write_timeout_results( + output_path, + source_output_path=tmp_path / 'results.json', + records=records, + ) + + payload = json.loads(output_path.read_text(encoding='utf-8')) + assert payload['source_output_path'] == str(tmp_path / 'results.json') + assert payload['summary']['total_requests'] == 2 + assert payload['summary']['timeouts'] == 1 + assert len(payload['timeouts']) == 1 + assert payload['timeouts'][0]['index'] == 1 + assert payload['timeouts'][0]['query_index'] == 0 + assert payload['timeouts'][0]['cancel_requested'] is True + + +def test_debug_event_writer_writes_json_lines(tmp_path: Path): + """Check debug events are written as JSONL records.""" + output_path = tmp_path / 'debug' / 'events.jsonl' + + with DebugEventWriter(output_path) as writer: + writer.log('request_sent', index=1, pending_count=2) + writer.log('request_timeout', index=1) + + events = [ + json.loads(line) + for line in output_path.read_text(encoding='utf-8').splitlines() + ] + assert [event['event'] for event in events] == [ + 'request_sent', + 'request_timeout', + ] + assert events[0]['index'] == 1 + assert 'monotonic_s' in events[0] + + def test_build_query_specs_uses_single_explicit_query(): """Check explicit query arguments override the built-in mix.""" specs = _build_query_specs( @@ -199,6 +320,11 @@ def test_run_experiment_rejects_invalid_request_count(): duration_s=None, timeout_s=5.0, wait_service_timeout_s=1.0, + request_gap_s=0.0, + max_in_flight=None, + executor='global', + executor_threads=2, + debug_events_output=None, query='match $x isa entity; fetch $x: attribute;', query_type='fetch', mode='read', From 43a20ac5962750b2650518df33d7d100a57074f2 Mon Sep 17 00:00:00 2001 From: Gustavo Date: Sat, 27 Jun 2026 14:36:52 -0300 Subject: [PATCH 19/36] Refactor stress experiment tool --- .../ros_typedb_tools/fake_query_service.py | 119 +++ .../ros_typedb_stress_experiment.py | 726 ++---------------- .../ros_typedb_tools/stress_config.py | 128 +++ .../ros_typedb_tools/stress_output.py | 199 +++++ .../ros_typedb_tools/stress_runner.py | 458 +++++++++++ ros_typedb_tools/setup.py | 3 +- .../test/test_ros_typedb_stress_experiment.py | 24 +- 7 files changed, 963 insertions(+), 694 deletions(-) create mode 100644 ros_typedb_tools/ros_typedb_tools/fake_query_service.py create mode 100644 ros_typedb_tools/ros_typedb_tools/stress_config.py create mode 100644 ros_typedb_tools/ros_typedb_tools/stress_output.py create mode 100644 ros_typedb_tools/ros_typedb_tools/stress_runner.py diff --git a/ros_typedb_tools/ros_typedb_tools/fake_query_service.py b/ros_typedb_tools/ros_typedb_tools/fake_query_service.py new file mode 100644 index 0000000..60743e7 --- /dev/null +++ b/ros_typedb_tools/ros_typedb_tools/fake_query_service.py @@ -0,0 +1,119 @@ +# Copyright 2026 Gustavo Rezende Silva +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +"""Fake Query service used to isolate ROS service-client behavior.""" + +from __future__ import annotations + +import argparse +import sys +import time + +import rclpy +from rclpy.executors import MultiThreadedExecutor +from rclpy.executors import SingleThreadedExecutor + +from ros_typedb_msgs.srv import Query + + +def build_fake_service_argument_parser() -> argparse.ArgumentParser: + """Build command-line arguments for the fake Query service.""" + parser = argparse.ArgumentParser( + prog='ros_typedb_fake_query_service', + description='Serve the ros_typedb Query service without TypeDB work.', + ) + parser.add_argument( + '--service-name', + default='/ros_typedb_interface/query', + help='Query service name. Defaults to /ros_typedb_interface/query.', + ) + parser.add_argument( + '--response-delay-s', + type=float, + default=0.0, + help='Optional delay before each response. Defaults to 0.', + ) + parser.add_argument( + '--executor', + choices=('single', 'multi'), + default='single', + help='Server executor. Defaults to single.', + ) + parser.add_argument( + '--executor-threads', + type=int, + default=2, + help='Thread count for --executor multi. Defaults to 2.', + ) + return parser + + +def _build_fake_service_executor(args: argparse.Namespace): + if args.executor == 'multi': + return MultiThreadedExecutor(num_threads=args.executor_threads) + return SingleThreadedExecutor() + + +def run_fake_query_service(args: argparse.Namespace) -> None: + """Run an immediate-response Query service for ROS client diagnostics.""" + if args.response_delay_s < 0: + raise ValueError( + '--response-delay-s must be greater than or equal to zero' + ) + if args.executor_threads < 1: + raise ValueError('--executor-threads must be greater than zero') + + node = rclpy.create_node('ros_typedb_fake_query_service') + executor = _build_fake_service_executor(args) + executor.add_node(node) + + def handle_query( + request: Query.Request, + response: Query.Response, + ) -> Query.Response: + if args.response_delay_s > 0: + time.sleep(args.response_delay_s) + response.success = True + response.error_message = '' + response.results = [] + return response + + service = node.create_service(Query, args.service_name, handle_query) + node.get_logger().info( + f'Fake Query service ready on {args.service_name}' + ) + try: + executor.spin() + finally: + node.destroy_service(service) + executor.remove_node(node) + executor.shutdown() + node.destroy_node() + + +def fake_query_service_main(argv: list[str] | None = None) -> int: + """Run the fake Query service CLI.""" + parser = build_fake_service_argument_parser() + args = parser.parse_args(argv) + + rclpy.init(args=None) + try: + run_fake_query_service(args) + return 0 + except (RuntimeError, ValueError, OSError, KeyboardInterrupt) as exc: + if isinstance(exc, KeyboardInterrupt): + return 0 + print(f'ros_typedb fake query service failed: {exc}', file=sys.stderr) + return 1 + finally: + rclpy.shutdown() diff --git a/ros_typedb_tools/ros_typedb_tools/ros_typedb_stress_experiment.py b/ros_typedb_tools/ros_typedb_tools/ros_typedb_stress_experiment.py index 91742bf..11ec31e 100644 --- a/ros_typedb_tools/ros_typedb_tools/ros_typedb_stress_experiment.py +++ b/ros_typedb_tools/ros_typedb_tools/ros_typedb_stress_experiment.py @@ -16,74 +16,19 @@ from __future__ import annotations import argparse -import json -import statistics import sys -import time -from collections.abc import Sequence -from dataclasses import asdict -from dataclasses import dataclass from pathlib import Path -from typing import Any import rclpy -from rclpy.executors import MultiThreadedExecutor -from rclpy.executors import SingleThreadedExecutor -from ros_typedb_msgs.srv import Query - - -QUERY_TYPE_BY_NAME = { - 'insert': Query.Request.INSERT, - 'delete': Query.Request.DELETE, - 'fetch': Query.Request.FETCH, - 'get': Query.Request.GET, - 'get_aggregate': Query.Request.GET_AGGREGATE, - 'define': Query.Request.DEFINE, - 'update': Query.Request.UPDATE, -} - -DEFAULT_READ_QUERY_SPECS = ( - ( - 'match $x isa entity; get $x;', - 'get', - ), - ( - 'match $x isa relation; get $x;', - 'get', - ), - ( - 'match $x isa attribute; get $x;', - 'get', - ), -) - - -@dataclass(frozen=True) -class QuerySpec: - """Query and service query type used by one request.""" - - query: str - query_type: str - - -@dataclass(frozen=True) -class RequestRecord: - """Result of one service request.""" - - index: int - client_id: int - query_index: int - query_type: str - query: str - started_at_s: float - ended_at_s: float - latency_s: float - success: bool - timed_out: bool - cancel_requested: bool - error_message: str - result_count: int +from ros_typedb_tools.stress_config import QUERY_TYPE_BY_NAME +from ros_typedb_tools.stress_config import build_query_specs +from ros_typedb_tools.stress_output import default_timeout_output_path +from ros_typedb_tools.stress_output import print_summary +from ros_typedb_tools.stress_output import summarize_records +from ros_typedb_tools.stress_output import write_results +from ros_typedb_tools.stress_output import write_timeout_results +from ros_typedb_tools.stress_runner import run_experiment def build_argument_parser() -> argparse.ArgumentParser: @@ -207,579 +152,43 @@ def build_argument_parser() -> argparse.ArgumentParser: return parser -def build_fake_service_argument_parser() -> argparse.ArgumentParser: - """Build command-line arguments for the fake Query service.""" - parser = argparse.ArgumentParser( - prog='ros_typedb_fake_query_service', - description='Serve the ros_typedb Query service without TypeDB work.', - ) - parser.add_argument( - '--service-name', - default='/ros_typedb_interface/query', - help='Query service name. Defaults to /ros_typedb_interface/query.', - ) - parser.add_argument( - '--response-delay-s', - type=float, - default=0.0, - help='Optional delay before each response. Defaults to 0.', - ) - parser.add_argument( - '--executor', - choices=('single', 'multi'), - default='single', - help='Server executor. Defaults to single.', - ) - parser.add_argument( - '--executor-threads', - type=int, - default=2, - help='Thread count for --executor multi. Defaults to 2.', - ) - return parser - - -def _latency_percentile( - sorted_latencies: Sequence[float], - rank: float, -) -> float: - if not sorted_latencies: - raise ValueError('cannot compute percentile for empty sequence') - if len(sorted_latencies) == 1: - return sorted_latencies[0] - - position = (len(sorted_latencies) - 1) * rank - lower_index = int(position) - upper_index = min(lower_index + 1, len(sorted_latencies) - 1) - fraction = position - lower_index - lower_value = sorted_latencies[lower_index] - upper_value = sorted_latencies[upper_index] - return lower_value + ((upper_value - lower_value) * fraction) - - -def summarize_records(records: list[RequestRecord]) -> dict[str, Any]: - """Build summary metrics from request records.""" - latencies = sorted(record.latency_s for record in records) - successes = sum(1 for record in records if record.success) - timeouts = sum(1 for record in records if record.timed_out) - failures = len(records) - successes - - summary: dict[str, Any] = { - 'total_requests': len(records), - 'successes': successes, - 'failures': failures, - 'timeouts': timeouts, - 'latency_s': {}, - } - if latencies: - summary['latency_s'] = { - 'min': min(latencies), - 'mean': statistics.fmean(latencies), - 'p50': _latency_percentile(latencies, 0.50), - 'p90': _latency_percentile(latencies, 0.90), - 'p95': _latency_percentile(latencies, 0.95), - 'p99': _latency_percentile(latencies, 0.99), - 'max': max(latencies), - } - return summary - - -def write_results( - output_path: Path, - *, - service_name: str, - query_type: str | None, - query: str | None, - timeout_s: float, - records: list[RequestRecord], - clients: int = 1, - duration_s: float | None = None, - mode: str = 'read', - query_mix: Sequence[QuerySpec] | None = None, - request_gap_s: float = 0.0, - max_in_flight: int | None = None, - executor: str = 'global', +def _write_requested_outputs( + args: argparse.Namespace, + records, ) -> None: - """Write experiment results to a JSON file.""" - payload = { - 'service_name': service_name, - 'mode': mode, - 'clients': clients, - 'duration_s': duration_s, - 'request_gap_s': request_gap_s, - 'max_in_flight': max_in_flight, - 'executor': executor, - 'query_type': query_type, - 'query': query, - 'query_mix': [ - asdict(query_spec) for query_spec in query_mix or () - ], - 'timeout_s': timeout_s, - 'summary': summarize_records(records), - 'requests': [asdict(record) for record in records], - } - output_path.parent.mkdir(parents=True, exist_ok=True) - output_path.write_text( - json.dumps(payload, indent=2, sort_keys=True) + '\n', - encoding='utf-8', - ) - - -def default_timeout_output_path(output_path: Path) -> Path: - """Build the default timeout-only output path for a result path.""" - suffix = output_path.suffix or '.json' - stem = output_path.stem if output_path.suffix else output_path.name - return output_path.with_name(f'{stem}_timeouts{suffix}') - - -def write_timeout_results( - output_path: Path, - *, - source_output_path: Path | None, - records: list[RequestRecord], -) -> None: - """Write timeout-only request records to a JSON file.""" - timeout_records = [ - record - for record in records - if record.timed_out - ] - payload = { - 'source_output_path': ( - str(source_output_path) if source_output_path is not None else None - ), - 'summary': { - 'timeouts': len(timeout_records), - 'total_requests': len(records), - }, - 'timeouts': [asdict(record) for record in timeout_records], - } - output_path.parent.mkdir(parents=True, exist_ok=True) - output_path.write_text( - json.dumps(payload, indent=2, sort_keys=True) + '\n', - encoding='utf-8', - ) - - -class DebugEventWriter: - """Write compact JSONL events for diagnosing client-side scheduling.""" - - def __init__(self, output_path: Path | None) -> None: - """Create an event writer for the optional output path.""" - self._output_path = output_path - self._stream = None - - def __enter__(self) -> 'DebugEventWriter': - """Open the event stream when debug output is enabled.""" - if self._output_path is not None: - self._output_path.parent.mkdir(parents=True, exist_ok=True) - self._stream = self._output_path.open('w', encoding='utf-8') - return self - - def __exit__(self, *args: object) -> None: - """Close the event stream when it was opened.""" - if self._stream is not None: - self._stream.close() - - def log(self, event: str, **fields: Any) -> None: - """Append one debug event if debug output is enabled.""" - if self._stream is None: - return - payload = { - 'event': event, - 'monotonic_s': time.monotonic(), - **fields, - } - self._stream.write(json.dumps(payload, sort_keys=True) + '\n') - - -def print_summary(summary: dict[str, Any]) -> None: - """Print compact experiment summary.""" - latency = summary['latency_s'] - print('ros_typedb stress experiment summary') - print(f" total_requests: {summary['total_requests']}") - print(f" successes: {summary['successes']}") - print(f" failures: {summary['failures']}") - print(f" timeouts: {summary['timeouts']}") - if latency: - print(f" latency_min_s: {latency['min']:.6f}") - print(f" latency_mean_s: {latency['mean']:.6f}") - print(f" latency_p50_s: {latency['p50']:.6f}") - print(f" latency_p90_s: {latency['p90']:.6f}") - print(f" latency_p95_s: {latency['p95']:.6f}") - print(f" latency_p99_s: {latency['p99']:.6f}") - print(f" latency_max_s: {latency['max']:.6f}") - - -def _build_query_request( - query: str, - query_type: str, - timeout_s: float, -) -> Query.Request: - request = Query.Request() - request.query = query - request.query_type = QUERY_TYPE_BY_NAME[query_type] - request.timeout_s = timeout_s - return request - - -def _build_query_specs(args: argparse.Namespace) -> list[QuerySpec]: - if args.query and not args.query_type: - raise ValueError('--query-type is required when --query is used') - if args.query_type and not args.query: - raise ValueError('--query-type requires --query') - - if args.query: - return [QuerySpec(query=args.query, query_type=args.query_type)] - - if args.mode == 'read': - return [ - QuerySpec(query=query, query_type=query_type) - for query, query_type in DEFAULT_READ_QUERY_SPECS - ] - - raise ValueError(f'unsupported mode: {args.mode}') - - -@dataclass -class _PendingRequest: - client_id: int - index: int - query_index: int - query_spec: QuerySpec - future: Any - started_at_s: float - - -def _build_executor(args: argparse.Namespace) -> Any | None: - if args.executor == 'global': - return None - if args.executor == 'single': - return SingleThreadedExecutor() - if args.executor == 'multi': - return MultiThreadedExecutor(num_threads=args.executor_threads) - raise ValueError(f'unsupported executor: {args.executor}') - - -def _spin_once(node: Any, executor: Any | None) -> None: - if executor is None: - rclpy.spin_once(node, timeout_sec=0.01) - else: - executor.spin_once(timeout_sec=0.01) - - -def _record_from_pending( - pending: _PendingRequest, - *, - ended_at_s: float, - success: bool, - timed_out: bool, - cancel_requested: bool, - error_message: str, - result_count: int, -) -> RequestRecord: - return RequestRecord( - index=pending.index, - client_id=pending.client_id, - query_index=pending.query_index, - query_type=pending.query_spec.query_type, - query=pending.query_spec.query, - started_at_s=pending.started_at_s, - ended_at_s=ended_at_s, - latency_s=ended_at_s - pending.started_at_s, - success=success, - timed_out=timed_out, - cancel_requested=cancel_requested, - error_message=error_message, - result_count=result_count, - ) - - -def run_experiment(args: argparse.Namespace) -> list[RequestRecord]: - """Run the stress experiment.""" - if args.requests < 1: - raise ValueError('--requests must be greater than zero') - if args.clients < 1: - raise ValueError('--clients must be greater than zero') - if args.duration_s is not None and args.duration_s <= 0: - raise ValueError('--duration-s must be greater than zero') - if args.timeout_s <= 0: - raise ValueError('--timeout-s must be greater than zero') - if args.wait_service_timeout_s <= 0: - raise ValueError('--wait-service-timeout-s must be greater than zero') - if args.request_gap_s < 0: - raise ValueError( - '--request-gap-s must be greater than or equal to zero' + output_path = None + if args.output: + query_specs = build_query_specs(args) + output_path = Path(args.output).expanduser() + write_results( + output_path, + service_name=args.service_name, + query_type=args.query_type, + query=args.query, + timeout_s=args.timeout_s, + records=records, + clients=args.clients, + duration_s=args.duration_s, + mode=args.mode, + query_mix=query_specs, + request_gap_s=args.request_gap_s, + max_in_flight=args.max_in_flight, + executor=args.executor, ) - if args.max_in_flight is not None and args.max_in_flight < 1: - raise ValueError('--max-in-flight must be greater than zero') - if args.executor_threads < 1: - raise ValueError('--executor-threads must be greater than zero') - query_specs = _build_query_specs(args) - max_in_flight = args.max_in_flight or args.clients - debug_events_output = ( - Path(args.debug_events_output).expanduser() - if args.debug_events_output - else None - ) - - node = rclpy.create_node('ros_typedb_stress_experiment') - executor = _build_executor(args) - if executor is not None: - executor.add_node(node) - clients = [ - node.create_client(Query, args.service_name) - for _ in range(args.clients) - ] - records: list[RequestRecord] = [] - pending_by_client: dict[int, _PendingRequest] = {} - last_started_by_client = [float('-inf') for _ in range(args.clients)] - next_index = 0 - deadline_s = None - if args.duration_s is not None: - deadline_s = time.monotonic() + args.duration_s - next_snapshot_s = time.monotonic() + 1.0 - - def should_send_request() -> bool: - if deadline_s is not None: - return time.monotonic() < deadline_s - return next_index < args.requests - - def start_request(client_id: int, started_at_s: float) -> None: - nonlocal next_index - query_index = next_index % len(query_specs) - query_spec = query_specs[query_index] - request = _build_query_request( - query_spec.query, - query_spec.query_type, - args.timeout_s, - ) - future = clients[client_id].call_async(request) - pending_by_client[client_id] = _PendingRequest( - client_id=client_id, - index=next_index, - query_index=query_index, - query_spec=query_spec, - future=future, - started_at_s=started_at_s, - ) - last_started_by_client[client_id] = started_at_s - next_index += 1 - - try: - with DebugEventWriter(debug_events_output) as debug_events: - debug_events.log( - 'experiment_started', - clients=args.clients, - executor=args.executor, - max_in_flight=max_in_flight, - request_gap_s=args.request_gap_s, - ) - wait_deadline_s = time.monotonic() + args.wait_service_timeout_s - for client_id, client in enumerate(clients): - wait_timeout_s = max(0.0, wait_deadline_s - time.monotonic()) - service_ready = client.wait_for_service( - timeout_sec=wait_timeout_s - ) - debug_events.log( - 'service_wait_finished', - client_id=client_id, - ready=service_ready, - ) - if not service_ready: - raise RuntimeError( - f'Query service is not available: {args.service_name}' - ) - - while should_send_request() or pending_by_client: - spin_started_s = time.monotonic() - _spin_once(node, executor) - now_s = time.monotonic() - debug_events.log( - 'spin_once', - duration_s=now_s - spin_started_s, - pending_count=len(pending_by_client), - ) - if now_s >= next_snapshot_s: - debug_events.log( - 'pending_snapshot', - next_index=next_index, - pending_count=len(pending_by_client), - pending_indices=[ - pending.index - for pending in pending_by_client.values() - ], - ) - next_snapshot_s = now_s + 1.0 - - completed_client_ids = [] - timed_out_this_cycle = False - - for client_id, pending in pending_by_client.items(): - future = pending.future - if not future.done(): - if now_s - pending.started_at_s < args.timeout_s: - continue - timed_out_this_cycle = True - future.cancel() - records.append( - _record_from_pending( - pending, - ended_at_s=now_s, - success=False, - timed_out=True, - cancel_requested=True, - error_message='client request timed out', - result_count=0, - ) - ) - debug_events.log( - 'request_timeout', - client_id=client_id, - index=pending.index, - latency_s=now_s - pending.started_at_s, - pending_count=len(pending_by_client), - query_index=pending.query_index, - ) - completed_client_ids.append(client_id) - continue - - debug_events.log( - 'future_done', - client_id=client_id, - index=pending.index, - latency_s=now_s - pending.started_at_s, - pending_count=len(pending_by_client), - query_index=pending.query_index, - ) - exception = future.exception() - if exception is not None: - records.append( - _record_from_pending( - pending, - ended_at_s=now_s, - success=False, - timed_out=False, - cancel_requested=False, - error_message=str(exception), - result_count=0, - ) - ) - debug_events.log( - 'request_exception', - client_id=client_id, - error_message=str(exception), - index=pending.index, - ) - completed_client_ids.append(client_id) - continue - - response = future.result() - records.append( - _record_from_pending( - pending, - ended_at_s=now_s, - success=response.success, - timed_out=False, - cancel_requested=False, - error_message=response.error_message, - result_count=len(response.results), - ) - ) - debug_events.log( - 'request_complete', - client_id=client_id, - index=pending.index, - result_count=len(response.results), - success=response.success, - ) - completed_client_ids.append(client_id) - - for client_id in completed_client_ids: - del pending_by_client[client_id] - - if timed_out_this_cycle: - debug_events.log( - 'timeout_barrier', - pending_count=len(pending_by_client), - ) - continue - - now_s = time.monotonic() - for client_id in range(args.clients): - if len(pending_by_client) >= max_in_flight: - break - if client_id in pending_by_client: - continue - if ( - args.request_gap_s > 0 - and now_s - last_started_by_client[client_id] - < args.request_gap_s - ): - continue - if should_send_request(): - start_request(client_id, now_s) - pending = pending_by_client[client_id] - debug_events.log( - 'request_sent', - client_id=client_id, - index=pending.index, - pending_count=len(pending_by_client), - query_index=pending.query_index, - query_type=pending.query_spec.query_type, - ) - debug_events.log( - 'experiment_finished', - records=len(records), - timeouts=sum(1 for record in records if record.timed_out), - ) - finally: - if executor is not None: - executor.remove_node(node) - executor.shutdown() - node.destroy_node() - return records - - -def run_fake_query_service(args: argparse.Namespace) -> None: - """Run an immediate-response Query service for ROS client diagnostics.""" - if args.response_delay_s < 0: - raise ValueError( - '--response-delay-s must be greater than or equal to zero' + print(f' output: {output_path}') + + timeout_output_path = None + if args.timeout_output: + timeout_output_path = Path(args.timeout_output).expanduser() + elif output_path is not None: + timeout_output_path = default_timeout_output_path(output_path) + if timeout_output_path is not None: + write_timeout_results( + timeout_output_path, + source_output_path=output_path, + records=records, ) - if args.executor_threads < 1: - raise ValueError('--executor-threads must be greater than zero') - - node = rclpy.create_node('ros_typedb_fake_query_service') - executor = ( - MultiThreadedExecutor(num_threads=args.executor_threads) - if args.executor == 'multi' - else SingleThreadedExecutor() - ) - executor.add_node(node) - - def handle_query( - request: Query.Request, - response: Query.Response, - ) -> Query.Response: - if args.response_delay_s > 0: - time.sleep(args.response_delay_s) - response.success = True - response.error_message = '' - response.results = [] - return response - - service = node.create_service(Query, args.service_name, handle_query) - node.get_logger().info( - f'Fake Query service ready on {args.service_name}' - ) - try: - executor.spin() - finally: - node.destroy_service(service) - executor.remove_node(node) - executor.shutdown() - node.destroy_node() + print(f' timeout_output: {timeout_output_path}') def main(argv: list[str] | None = None) -> int: @@ -792,38 +201,7 @@ def main(argv: list[str] | None = None) -> int: records = run_experiment(args) summary = summarize_records(records) print_summary(summary) - output_path = None - if args.output: - query_specs = _build_query_specs(args) - output_path = Path(args.output).expanduser() - write_results( - output_path, - service_name=args.service_name, - query_type=args.query_type, - query=args.query, - timeout_s=args.timeout_s, - records=records, - clients=args.clients, - duration_s=args.duration_s, - mode=args.mode, - query_mix=query_specs, - request_gap_s=args.request_gap_s, - max_in_flight=args.max_in_flight, - executor=args.executor, - ) - print(f' output: {output_path}') - timeout_output_path = None - if args.timeout_output: - timeout_output_path = Path(args.timeout_output).expanduser() - elif output_path is not None: - timeout_output_path = default_timeout_output_path(output_path) - if timeout_output_path is not None: - write_timeout_results( - timeout_output_path, - source_output_path=output_path, - records=records, - ) - print(f' timeout_output: {timeout_output_path}') + _write_requested_outputs(args, records) if args.fail_on_failure and summary['failures'] > 0: return 1 return 0 @@ -834,23 +212,5 @@ def main(argv: list[str] | None = None) -> int: rclpy.shutdown() -def fake_query_service_main(argv: list[str] | None = None) -> int: - """Run the fake Query service CLI.""" - parser = build_fake_service_argument_parser() - args = parser.parse_args(argv) - - rclpy.init(args=None) - try: - run_fake_query_service(args) - return 0 - except (RuntimeError, ValueError, OSError, KeyboardInterrupt) as exc: - if isinstance(exc, KeyboardInterrupt): - return 0 - print(f'ros_typedb fake query service failed: {exc}', file=sys.stderr) - return 1 - finally: - rclpy.shutdown() - - if __name__ == '__main__': raise SystemExit(main()) diff --git a/ros_typedb_tools/ros_typedb_tools/stress_config.py b/ros_typedb_tools/ros_typedb_tools/stress_config.py new file mode 100644 index 0000000..5db231c --- /dev/null +++ b/ros_typedb_tools/ros_typedb_tools/stress_config.py @@ -0,0 +1,128 @@ +# Copyright 2026 Gustavo Rezende Silva +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +"""Shared configuration helpers for the ros_typedb stress tools.""" + +from __future__ import annotations + +import argparse +from dataclasses import dataclass + +from ros_typedb_msgs.srv import Query + + +QUERY_TYPE_BY_NAME = { + 'insert': Query.Request.INSERT, + 'delete': Query.Request.DELETE, + 'fetch': Query.Request.FETCH, + 'get': Query.Request.GET, + 'get_aggregate': Query.Request.GET_AGGREGATE, + 'define': Query.Request.DEFINE, + 'update': Query.Request.UPDATE, +} + +DEFAULT_READ_QUERY_SPECS = ( + ( + 'match $x isa entity; get $x;', + 'get', + ), + ( + 'match $x isa relation; get $x;', + 'get', + ), + ( + 'match $x isa attribute; get $x;', + 'get', + ), +) + + +@dataclass(frozen=True) +class QuerySpec: + """Query and service query type used by one request.""" + + query: str + query_type: str + + +@dataclass(frozen=True) +class RequestRecord: + """Result of one service request.""" + + index: int + client_id: int + query_index: int + query_type: str + query: str + started_at_s: float + ended_at_s: float + latency_s: float + success: bool + timed_out: bool + cancel_requested: bool + error_message: str + result_count: int + + +def validate_experiment_args(args: argparse.Namespace) -> None: + """Validate stress arguments before ROS resources are created.""" + if args.requests < 1: + raise ValueError('--requests must be greater than zero') + if args.clients < 1: + raise ValueError('--clients must be greater than zero') + if args.duration_s is not None and args.duration_s <= 0: + raise ValueError('--duration-s must be greater than zero') + if args.timeout_s <= 0: + raise ValueError('--timeout-s must be greater than zero') + if args.wait_service_timeout_s <= 0: + raise ValueError('--wait-service-timeout-s must be greater than zero') + if args.request_gap_s < 0: + raise ValueError( + '--request-gap-s must be greater than or equal to zero' + ) + if args.max_in_flight is not None and args.max_in_flight < 1: + raise ValueError('--max-in-flight must be greater than zero') + if args.executor_threads < 1: + raise ValueError('--executor-threads must be greater than zero') + + +def build_query_specs(args: argparse.Namespace) -> list[QuerySpec]: + """Build the query sequence used by the stress experiment.""" + if args.query and not args.query_type: + raise ValueError('--query-type is required when --query is used') + if args.query_type and not args.query: + raise ValueError('--query-type requires --query') + + if args.query: + return [QuerySpec(query=args.query, query_type=args.query_type)] + + if args.mode == 'read': + return [ + QuerySpec(query=query, query_type=query_type) + for query, query_type in DEFAULT_READ_QUERY_SPECS + ] + + raise ValueError(f'unsupported mode: {args.mode}') + + +def build_query_request( + query: str, + query_type: str, + timeout_s: float, +) -> Query.Request: + """Build a Query service request from a TypeDB query spec.""" + request = Query.Request() + request.query = query + request.query_type = QUERY_TYPE_BY_NAME[query_type] + request.timeout_s = timeout_s + return request diff --git a/ros_typedb_tools/ros_typedb_tools/stress_output.py b/ros_typedb_tools/ros_typedb_tools/stress_output.py new file mode 100644 index 0000000..c8427bf --- /dev/null +++ b/ros_typedb_tools/ros_typedb_tools/stress_output.py @@ -0,0 +1,199 @@ +# Copyright 2026 Gustavo Rezende Silva +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +"""Output helpers for ros_typedb stress experiments.""" + +from __future__ import annotations + +import json +import statistics +import time +from collections.abc import Sequence +from dataclasses import asdict +from pathlib import Path +from typing import Any + +from ros_typedb_tools.stress_config import QuerySpec +from ros_typedb_tools.stress_config import RequestRecord + + +def _latency_percentile( + sorted_latencies: Sequence[float], + rank: float, +) -> float: + if not sorted_latencies: + raise ValueError('cannot compute percentile for empty sequence') + if len(sorted_latencies) == 1: + return sorted_latencies[0] + + position = (len(sorted_latencies) - 1) * rank + lower_index = int(position) + upper_index = min(lower_index + 1, len(sorted_latencies) - 1) + fraction = position - lower_index + lower_value = sorted_latencies[lower_index] + upper_value = sorted_latencies[upper_index] + return lower_value + ((upper_value - lower_value) * fraction) + + +def summarize_records(records: list[RequestRecord]) -> dict[str, Any]: + """Build summary metrics from request records.""" + latencies = sorted(record.latency_s for record in records) + successes = sum(1 for record in records if record.success) + timeouts = sum(1 for record in records if record.timed_out) + failures = len(records) - successes + + summary: dict[str, Any] = { + 'total_requests': len(records), + 'successes': successes, + 'failures': failures, + 'timeouts': timeouts, + 'latency_s': {}, + } + if latencies: + summary['latency_s'] = { + 'min': min(latencies), + 'mean': statistics.fmean(latencies), + 'p50': _latency_percentile(latencies, 0.50), + 'p90': _latency_percentile(latencies, 0.90), + 'p95': _latency_percentile(latencies, 0.95), + 'p99': _latency_percentile(latencies, 0.99), + 'max': max(latencies), + } + return summary + + +def write_results( + output_path: Path, + *, + service_name: str, + query_type: str | None, + query: str | None, + timeout_s: float, + records: list[RequestRecord], + clients: int = 1, + duration_s: float | None = None, + mode: str = 'read', + query_mix: Sequence[QuerySpec] | None = None, + request_gap_s: float = 0.0, + max_in_flight: int | None = None, + executor: str = 'global', +) -> None: + """Write experiment results to a JSON file.""" + payload = { + 'service_name': service_name, + 'mode': mode, + 'clients': clients, + 'duration_s': duration_s, + 'request_gap_s': request_gap_s, + 'max_in_flight': max_in_flight, + 'executor': executor, + 'query_type': query_type, + 'query': query, + 'query_mix': [ + asdict(query_spec) for query_spec in query_mix or () + ], + 'timeout_s': timeout_s, + 'summary': summarize_records(records), + 'requests': [asdict(record) for record in records], + } + output_path.parent.mkdir(parents=True, exist_ok=True) + output_path.write_text( + json.dumps(payload, indent=2, sort_keys=True) + '\n', + encoding='utf-8', + ) + + +def default_timeout_output_path(output_path: Path) -> Path: + """Build the default timeout-only output path for a result path.""" + suffix = output_path.suffix or '.json' + stem = output_path.stem if output_path.suffix else output_path.name + return output_path.with_name(f'{stem}_timeouts{suffix}') + + +def write_timeout_results( + output_path: Path, + *, + source_output_path: Path | None, + records: list[RequestRecord], +) -> None: + """Write timeout-only request records to a JSON file.""" + timeout_records = [ + record + for record in records + if record.timed_out + ] + payload = { + 'source_output_path': ( + str(source_output_path) if source_output_path is not None else None + ), + 'summary': { + 'timeouts': len(timeout_records), + 'total_requests': len(records), + }, + 'timeouts': [asdict(record) for record in timeout_records], + } + output_path.parent.mkdir(parents=True, exist_ok=True) + output_path.write_text( + json.dumps(payload, indent=2, sort_keys=True) + '\n', + encoding='utf-8', + ) + + +class DebugEventWriter: + """Write compact JSONL events for diagnosing client-side scheduling.""" + + def __init__(self, output_path: Path | None) -> None: + """Create an event writer for the optional output path.""" + self._output_path = output_path + self._stream = None + + def __enter__(self) -> 'DebugEventWriter': + """Open the event stream when debug output is enabled.""" + if self._output_path is not None: + self._output_path.parent.mkdir(parents=True, exist_ok=True) + self._stream = self._output_path.open('w', encoding='utf-8') + return self + + def __exit__(self, *args: object) -> None: + """Close the event stream when it was opened.""" + if self._stream is not None: + self._stream.close() + + def log(self, event: str, **fields: Any) -> None: + """Append one debug event if debug output is enabled.""" + if self._stream is None: + return + payload = { + 'event': event, + 'monotonic_s': time.monotonic(), + **fields, + } + self._stream.write(json.dumps(payload, sort_keys=True) + '\n') + + +def print_summary(summary: dict[str, Any]) -> None: + """Print compact experiment summary.""" + latency = summary['latency_s'] + print('ros_typedb stress experiment summary') + print(f" total_requests: {summary['total_requests']}") + print(f" successes: {summary['successes']}") + print(f" failures: {summary['failures']}") + print(f" timeouts: {summary['timeouts']}") + if latency: + print(f" latency_min_s: {latency['min']:.6f}") + print(f" latency_mean_s: {latency['mean']:.6f}") + print(f" latency_p50_s: {latency['p50']:.6f}") + print(f" latency_p90_s: {latency['p90']:.6f}") + print(f" latency_p95_s: {latency['p95']:.6f}") + print(f" latency_p99_s: {latency['p99']:.6f}") + print(f" latency_max_s: {latency['max']:.6f}") diff --git a/ros_typedb_tools/ros_typedb_tools/stress_runner.py b/ros_typedb_tools/ros_typedb_tools/stress_runner.py new file mode 100644 index 0000000..93a4a51 --- /dev/null +++ b/ros_typedb_tools/ros_typedb_tools/stress_runner.py @@ -0,0 +1,458 @@ +# Copyright 2026 Gustavo Rezende Silva +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +"""ROS service-client runner for the ros_typedb stress experiment.""" + +from __future__ import annotations + +import argparse +import time +from dataclasses import dataclass +from pathlib import Path +from typing import Any + +import rclpy +from rclpy.executors import MultiThreadedExecutor +from rclpy.executors import SingleThreadedExecutor + +from ros_typedb_msgs.srv import Query + +from ros_typedb_tools.stress_config import QuerySpec +from ros_typedb_tools.stress_config import RequestRecord +from ros_typedb_tools.stress_config import build_query_request +from ros_typedb_tools.stress_config import build_query_specs +from ros_typedb_tools.stress_config import validate_experiment_args +from ros_typedb_tools.stress_output import DebugEventWriter + + +@dataclass +class _PendingRequest: + client_id: int + index: int + query_index: int + query_spec: QuerySpec + future: Any + started_at_s: float + + +@dataclass +class _RunState: + records: list[RequestRecord] + pending_by_client: dict[int, _PendingRequest] + last_started_by_client: list[float] + next_index: int + deadline_s: float | None + next_snapshot_s: float + + +def _build_executor(args: argparse.Namespace) -> Any | None: + if args.executor == 'global': + return None + if args.executor == 'single': + return SingleThreadedExecutor() + if args.executor == 'multi': + return MultiThreadedExecutor(num_threads=args.executor_threads) + raise ValueError(f'unsupported executor: {args.executor}') + + +def _spin_once(node: Any, executor: Any | None) -> None: + if executor is None: + rclpy.spin_once(node, timeout_sec=0.01) + else: + executor.spin_once(timeout_sec=0.01) + + +def _should_send_request( + state: _RunState, + *, + requests: int, +) -> bool: + if state.deadline_s is not None: + return time.monotonic() < state.deadline_s + return state.next_index < requests + + +def _record_from_pending( + pending: _PendingRequest, + *, + ended_at_s: float, + success: bool, + timed_out: bool, + cancel_requested: bool, + error_message: str, + result_count: int, +) -> RequestRecord: + return RequestRecord( + index=pending.index, + client_id=pending.client_id, + query_index=pending.query_index, + query_type=pending.query_spec.query_type, + query=pending.query_spec.query, + started_at_s=pending.started_at_s, + ended_at_s=ended_at_s, + latency_s=ended_at_s - pending.started_at_s, + success=success, + timed_out=timed_out, + cancel_requested=cancel_requested, + error_message=error_message, + result_count=result_count, + ) + + +def _wait_for_all_clients( + clients: list[Any], + *, + service_name: str, + wait_service_timeout_s: float, + debug_events: DebugEventWriter, +) -> None: + wait_deadline_s = time.monotonic() + wait_service_timeout_s + for client_id, client in enumerate(clients): + wait_timeout_s = max(0.0, wait_deadline_s - time.monotonic()) + service_ready = client.wait_for_service(timeout_sec=wait_timeout_s) + debug_events.log( + 'service_wait_finished', + client_id=client_id, + ready=service_ready, + ) + if not service_ready: + raise RuntimeError( + f'Query service is not available: {service_name}' + ) + + +def _start_request( + *, + client_id: int, + clients: list[Any], + query_specs: list[QuerySpec], + state: _RunState, + timeout_s: float, + started_at_s: float, +) -> _PendingRequest: + query_index = state.next_index % len(query_specs) + query_spec = query_specs[query_index] + request = build_query_request( + query_spec.query, + query_spec.query_type, + timeout_s, + ) + future = clients[client_id].call_async(request) + pending = _PendingRequest( + client_id=client_id, + index=state.next_index, + query_index=query_index, + query_spec=query_spec, + future=future, + started_at_s=started_at_s, + ) + state.pending_by_client[client_id] = pending + state.last_started_by_client[client_id] = started_at_s + state.next_index += 1 + return pending + + +def _send_available_requests( + *, + args: argparse.Namespace, + clients: list[Any], + query_specs: list[QuerySpec], + state: _RunState, + max_in_flight: int, + debug_events: DebugEventWriter, +) -> None: + now_s = time.monotonic() + for client_id in range(args.clients): + if len(state.pending_by_client) >= max_in_flight: + break + if client_id in state.pending_by_client: + continue + if ( + args.request_gap_s > 0 + and now_s - state.last_started_by_client[client_id] + < args.request_gap_s + ): + continue + if not _should_send_request(state, requests=args.requests): + continue + pending = _start_request( + client_id=client_id, + clients=clients, + query_specs=query_specs, + state=state, + timeout_s=args.timeout_s, + started_at_s=now_s, + ) + debug_events.log( + 'request_sent', + client_id=client_id, + index=pending.index, + pending_count=len(state.pending_by_client), + query_index=pending.query_index, + query_type=pending.query_spec.query_type, + ) + + +def _log_pending_snapshot( + *, + state: _RunState, + now_s: float, + debug_events: DebugEventWriter, +) -> None: + if now_s < state.next_snapshot_s: + return + debug_events.log( + 'pending_snapshot', + next_index=state.next_index, + pending_count=len(state.pending_by_client), + pending_indices=[ + pending.index + for pending in state.pending_by_client.values() + ], + ) + state.next_snapshot_s = now_s + 1.0 + + +def _record_timeout( + *, + pending: _PendingRequest, + now_s: float, + pending_count: int, + state: _RunState, + debug_events: DebugEventWriter, +) -> None: + pending.future.cancel() + state.records.append( + _record_from_pending( + pending, + ended_at_s=now_s, + success=False, + timed_out=True, + cancel_requested=True, + error_message='client request timed out', + result_count=0, + ) + ) + debug_events.log( + 'request_timeout', + client_id=pending.client_id, + index=pending.index, + latency_s=now_s - pending.started_at_s, + pending_count=pending_count, + query_index=pending.query_index, + ) + + +def _record_done_future( + *, + pending: _PendingRequest, + now_s: float, + pending_count: int, + state: _RunState, + debug_events: DebugEventWriter, +) -> None: + debug_events.log( + 'future_done', + client_id=pending.client_id, + index=pending.index, + latency_s=now_s - pending.started_at_s, + pending_count=pending_count, + query_index=pending.query_index, + ) + exception = pending.future.exception() + if exception is not None: + state.records.append( + _record_from_pending( + pending, + ended_at_s=now_s, + success=False, + timed_out=False, + cancel_requested=False, + error_message=str(exception), + result_count=0, + ) + ) + debug_events.log( + 'request_exception', + client_id=pending.client_id, + error_message=str(exception), + index=pending.index, + ) + return + + response = pending.future.result() + state.records.append( + _record_from_pending( + pending, + ended_at_s=now_s, + success=response.success, + timed_out=False, + cancel_requested=False, + error_message=response.error_message, + result_count=len(response.results), + ) + ) + debug_events.log( + 'request_complete', + client_id=pending.client_id, + index=pending.index, + result_count=len(response.results), + success=response.success, + ) + + +def _process_pending_requests( + *, + args: argparse.Namespace, + state: _RunState, + now_s: float, + debug_events: DebugEventWriter, +) -> bool: + completed_client_ids = [] + timed_out_this_cycle = False + pending_count = len(state.pending_by_client) + + for client_id, pending in state.pending_by_client.items(): + future = pending.future + if not future.done(): + if now_s - pending.started_at_s < args.timeout_s: + continue + timed_out_this_cycle = True + _record_timeout( + pending=pending, + now_s=now_s, + pending_count=pending_count, + state=state, + debug_events=debug_events, + ) + completed_client_ids.append(client_id) + continue + + _record_done_future( + pending=pending, + now_s=now_s, + pending_count=pending_count, + state=state, + debug_events=debug_events, + ) + completed_client_ids.append(client_id) + + for client_id in completed_client_ids: + del state.pending_by_client[client_id] + + if timed_out_this_cycle: + debug_events.log( + 'timeout_barrier', + pending_count=len(state.pending_by_client), + ) + return timed_out_this_cycle + + +def run_experiment(args: argparse.Namespace) -> list[RequestRecord]: + """Run the stress experiment.""" + validate_experiment_args(args) + query_specs = build_query_specs(args) + max_in_flight = args.max_in_flight or args.clients + debug_events_output = ( + Path(args.debug_events_output).expanduser() + if args.debug_events_output + else None + ) + + node = rclpy.create_node('ros_typedb_stress_experiment') + executor = _build_executor(args) + if executor is not None: + executor.add_node(node) + clients = [ + node.create_client(Query, args.service_name) + for _ in range(args.clients) + ] + deadline_s = ( + time.monotonic() + args.duration_s + if args.duration_s is not None + else None + ) + state = _RunState( + records=[], + pending_by_client={}, + last_started_by_client=[float('-inf') for _ in range(args.clients)], + next_index=0, + deadline_s=deadline_s, + next_snapshot_s=time.monotonic() + 1.0, + ) + + try: + with DebugEventWriter(debug_events_output) as debug_events: + debug_events.log( + 'experiment_started', + clients=args.clients, + executor=args.executor, + max_in_flight=max_in_flight, + request_gap_s=args.request_gap_s, + ) + _wait_for_all_clients( + clients, + service_name=args.service_name, + wait_service_timeout_s=args.wait_service_timeout_s, + debug_events=debug_events, + ) + + while ( + _should_send_request(state, requests=args.requests) + or state.pending_by_client + ): + spin_started_s = time.monotonic() + _spin_once(node, executor) + now_s = time.monotonic() + debug_events.log( + 'spin_once', + duration_s=now_s - spin_started_s, + pending_count=len(state.pending_by_client), + ) + _log_pending_snapshot( + state=state, + now_s=now_s, + debug_events=debug_events, + ) + + timed_out_this_cycle = _process_pending_requests( + args=args, + state=state, + now_s=now_s, + debug_events=debug_events, + ) + if timed_out_this_cycle: + continue + + _send_available_requests( + args=args, + clients=clients, + query_specs=query_specs, + state=state, + max_in_flight=max_in_flight, + debug_events=debug_events, + ) + + debug_events.log( + 'experiment_finished', + records=len(state.records), + timeouts=sum( + 1 for record in state.records if record.timed_out + ), + ) + finally: + if executor is not None: + executor.remove_node(node) + executor.shutdown() + node.destroy_node() + return state.records diff --git a/ros_typedb_tools/setup.py b/ros_typedb_tools/setup.py index a58c152..64af5ae 100644 --- a/ros_typedb_tools/setup.py +++ b/ros_typedb_tools/setup.py @@ -43,8 +43,7 @@ ), ( 'ros_typedb_fake_query_service = ' - 'ros_typedb_tools.ros_typedb_stress_experiment:' - 'fake_query_service_main' + 'ros_typedb_tools.fake_query_service:fake_query_service_main' ), ], }, diff --git a/ros_typedb_tools/test/test_ros_typedb_stress_experiment.py b/ros_typedb_tools/test/test_ros_typedb_stress_experiment.py index 86e83f6..a38f3b3 100644 --- a/ros_typedb_tools/test/test_ros_typedb_stress_experiment.py +++ b/ros_typedb_tools/test/test_ros_typedb_stress_experiment.py @@ -6,19 +6,25 @@ import pytest +from ros_typedb_tools.fake_query_service import ( + build_fake_service_argument_parser, +) from ros_typedb_tools.ros_typedb_stress_experiment import ( - DebugEventWriter, + build_argument_parser, +) +from ros_typedb_tools.stress_config import ( QUERY_TYPE_BY_NAME, RequestRecord, - _build_query_specs, - build_argument_parser, - build_fake_service_argument_parser, + build_query_specs, +) +from ros_typedb_tools.stress_output import ( + DebugEventWriter, default_timeout_output_path, - run_experiment, summarize_records, write_results, write_timeout_results, ) +from ros_typedb_tools.stress_runner import run_experiment def _record( @@ -192,7 +198,7 @@ def test_write_results_writes_summary_and_request_records(tmp_path: Path): clients=2, duration_s=10.0, mode='read', - query_mix=_build_query_specs( + query_mix=build_query_specs( SimpleNamespace(query=None, query_type=None, mode='read') ), request_gap_s=0.01, @@ -277,7 +283,7 @@ def test_debug_event_writer_writes_json_lines(tmp_path: Path): def test_build_query_specs_uses_single_explicit_query(): """Check explicit query arguments override the built-in mix.""" - specs = _build_query_specs( + specs = build_query_specs( SimpleNamespace( query='match $x isa entity; fetch $x: attribute;', query_type='fetch', @@ -292,7 +298,7 @@ def test_build_query_specs_uses_single_explicit_query(): def test_build_query_specs_uses_read_query_mix_by_default(): """Check read mode provides runnable queries without explicit input.""" - specs = _build_query_specs( + specs = build_query_specs( SimpleNamespace(query=None, query_type=None, mode='read') ) @@ -309,7 +315,7 @@ def test_build_query_specs_rejects_query_without_query_type(): ) with pytest.raises(ValueError, match='--query-type'): - _build_query_specs(args) + build_query_specs(args) def test_run_experiment_rejects_invalid_request_count(): From 108a7037bee1fff3a609e1605ef99385dbf41107 Mon Sep 17 00:00:00 2001 From: Gustavo Date: Sat, 27 Jun 2026 19:00:13 -0300 Subject: [PATCH 20/36] Add stress invariant profiles --- AGENTS.md | 89 ++++++++ README.md | 63 ++++++ ros_typedb_tools/README.md | 78 +++++++ .../profiles/plan_schema_invariants.json | 47 +++++ .../profiles/test_data_invariants.json | 29 +++ .../ros_typedb_stress_experiment.py | 43 +++- .../ros_typedb_tools/stress_config.py | 89 ++++++++ .../ros_typedb_tools/stress_invariants.py | 114 +++++++++++ .../ros_typedb_tools/stress_output.py | 34 +++- .../ros_typedb_tools/stress_runner.py | 192 +++++++++++++++++- ros_typedb_tools/setup.py | 3 + .../test/test_ros_typedb_stress_experiment.py | 191 +++++++++++++++++ 12 files changed, 961 insertions(+), 11 deletions(-) create mode 100644 ros_typedb_tools/ros_typedb_tools/profiles/plan_schema_invariants.json create mode 100644 ros_typedb_tools/ros_typedb_tools/profiles/test_data_invariants.json create mode 100644 ros_typedb_tools/ros_typedb_tools/stress_invariants.py diff --git a/AGENTS.md b/AGENTS.md index 18c5b6b..19dd3aa 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -112,6 +112,95 @@ For TypeDB `fetch` queries, do not fetch a bare concept variable directly (`fetch $x;`). Fetch attributes explicitly; to fetch all attributes owned by a matched concept, use `fetch $x: attribute;`. +TypeDB `fetch` result shapes differ between explicit and wildcard attribute +fetches. Explicit fetches such as `fetch $x: email;` are keyed by the concrete +attribute label and usually contain a homogeneous list of values. Wildcard +fetches such as `fetch $x: attribute;` are keyed by `attribute` and can contain +mixed labels and value types in the same list. Normalize wildcard results by +grouping them by concrete `(label, value_type)` before converting them to ROS +`Attribute` messages. + +When changing ROS fetch-result conversion, cover both unit-level JSON/dict +shapes and launch integration tests against the real TypeDB service. The +existing integration fixture data already includes mixed attribute value types +on `person` (`string`, `long`, `double`, `boolean`, `datetime`) and is enough to +reproduce wildcard attribute conversion bugs without extending the fixture. + +ROS service callbacks should catch and log query execution or result conversion +exceptions, then return `success=False` with `error_message` populated. Do not +let malformed or unexpected query results escape the callback and crash the +`ros_typedb_interface` node. + +## ros_typedb_tools Stress Diagnostics + +The stress experiment CLI is intentionally split by responsibility: + +- `ros_typedb_stress_experiment.py`: thin stress CLI entry point. +- `stress_config.py`: query specs, request records, validation, and Query + request construction. +- `stress_output.py`: summaries, result JSON, timeout JSON, and debug JSONL + writing. +- `stress_runner.py`: ROS client loop, executor handling, service readiness, + pending future processing, and timeout-barrier behavior. +- `stress_invariants.py`: invariant response evaluation and scalar extraction. +- `fake_query_service.py`: fake `Query` service used to isolate ROS + service-client behavior from TypeDB work. + +When debugging Stage 2 stress timeouts, use the fake query service and the +scenario runner before changing `ros_typedb_interface` behavior. If the fake +service reproduces the same timeout-wave pattern, suspect ROS/rclpy/RMW/DDS or +stress-harness behavior rather than TypeDB queries, schema/data, or result +conversion. + +Observed stress-test boundary: high-rate ROS `Query` service calls can produce +timeout waves even with fake services that do no TypeDB work. This reproduced +with Python and C++ clients/servers, Fast DDS and CycloneDDS, and both Humble +and Lyrical environments. Treat this as a ROS service transport/executor stress +limit, not as evidence of a `ros_typedb` database/query bug. For meaningful +`ros_typedb` robustness tests, cap outstanding service requests; in local +testing `--max-in-flight 10` with `--request-gap-s 0.01` was stable, while +roughly 12-15 in-flight requests started producing timeout waves. + +Move on from this transport limit unless the task is specifically to diagnose +ROS service middleware behavior. Later stress stages should run inside the +bounded envelope and focus on driver behavior: mixed read/write workloads, +lifecycle transitions under load, TypeDB restart/reconnect, malformed query +handling, schema/data reload behavior, and long-duration soak tests. + +Stress invariants are correctness checks that run alongside load to verify the +database still contains expected baseline facts. Keep invariant profiles as +packaged JSON files under `ros_typedb_tools/ros_typedb_tools/profiles/`, not as +hard-coded Python data. Update `setup.py` package data when adding new profile +file patterns. Current built-in profiles are: + +- `test-data`: for `ros_typedb/test/typedb_test_data/schema.tql` and + `data.tql`; checks `person`, `company`, `employment`, and the + `boss@tudelft.nl` sentinel. +- `plan-schema`: for `ros_typedb_examples/data/plan_schema.tql` and + `plan_data.tql`; checks `Plan`, `Action`, `Proposition`, plan/action + relations, and the `collect-water-sample` sentinel action. + +Do not use a built-in invariant profile with a different schema/data pair. A +passing request-load test only proves the service answered; passing invariants +prove the expected baseline data remained visible. + +Useful stress diagnostics: + +```bash +bash src/ros_typedb/ros_typedb_tools/scripts/run_stage2_timeout_scenarios.sh +``` + +The script runs real read/global, real read/multi, fake read/global, fake +read/multi, and real explicit entity-get/global scenarios. Use +`--debug-events-output` traces to inspect `request_sent`, `future_done`, +`request_timeout`, `timeout_barrier`, and `pending_snapshot` events. + +When changing `setup.py` console entry points in an `ament_python` package, +`colcon build --symlink-install` may leave stale generated entry-point scripts. +If a `ros2 run` command still imports an old module after a source change, +remove that package's generated `build/` and `install/` directories +inside the workspace and rebuild the package. + For PR support notes under `docs-agents/`, keep changelogs brief: summary, compact bullet list of changes, and verification commands. Avoid copying the more verbose style from planning documents. diff --git a/README.md b/README.md index c0e229d..3085339 100644 --- a/README.md +++ b/README.md @@ -167,6 +167,69 @@ ros2 run ros_typedb_tools ros_typedb_stress_experiment \ Use `--service-name` if the node name is different from the default `/ros_typedb_interface/query`. +An invariant is a correctness check that should remain true while the stress +load is running. Request metrics show whether the service answered; invariants +show whether the database still contains expected baseline facts. + +Use `--invariant-profile test-data` only when `ros_typedb_interface` was started +with the bundled test schema/data from `ros_typedb/test/typedb_test_data/`: + +```bash +ros2 run ros_typedb_tools ros_typedb_stress_experiment \ + --clients 10 \ + --duration-s 60 \ + --timeout-s 10 \ + --mode read \ + --invariant-profile test-data \ + --output /tmp/ros_typedb_read_invariants_results.json +``` + +The `test-data` profile is stored in +`ros_typedb_tools/ros_typedb_tools/profiles/test_data_invariants.json`. It +periodically checks stable counts for `person`, `company`, and `employment`, +plus the `boss@tudelft.nl` sentinel person. This profile is not appropriate for +other schemas, such as the `ros_typedb_examples` plan schema. The command exits +with status 1 if an invariant fails. + +For the `ros_typedb_examples` plan schema/data, use: + +```bash +ros2 run ros_typedb_tools ros_typedb_stress_experiment \ + --clients 10 \ + --duration-s 60 \ + --timeout-s 10 \ + --mode read \ + --invariant-profile plan-schema \ + --output /tmp/ros_typedb_plan_invariants_results.json +``` + +The `plan-schema` profile is stored in +`ros_typedb_tools/ros_typedb_tools/profiles/plan_schema_invariants.json`. It +checks counts for `Plan`, `Action`, `Proposition`, and the plan/action relation +types, plus a sentinel action named `collect-water-sample`. + +For robustness experiments, keep the ROS service load bounded. High-rate +unbounded service calls produced timeout waves even with fake Query services +implemented in both Python and C++, across Fast DDS, CycloneDDS, ROS Humble, +and ROS Lyrical. That points to a ROS service transport/executor stress limit +rather than TypeDB query execution or `ros_typedb` result conversion. + +A practical bounded read-stress baseline is: + +```bash +ros2 run ros_typedb_tools ros_typedb_stress_experiment \ + --clients 20 \ + --duration-s 60 \ + --timeout-s 10 \ + --mode read \ + --request-gap-s 0.01 \ + --max-in-flight 10 \ + --output /tmp/ros_typedb_read_bounded_results.json +``` + +Use higher in-flight counts only when intentionally testing ROS service +transport saturation, not when validating `ros_typedb` database robustness. + ## Extend the package To extend this package with custom functionalities, you can create a new ROS Node inheriting from ROSTypeDBInterface and a new typedb interface inheriting from TypeDBInterface. Then you simply need to add the new functionalities you need into your class. diff --git a/ros_typedb_tools/README.md b/ros_typedb_tools/README.md index 857cd51..ce86c8c 100644 --- a/ros_typedb_tools/README.md +++ b/ros_typedb_tools/README.md @@ -150,6 +150,52 @@ The command exits successfully when the experiment runs to completion, even if some requests fail. Use `--fail-on-failure` when a nonzero exit code is useful for CI or scripted thresholds. +An invariant is a correctness check that should remain true while the stress +load is running. Request metrics show whether the service answered; invariants +show whether the database still contains expected baseline facts. + +Correctness invariant example for the bundled `ros_typedb` test schema/data: + +```bash +ros2 run ros_typedb_tools ros_typedb_stress_experiment \ + --clients 10 \ + --duration-s 60 \ + --timeout-s 10 \ + --mode read \ + --invariant-profile test-data \ + --output ~/results/ros_typedb_read_invariants_results.json +``` + +Use `--invariant-profile test-data` only when `ros_typedb_interface` was started +with the schema/data from `ros_typedb/test/typedb_test_data/`. The profile is +stored in `ros_typedb_tools/ros_typedb_tools/profiles/test_data_invariants.json` +and checks stable aggregate counts for `person`, `company`, and `employment`, +plus the `boss@tudelft.nl` sentinel person. This profile is not appropriate for +other schemas, such as the `ros_typedb_examples` plan schema. + +For `ros_typedb_examples/data/plan_schema.tql` and `plan_data.tql`, use: + +```bash +ros2 run ros_typedb_tools ros_typedb_stress_experiment \ + --clients 10 \ + --duration-s 60 \ + --timeout-s 10 \ + --mode read \ + --invariant-profile plan-schema \ + --output ~/results/ros_typedb_plan_invariants_results.json +``` + +The `plan-schema` profile is stored in +`ros_typedb_tools/ros_typedb_tools/profiles/plan_schema_invariants.json` and +checks counts for `Plan`, `Action`, `Proposition`, and the plan/action relation +types, plus a sentinel action named `collect-water-sample`. + +Checks run every 10 seconds by default and once at the end. Use +`--invariant-period-s` to adjust the periodic interval, or +`--invariant-period-s 0` to run only the final check. Invariant failures are +reported separately from request failures and always make the command exit with +status 1. + For ROS client/executor debugging, `--debug-events-output` writes JSONL events for request sends, `spin_once()` calls, completed futures, timeouts, and periodic pending-request snapshots. Use `--executor single` or `--executor multi` to @@ -158,6 +204,30 @@ Use `--request-gap-s` to add a small per-client delay between requests, or `--max-in-flight` to cap total outstanding service requests independently of the number of clients. +For `ros_typedb` robustness experiments, prefer bounded service concurrency. +High-rate unbounded Query service calls produced timeout waves even when TypeDB +was removed from the path by using fake Query services. The same behavior was +observed with Python and C++ clients/servers, Fast DDS and CycloneDDS, and ROS +Humble and Lyrical. Treat that as a ROS service transport/executor stress limit, +not as a database or result-conversion failure. + +A practical bounded baseline is: + +```bash +ros2 run ros_typedb_tools ros_typedb_stress_experiment \ + --clients 20 \ + --duration-s 60 \ + --timeout-s 10 \ + --mode read \ + --request-gap-s 0.01 \ + --max-in-flight 10 \ + --output ~/results/ros_typedb_read_bounded_results.json +``` + +Use larger `--max-in-flight` values only when intentionally probing ROS service +transport saturation. In local tests, about 12-15 in-flight requests started to +produce timeout waves, while 10 or fewer stayed stable. + To isolate ROS service/client behavior from TypeDB work, run a fake Query service in one terminal: @@ -193,6 +263,9 @@ Useful options: - `--max-in-flight`: maximum number of outstanding requests across all clients - `--executor`: client executor mode, one of `global`, `single`, or `multi` - `--debug-events-output`: optional JSONL debug event path +- `--invariant-profile`: optional correctness profile, such as `test-data` or + `plan-schema` +- `--invariant-period-s`: seconds between periodic invariant checks - `--timeout-s`: per-request client and server timeout - `--output`: optional JSON results path - `--timeout-output`: optional timeout-only JSON results path @@ -223,3 +296,8 @@ Start new terminal in the container: ```Bash docker exec -it ros_typedb bash ``` + + + export RMW_IMPLEMENTATION=rmw_fastrtps_cpp + +export RMW_IMPLEMENTATION=rmw_cyclonedds_cpp diff --git a/ros_typedb_tools/ros_typedb_tools/profiles/plan_schema_invariants.json b/ros_typedb_tools/ros_typedb_tools/profiles/plan_schema_invariants.json new file mode 100644 index 0000000..ec56818 --- /dev/null +++ b/ros_typedb_tools/ros_typedb_tools/profiles/plan_schema_invariants.json @@ -0,0 +1,47 @@ +{ + "description": "Correctness invariants for ros_typedb_examples/data/plan_schema.tql and plan_data.tql.", + "invariants": [ + { + "name": "plan-count", + "query": "match $p isa Plan; get $p; count;", + "query_type": "get_aggregate", + "expected_value": 2 + }, + { + "name": "action-count", + "query": "match $a isa Action; get $a; count;", + "query_type": "get_aggregate", + "expected_value": 4 + }, + { + "name": "proposition-count", + "query": "match $p isa Proposition; get $p; count;", + "query_type": "get_aggregate", + "expected_value": 7 + }, + { + "name": "has-action-count", + "query": "match $r isa has_action; get $r; count;", + "query_type": "get_aggregate", + "expected_value": 6 + }, + { + "name": "action-preconditions-count", + "query": "match $r isa action_preconditions; get $r; count;", + "query_type": "get_aggregate", + "expected_value": 6 + }, + { + "name": "action-effects-count", + "query": "match $r isa action_effects; get $r; count;", + "query_type": "get_aggregate", + "expected_value": 4 + }, + { + "name": "collect-water-sample-sentinel-count", + "query": "match $a isa Action, has action_name \"collect-water-sample\"; get $a; count;", + "query_type": "get_aggregate", + "expected_value": 1 + } + ] +} diff --git a/ros_typedb_tools/ros_typedb_tools/profiles/test_data_invariants.json b/ros_typedb_tools/ros_typedb_tools/profiles/test_data_invariants.json new file mode 100644 index 0000000..f5fe7ab --- /dev/null +++ b/ros_typedb_tools/ros_typedb_tools/profiles/test_data_invariants.json @@ -0,0 +1,29 @@ +{ + "description": "Correctness invariants for ros_typedb/test/typedb_test_data/schema.tql and data.tql.", + "invariants": [ + { + "name": "person-count", + "query": "match $p isa person; get $p; count;", + "query_type": "get_aggregate", + "expected_value": 20 + }, + { + "name": "company-count", + "query": "match $c isa company; get $c; count;", + "query_type": "get_aggregate", + "expected_value": 1 + }, + { + "name": "employment-count", + "query": "match $e isa employment; get $e; count;", + "query_type": "get_aggregate", + "expected_value": 3 + }, + { + "name": "boss-sentinel-count", + "query": "match $p isa person, has email \"boss@tudelft.nl\"; get $p; count;", + "query_type": "get_aggregate", + "expected_value": 1 + } + ] +} diff --git a/ros_typedb_tools/ros_typedb_tools/ros_typedb_stress_experiment.py b/ros_typedb_tools/ros_typedb_tools/ros_typedb_stress_experiment.py index 11ec31e..75a19bf 100644 --- a/ros_typedb_tools/ros_typedb_tools/ros_typedb_stress_experiment.py +++ b/ros_typedb_tools/ros_typedb_tools/ros_typedb_stress_experiment.py @@ -22,9 +22,12 @@ import rclpy from ros_typedb_tools.stress_config import QUERY_TYPE_BY_NAME +from ros_typedb_tools.stress_config import INVARIANT_PROFILE_NAMES +from ros_typedb_tools.stress_config import StressExperimentResult from ros_typedb_tools.stress_config import build_query_specs from ros_typedb_tools.stress_output import default_timeout_output_path from ros_typedb_tools.stress_output import print_summary +from ros_typedb_tools.stress_output import summarize_invariant_records from ros_typedb_tools.stress_output import summarize_records from ros_typedb_tools.stress_output import write_results from ros_typedb_tools.stress_output import write_timeout_results @@ -119,6 +122,24 @@ def build_argument_parser() -> argparse.ArgumentParser: 'Client and server timeout per request in seconds. Defaults to 10.' ), ) + parser.add_argument( + '--invariant-profile', + choices=INVARIANT_PROFILE_NAMES, + default='none', + help=( + 'Optional built-in invariant profile. Use test-data with the ' + 'bundled ros_typedb test schema/data. Defaults to none.' + ), + ) + parser.add_argument( + '--invariant-period-s', + type=float, + default=10.0, + help=( + 'Seconds between periodic invariant checks. Use 0 to run only ' + 'the final check. Defaults to 10.' + ), + ) parser.add_argument( '--wait-service-timeout-s', type=float, @@ -154,7 +175,7 @@ def build_argument_parser() -> argparse.ArgumentParser: def _write_requested_outputs( args: argparse.Namespace, - records, + result: StressExperimentResult, ) -> None: output_path = None if args.output: @@ -166,7 +187,7 @@ def _write_requested_outputs( query_type=args.query_type, query=args.query, timeout_s=args.timeout_s, - records=records, + records=result.records, clients=args.clients, duration_s=args.duration_s, mode=args.mode, @@ -174,6 +195,9 @@ def _write_requested_outputs( request_gap_s=args.request_gap_s, max_in_flight=args.max_in_flight, executor=args.executor, + invariant_profile=args.invariant_profile, + invariant_period_s=args.invariant_period_s, + invariant_records=result.invariant_records, ) print(f' output: {output_path}') @@ -186,7 +210,7 @@ def _write_requested_outputs( write_timeout_results( timeout_output_path, source_output_path=output_path, - records=records, + records=result.records, ) print(f' timeout_output: {timeout_output_path}') @@ -198,10 +222,15 @@ def main(argv: list[str] | None = None) -> int: rclpy.init(args=None) try: - records = run_experiment(args) - summary = summarize_records(records) - print_summary(summary) - _write_requested_outputs(args, records) + result = run_experiment(args) + summary = summarize_records(result.records) + invariant_summary = summarize_invariant_records( + result.invariant_records + ) + print_summary(summary, invariant_summary) + _write_requested_outputs(args, result) + if invariant_summary['failures'] > 0: + return 1 if args.fail_on_failure and summary['failures'] > 0: return 1 return 0 diff --git a/ros_typedb_tools/ros_typedb_tools/stress_config.py b/ros_typedb_tools/ros_typedb_tools/stress_config.py index 5db231c..4cfabdd 100644 --- a/ros_typedb_tools/ros_typedb_tools/stress_config.py +++ b/ros_typedb_tools/ros_typedb_tools/stress_config.py @@ -16,7 +16,10 @@ from __future__ import annotations import argparse +import json from dataclasses import dataclass +from pathlib import Path +from typing import Any from ros_typedb_msgs.srv import Query @@ -46,6 +49,13 @@ ), ) +INVARIANT_PROFILE_FILES = { + 'plan-schema': 'plan_schema_invariants.json', + 'test-data': 'test_data_invariants.json', +} +INVARIANT_PROFILE_NAMES = ('none', *INVARIANT_PROFILE_FILES) +INVARIANT_PROFILE_DIR = Path(__file__).with_name('profiles') + @dataclass(frozen=True) class QuerySpec: @@ -74,6 +84,43 @@ class RequestRecord: result_count: int +@dataclass(frozen=True) +class InvariantSpec: + """Query and expected scalar value for one invariant check.""" + + name: str + query: str + query_type: str + expected_value: Any + + +@dataclass(frozen=True) +class InvariantRecord: + """Result of one invariant query.""" + + index: int + phase: str + name: str + query_type: str + query: str + started_at_s: float + ended_at_s: float + latency_s: float + success: bool + timed_out: bool + error_message: str + expected_value: Any + actual_value: Any + + +@dataclass(frozen=True) +class StressExperimentResult: + """Complete stress experiment result.""" + + records: list[RequestRecord] + invariant_records: list[InvariantRecord] + + def validate_experiment_args(args: argparse.Namespace) -> None: """Validate stress arguments before ROS resources are created.""" if args.requests < 1: @@ -94,6 +141,14 @@ def validate_experiment_args(args: argparse.Namespace) -> None: raise ValueError('--max-in-flight must be greater than zero') if args.executor_threads < 1: raise ValueError('--executor-threads must be greater than zero') + if args.invariant_profile not in INVARIANT_PROFILE_NAMES: + raise ValueError( + f'unsupported invariant profile: {args.invariant_profile}' + ) + if args.invariant_period_s < 0: + raise ValueError( + '--invariant-period-s must be greater than or equal to zero' + ) def build_query_specs(args: argparse.Namespace) -> list[QuerySpec]: @@ -115,6 +170,40 @@ def build_query_specs(args: argparse.Namespace) -> list[QuerySpec]: raise ValueError(f'unsupported mode: {args.mode}') +def build_invariant_specs(args: argparse.Namespace) -> list[InvariantSpec]: + """Build invariant checks for the selected invariant profile.""" + if args.invariant_profile == 'none': + return [] + + profile_filename = INVARIANT_PROFILE_FILES.get(args.invariant_profile) + if profile_filename is None: + raise ValueError( + f'unsupported invariant profile: {args.invariant_profile}' + ) + + return _load_invariant_profile(INVARIANT_PROFILE_DIR / profile_filename) + + +def _load_invariant_profile(profile_path: Path) -> list[InvariantSpec]: + """Load invariant checks from a packaged JSON profile.""" + payload = json.loads(profile_path.read_text(encoding='utf-8')) + invariants = payload.get('invariants', []) + specs = [] + for invariant in invariants: + query_type = invariant['query_type'] + if query_type not in QUERY_TYPE_BY_NAME: + raise ValueError(f'unsupported invariant query type: {query_type}') + specs.append( + InvariantSpec( + name=invariant['name'], + query=invariant['query'], + query_type=query_type, + expected_value=invariant['expected_value'], + ) + ) + return specs + + def build_query_request( query: str, query_type: str, diff --git a/ros_typedb_tools/ros_typedb_tools/stress_invariants.py b/ros_typedb_tools/ros_typedb_tools/stress_invariants.py new file mode 100644 index 0000000..98ba879 --- /dev/null +++ b/ros_typedb_tools/ros_typedb_tools/stress_invariants.py @@ -0,0 +1,114 @@ +# Copyright 2026 Gustavo Rezende Silva +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +"""Invariant evaluation helpers for ros_typedb stress experiments.""" + +from __future__ import annotations + +import time +from typing import Any + +from rcl_interfaces.msg import ParameterType + +from ros_typedb_tools.stress_config import InvariantRecord +from ros_typedb_tools.stress_config import InvariantSpec + + +def _parameter_value_to_python(parameter_value: Any) -> Any: + """Convert a ROS ParameterValue into the Python scalar it carries.""" + if parameter_value.type == ParameterType.PARAMETER_BOOL: + return parameter_value.bool_value + if parameter_value.type == ParameterType.PARAMETER_INTEGER: + return parameter_value.integer_value + if parameter_value.type == ParameterType.PARAMETER_DOUBLE: + return parameter_value.double_value + if parameter_value.type == ParameterType.PARAMETER_STRING: + return parameter_value.string_value + return None + + +def _extract_first_scalar(response: Any) -> Any: + """Extract the first scalar attribute value from a Query response.""" + if not response.results: + return None + if not response.results[0].results: + return None + first_result = response.results[0].results[0] + return _parameter_value_to_python(first_result.attribute.value) + + +def evaluate_invariant_response( + *, + index: int, + phase: str, + spec: InvariantSpec, + started_at_s: float, + ended_at_s: float, + response: Any | None, + timed_out: bool = False, + exception_message: str = '', +) -> InvariantRecord: + """Evaluate one invariant service response.""" + actual_value = _extract_first_scalar(response) if response else None + error_parts = [] + + if timed_out: + error_parts.append('invariant request timed out') + if exception_message: + error_parts.append(exception_message) + if response is None: + error_parts.append('no response') + elif not response.success: + error_parts.append( + response.error_message or 'service returned failure' + ) + if actual_value != spec.expected_value: + error_parts.append( + f'expected {spec.expected_value!r}, got {actual_value!r}' + ) + + return InvariantRecord( + index=index, + phase=phase, + name=spec.name, + query_type=spec.query_type, + query=spec.query, + started_at_s=started_at_s, + ended_at_s=ended_at_s, + latency_s=ended_at_s - started_at_s, + success=not error_parts, + timed_out=timed_out, + error_message='; '.join(error_parts), + expected_value=spec.expected_value, + actual_value=actual_value, + ) + + +def build_timeout_invariant_record( + *, + index: int, + phase: str, + spec: InvariantSpec, + started_at_s: float, +) -> InvariantRecord: + """Build an invariant record for a client-side timeout.""" + ended_at_s = time.monotonic() + return evaluate_invariant_response( + index=index, + phase=phase, + spec=spec, + started_at_s=started_at_s, + ended_at_s=ended_at_s, + response=None, + timed_out=True, + ) diff --git a/ros_typedb_tools/ros_typedb_tools/stress_output.py b/ros_typedb_tools/ros_typedb_tools/stress_output.py index c8427bf..3e9bcb9 100644 --- a/ros_typedb_tools/ros_typedb_tools/stress_output.py +++ b/ros_typedb_tools/ros_typedb_tools/stress_output.py @@ -23,6 +23,7 @@ from pathlib import Path from typing import Any +from ros_typedb_tools.stress_config import InvariantRecord from ros_typedb_tools.stress_config import QuerySpec from ros_typedb_tools.stress_config import RequestRecord @@ -72,6 +73,21 @@ def summarize_records(records: list[RequestRecord]) -> dict[str, Any]: return summary +def summarize_invariant_records( + records: list[InvariantRecord], +) -> dict[str, Any]: + """Build summary metrics from invariant records.""" + successes = sum(1 for record in records if record.success) + timeouts = sum(1 for record in records if record.timed_out) + failures = len(records) - successes + return { + 'total_invariants': len(records), + 'successes': successes, + 'failures': failures, + 'timeouts': timeouts, + } + + def write_results( output_path: Path, *, @@ -87,8 +103,12 @@ def write_results( request_gap_s: float = 0.0, max_in_flight: int | None = None, executor: str = 'global', + invariant_profile: str = 'none', + invariant_period_s: float = 10.0, + invariant_records: list[InvariantRecord] | None = None, ) -> None: """Write experiment results to a JSON file.""" + invariants = invariant_records or [] payload = { 'service_name': service_name, 'mode': mode, @@ -97,6 +117,8 @@ def write_results( 'request_gap_s': request_gap_s, 'max_in_flight': max_in_flight, 'executor': executor, + 'invariant_profile': invariant_profile, + 'invariant_period_s': invariant_period_s, 'query_type': query_type, 'query': query, 'query_mix': [ @@ -104,7 +126,9 @@ def write_results( ], 'timeout_s': timeout_s, 'summary': summarize_records(records), + 'invariant_summary': summarize_invariant_records(invariants), 'requests': [asdict(record) for record in records], + 'invariants': [asdict(record) for record in invariants], } output_path.parent.mkdir(parents=True, exist_ok=True) output_path.write_text( @@ -181,7 +205,10 @@ def log(self, event: str, **fields: Any) -> None: self._stream.write(json.dumps(payload, sort_keys=True) + '\n') -def print_summary(summary: dict[str, Any]) -> None: +def print_summary( + summary: dict[str, Any], + invariant_summary: dict[str, Any] | None = None, +) -> None: """Print compact experiment summary.""" latency = summary['latency_s'] print('ros_typedb stress experiment summary') @@ -197,3 +224,8 @@ def print_summary(summary: dict[str, Any]) -> None: print(f" latency_p95_s: {latency['p95']:.6f}") print(f" latency_p99_s: {latency['p99']:.6f}") print(f" latency_max_s: {latency['max']:.6f}") + if invariant_summary is not None: + print(f" invariants_total: {invariant_summary['total_invariants']}") + print(f" invariants_successes: {invariant_summary['successes']}") + print(f" invariants_failures: {invariant_summary['failures']}") + print(f" invariants_timeouts: {invariant_summary['timeouts']}") diff --git a/ros_typedb_tools/ros_typedb_tools/stress_runner.py b/ros_typedb_tools/ros_typedb_tools/stress_runner.py index 93a4a51..a8b2203 100644 --- a/ros_typedb_tools/ros_typedb_tools/stress_runner.py +++ b/ros_typedb_tools/ros_typedb_tools/stress_runner.py @@ -27,11 +27,17 @@ from ros_typedb_msgs.srv import Query +from ros_typedb_tools.stress_config import InvariantRecord +from ros_typedb_tools.stress_config import InvariantSpec from ros_typedb_tools.stress_config import QuerySpec from ros_typedb_tools.stress_config import RequestRecord +from ros_typedb_tools.stress_config import StressExperimentResult +from ros_typedb_tools.stress_config import build_invariant_specs from ros_typedb_tools.stress_config import build_query_request from ros_typedb_tools.stress_config import build_query_specs from ros_typedb_tools.stress_config import validate_experiment_args +from ros_typedb_tools.stress_invariants import build_timeout_invariant_record +from ros_typedb_tools.stress_invariants import evaluate_invariant_response from ros_typedb_tools.stress_output import DebugEventWriter @@ -48,11 +54,14 @@ class _PendingRequest: @dataclass class _RunState: records: list[RequestRecord] + invariant_records: list[InvariantRecord] pending_by_client: dict[int, _PendingRequest] last_started_by_client: list[float] next_index: int + next_invariant_index: int deadline_s: float | None next_snapshot_s: float + next_invariant_s: float def _build_executor(args: argparse.Namespace) -> Any | None: @@ -358,10 +367,146 @@ def _process_pending_requests( return timed_out_this_cycle -def run_experiment(args: argparse.Namespace) -> list[RequestRecord]: +def _run_invariant_check( + *, + node: Any, + executor: Any | None, + client: Any, + spec: InvariantSpec, + index: int, + phase: str, + timeout_s: float, + debug_events: DebugEventWriter, +) -> InvariantRecord: + started_at_s = time.monotonic() + request = build_query_request( + spec.query, + spec.query_type, + timeout_s, + ) + future = client.call_async(request) + debug_events.log( + 'invariant_sent', + index=index, + name=spec.name, + phase=phase, + ) + + while True: + _spin_once(node, executor) + now_s = time.monotonic() + if future.done(): + exception = future.exception() + if exception is not None: + record = evaluate_invariant_response( + index=index, + phase=phase, + spec=spec, + started_at_s=started_at_s, + ended_at_s=now_s, + response=None, + exception_message=str(exception), + ) + else: + record = evaluate_invariant_response( + index=index, + phase=phase, + spec=spec, + started_at_s=started_at_s, + ended_at_s=now_s, + response=future.result(), + ) + debug_events.log( + 'invariant_complete', + index=index, + name=spec.name, + phase=phase, + success=record.success, + latency_s=record.latency_s, + error_message=record.error_message, + ) + return record + + if now_s - started_at_s >= timeout_s: + future.cancel() + record = build_timeout_invariant_record( + index=index, + phase=phase, + spec=spec, + started_at_s=started_at_s, + ) + debug_events.log( + 'invariant_timeout', + index=index, + name=spec.name, + phase=phase, + latency_s=record.latency_s, + ) + return record + + +def _run_invariant_profile( + *, + node: Any, + executor: Any | None, + client: Any, + specs: list[InvariantSpec], + state: _RunState, + phase: str, + timeout_s: float, + debug_events: DebugEventWriter, +) -> None: + for spec in specs: + record = _run_invariant_check( + node=node, + executor=executor, + client=client, + spec=spec, + index=state.next_invariant_index, + phase=phase, + timeout_s=timeout_s, + debug_events=debug_events, + ) + state.invariant_records.append(record) + state.next_invariant_index += 1 + + +def _maybe_run_periodic_invariants( + *, + args: argparse.Namespace, + node: Any, + executor: Any | None, + invariant_client: Any | None, + invariant_specs: list[InvariantSpec], + state: _RunState, + now_s: float, + debug_events: DebugEventWriter, +) -> None: + if not invariant_specs or invariant_client is None: + return + if args.invariant_period_s == 0: + return + if now_s < state.next_invariant_s: + return + + _run_invariant_profile( + node=node, + executor=executor, + client=invariant_client, + specs=invariant_specs, + state=state, + phase='periodic', + timeout_s=args.timeout_s, + debug_events=debug_events, + ) + state.next_invariant_s = time.monotonic() + args.invariant_period_s + + +def run_experiment(args: argparse.Namespace) -> StressExperimentResult: """Run the stress experiment.""" validate_experiment_args(args) query_specs = build_query_specs(args) + invariant_specs = build_invariant_specs(args) max_in_flight = args.max_in_flight or args.clients debug_events_output = ( Path(args.debug_events_output).expanduser() @@ -377,6 +522,11 @@ def run_experiment(args: argparse.Namespace) -> list[RequestRecord]: node.create_client(Query, args.service_name) for _ in range(args.clients) ] + invariant_client = ( + node.create_client(Query, args.service_name) + if invariant_specs + else None + ) deadline_s = ( time.monotonic() + args.duration_s if args.duration_s is not None @@ -384,11 +534,14 @@ def run_experiment(args: argparse.Namespace) -> list[RequestRecord]: ) state = _RunState( records=[], + invariant_records=[], pending_by_client={}, last_started_by_client=[float('-inf') for _ in range(args.clients)], next_index=0, + next_invariant_index=0, deadline_s=deadline_s, next_snapshot_s=time.monotonic() + 1.0, + next_invariant_s=time.monotonic() + args.invariant_period_s, ) try: @@ -399,9 +552,11 @@ def run_experiment(args: argparse.Namespace) -> list[RequestRecord]: executor=args.executor, max_in_flight=max_in_flight, request_gap_s=args.request_gap_s, + invariant_profile=args.invariant_profile, + invariant_period_s=args.invariant_period_s, ) _wait_for_all_clients( - clients, + clients + ([invariant_client] if invariant_client else []), service_name=args.service_name, wait_service_timeout_s=args.wait_service_timeout_s, debug_events=debug_events, @@ -442,17 +597,48 @@ def run_experiment(args: argparse.Namespace) -> list[RequestRecord]: max_in_flight=max_in_flight, debug_events=debug_events, ) + _maybe_run_periodic_invariants( + args=args, + node=node, + executor=executor, + invariant_client=invariant_client, + invariant_specs=invariant_specs, + state=state, + now_s=now_s, + debug_events=debug_events, + ) + + if invariant_specs and invariant_client is not None: + _run_invariant_profile( + node=node, + executor=executor, + client=invariant_client, + specs=invariant_specs, + state=state, + phase='final', + timeout_s=args.timeout_s, + debug_events=debug_events, + ) debug_events.log( 'experiment_finished', records=len(state.records), + invariant_records=len(state.invariant_records), timeouts=sum( 1 for record in state.records if record.timed_out ), + invariant_failures=sum( + 1 + for record in state.invariant_records + if not record.success + ), ) finally: if executor is not None: executor.remove_node(node) executor.shutdown() node.destroy_node() - return state.records + return StressExperimentResult( + records=state.records, + invariant_records=state.invariant_records, + ) diff --git a/ros_typedb_tools/setup.py b/ros_typedb_tools/setup.py index 64af5ae..ef20ffa 100644 --- a/ros_typedb_tools/setup.py +++ b/ros_typedb_tools/setup.py @@ -8,6 +8,9 @@ name=package_name, version='0.0.0', packages=find_packages(exclude=['test']), + package_data={ + package_name: ['profiles/*.json'], + }, data_files=[ ('share/ament_index/resource_index/packages', ['resource/' + package_name]), diff --git a/ros_typedb_tools/test/test_ros_typedb_stress_experiment.py b/ros_typedb_tools/test/test_ros_typedb_stress_experiment.py index a38f3b3..0a2e2be 100644 --- a/ros_typedb_tools/test/test_ros_typedb_stress_experiment.py +++ b/ros_typedb_tools/test/test_ros_typedb_stress_experiment.py @@ -5,6 +5,8 @@ from types import SimpleNamespace import pytest +from rcl_interfaces.msg import ParameterType +from rcl_interfaces.msg import ParameterValue from ros_typedb_tools.fake_query_service import ( build_fake_service_argument_parser, @@ -14,17 +16,25 @@ ) from ros_typedb_tools.stress_config import ( QUERY_TYPE_BY_NAME, + InvariantRecord, + InvariantSpec, RequestRecord, + build_invariant_specs, build_query_specs, ) +from ros_typedb_tools.stress_invariants import evaluate_invariant_response from ros_typedb_tools.stress_output import ( DebugEventWriter, default_timeout_output_path, + summarize_invariant_records, summarize_records, write_results, write_timeout_results, ) from ros_typedb_tools.stress_runner import run_experiment +from ros_typedb_msgs.msg import Attribute +from ros_typedb_msgs.msg import QueryResult +from ros_typedb_msgs.msg import ResultTree def _record( @@ -51,6 +61,47 @@ def _record( ) +def _invariant_record( + index: int, + *, + success: bool, + timed_out: bool = False, +) -> InvariantRecord: + return InvariantRecord( + index=index, + phase='final', + name='person-count', + query_type='get_aggregate', + query='match $p isa person; get $p; count;', + started_at_s=20.0 + index, + ended_at_s=20.1 + index, + latency_s=0.1, + success=success, + timed_out=timed_out, + error_message='' if success else 'failed', + expected_value=20, + actual_value=20 if success else 0, + ) + + +def _aggregate_count_response(value: int, *, success: bool = True): + parameter_value = ParameterValue() + parameter_value.type = ParameterType.PARAMETER_INTEGER + parameter_value.integer_value = value + attribute = Attribute() + attribute.value = parameter_value + query_result = QueryResult() + query_result.type = QueryResult.ATTRIBUTE + query_result.attribute = attribute + result_tree = ResultTree() + result_tree.results.append(query_result) + return SimpleNamespace( + success=success, + error_message='', + results=[result_tree], + ) + + def test_parser_accepts_stage_one_arguments(): """Check the Stage 1 command-line arguments.""" parser = build_argument_parser() @@ -124,6 +175,29 @@ def test_parser_accepts_stage_two_arguments(): assert args.executor == 'global' +def test_parser_accepts_stage_three_arguments(): + """Check Stage 3 invariant arguments.""" + parser = build_argument_parser() + + args = parser.parse_args( + [ + '--clients', + '10', + '--duration-s', + '60', + '--mode', + 'read', + '--invariant-profile', + 'test-data', + '--invariant-period-s', + '5', + ] + ) + + assert args.invariant_profile == 'test-data' + assert args.invariant_period_s == 5 + + def test_fake_service_parser_accepts_diagnostic_arguments(): """Check the fake service CLI options.""" parser = build_fake_service_argument_parser() @@ -183,6 +257,22 @@ def test_summarize_records_counts_success_failure_and_timeouts(): assert summary['latency_s']['max'] == pytest.approx(0.4) +def test_summarize_invariant_records_counts_failures_and_timeouts(): + """Check invariant summary counters.""" + records = [ + _invariant_record(0, success=True), + _invariant_record(1, success=False), + _invariant_record(2, success=False, timed_out=True), + ] + + summary = summarize_invariant_records(records) + + assert summary['total_invariants'] == 3 + assert summary['successes'] == 1 + assert summary['failures'] == 2 + assert summary['timeouts'] == 1 + + def test_write_results_writes_summary_and_request_records(tmp_path: Path): """Check JSON output includes metadata, summary, and records.""" output_path = tmp_path / 'stress' / 'results.json' @@ -204,6 +294,9 @@ def test_write_results_writes_summary_and_request_records(tmp_path: Path): request_gap_s=0.01, max_in_flight=1, executor='single', + invariant_profile='test-data', + invariant_period_s=5.0, + invariant_records=[_invariant_record(0, success=True)], ) payload = json.loads(output_path.read_text(encoding='utf-8')) @@ -213,6 +306,8 @@ def test_write_results_writes_summary_and_request_records(tmp_path: Path): assert payload['request_gap_s'] == 0.01 assert payload['max_in_flight'] == 1 assert payload['executor'] == 'single' + assert payload['invariant_profile'] == 'test-data' + assert payload['invariant_period_s'] == 5.0 assert payload['mode'] == 'read' assert payload['query_type'] == 'fetch' assert payload['query_mix'][0]['query_type'] in QUERY_TYPE_BY_NAME @@ -225,6 +320,8 @@ def test_write_results_writes_summary_and_request_records(tmp_path: Path): 'match $x isa entity; fetch $x: attribute;' ) assert payload['requests'][0]['success'] is True + assert payload['invariant_summary']['successes'] == 1 + assert payload['invariants'][0]['name'] == 'person-count' def test_default_timeout_output_path_uses_result_stem(): @@ -318,6 +415,91 @@ def test_build_query_specs_rejects_query_without_query_type(): build_query_specs(args) +def test_build_invariant_specs_uses_test_data_profile(): + """Check the built-in test-data profile.""" + specs = build_invariant_specs( + SimpleNamespace(invariant_profile='test-data') + ) + + assert [spec.name for spec in specs] == [ + 'person-count', + 'company-count', + 'employment-count', + 'boss-sentinel-count', + ] + assert all(spec.query_type == 'get_aggregate' for spec in specs) + + +def test_build_invariant_specs_uses_plan_schema_profile(): + """Check the built-in plan-schema profile.""" + specs = build_invariant_specs( + SimpleNamespace(invariant_profile='plan-schema') + ) + + assert [spec.name for spec in specs] == [ + 'plan-count', + 'action-count', + 'proposition-count', + 'has-action-count', + 'action-preconditions-count', + 'action-effects-count', + 'collect-water-sample-sentinel-count', + ] + assert all(spec.query_type == 'get_aggregate' for spec in specs) + + +def test_build_invariant_specs_returns_empty_list_for_none_profile(): + """Check invariant checks are optional.""" + specs = build_invariant_specs(SimpleNamespace(invariant_profile='none')) + + assert specs == [] + + +def test_evaluate_invariant_response_compares_aggregate_value(): + """Check invariant evaluation reads aggregate scalar values.""" + spec = InvariantSpec( + name='person-count', + query='match $p isa person; get $p; count;', + query_type='get_aggregate', + expected_value=20, + ) + + record = evaluate_invariant_response( + index=0, + phase='final', + spec=spec, + started_at_s=1.0, + ended_at_s=1.2, + response=_aggregate_count_response(20), + ) + + assert record.success is True + assert record.actual_value == 20 + + +def test_evaluate_invariant_response_reports_unexpected_value(): + """Check invariant evaluation reports mismatched aggregate values.""" + spec = InvariantSpec( + name='person-count', + query='match $p isa person; get $p; count;', + query_type='get_aggregate', + expected_value=20, + ) + + record = evaluate_invariant_response( + index=0, + phase='final', + spec=spec, + started_at_s=1.0, + ended_at_s=1.2, + response=_aggregate_count_response(0), + ) + + assert record.success is False + assert record.actual_value == 0 + assert 'expected 20' in record.error_message + + def test_run_experiment_rejects_invalid_request_count(): """Check request count validation before ROS service use.""" args = SimpleNamespace( @@ -331,6 +513,8 @@ def test_run_experiment_rejects_invalid_request_count(): executor='global', executor_threads=2, debug_events_output=None, + invariant_profile='none', + invariant_period_s=10.0, query='match $x isa entity; fetch $x: attribute;', query_type='fetch', mode='read', @@ -348,6 +532,13 @@ def test_run_experiment_rejects_invalid_client_count(): duration_s=None, timeout_s=5.0, wait_service_timeout_s=1.0, + request_gap_s=0.0, + max_in_flight=None, + executor='global', + executor_threads=2, + debug_events_output=None, + invariant_profile='none', + invariant_period_s=10.0, query='match $x isa entity; fetch $x: attribute;', query_type='fetch', mode='read', From bec3b3aa37fa5ef4b731f9350d2ae2e463f00f30 Mon Sep 17 00:00:00 2001 From: Gustavo Date: Sun, 28 Jun 2026 13:14:31 -0300 Subject: [PATCH 21/36] Refactor stress profile resolution --- .../profiles/plan_schema_invariants.json | 2 +- .../profiles/plan_schema_mixed.json | 33 +++ .../plan_schema_mixed_invariants.json | 41 ++++ .../profiles/test_data_mixed.json | 33 +++ .../ros_typedb_stress_experiment.py | 28 ++- .../ros_typedb_tools/stress_config.py | 47 +++- .../ros_typedb_tools/stress_mixed.py | 168 ++++++++++++++ .../ros_typedb_tools/stress_output.py | 8 +- .../ros_typedb_tools/stress_resolution.py | 87 +++++++ .../ros_typedb_tools/stress_runner.py | 212 +++++++++++++++--- .../test/test_ros_typedb_stress_experiment.py | 188 +++++++++++++++- 11 files changed, 800 insertions(+), 47 deletions(-) create mode 100644 ros_typedb_tools/ros_typedb_tools/profiles/plan_schema_mixed.json create mode 100644 ros_typedb_tools/ros_typedb_tools/profiles/plan_schema_mixed_invariants.json create mode 100644 ros_typedb_tools/ros_typedb_tools/profiles/test_data_mixed.json create mode 100644 ros_typedb_tools/ros_typedb_tools/stress_mixed.py create mode 100644 ros_typedb_tools/ros_typedb_tools/stress_resolution.py diff --git a/ros_typedb_tools/ros_typedb_tools/profiles/plan_schema_invariants.json b/ros_typedb_tools/ros_typedb_tools/profiles/plan_schema_invariants.json index ec56818..93ce6d6 100644 --- a/ros_typedb_tools/ros_typedb_tools/profiles/plan_schema_invariants.json +++ b/ros_typedb_tools/ros_typedb_tools/profiles/plan_schema_invariants.json @@ -29,7 +29,7 @@ "name": "action-preconditions-count", "query": "match $r isa action_preconditions; get $r; count;", "query_type": "get_aggregate", - "expected_value": 6 + "expected_value": 4 }, { "name": "action-effects-count", diff --git a/ros_typedb_tools/ros_typedb_tools/profiles/plan_schema_mixed.json b/ros_typedb_tools/ros_typedb_tools/profiles/plan_schema_mixed.json new file mode 100644 index 0000000..7a0cecd --- /dev/null +++ b/ros_typedb_tools/ros_typedb_tools/profiles/plan_schema_mixed.json @@ -0,0 +1,33 @@ +{ + "description": "Mixed read/write profile for ros_typedb_examples/data/plan_schema.tql.", + "cleanup_query": "match $action isa Action, has action_name \"{key}\"; delete $action isa Action;", + "queries": [ + { + "query": "match $x isa entity; get $x;", + "query_type": "get" + }, + { + "query": "insert $action isa Action, has action_name \"{key}\", has action_duration {cycle_float}, has result true;", + "query_type": "insert", + "tracks_cleanup": true + }, + { + "query": "match $x isa relation; get $x;", + "query_type": "get" + }, + { + "query": "match $action isa Action, has action_name \"{key}\", has action_duration $duration; delete $action has $duration; insert $action has action_duration {cycle_plus_one_float};", + "query_type": "update", + "tracks_cleanup": true + }, + { + "query": "match $x isa attribute; get $x;", + "query_type": "get" + }, + { + "query": "match $action isa Action, has action_name \"{key}\"; delete $action isa Action;", + "query_type": "delete", + "clears_cleanup": true + } + ] +} diff --git a/ros_typedb_tools/ros_typedb_tools/profiles/plan_schema_mixed_invariants.json b/ros_typedb_tools/ros_typedb_tools/profiles/plan_schema_mixed_invariants.json new file mode 100644 index 0000000..96f8cc0 --- /dev/null +++ b/ros_typedb_tools/ros_typedb_tools/profiles/plan_schema_mixed_invariants.json @@ -0,0 +1,41 @@ +{ + "description": "Mixed-mode invariants for ros_typedb_examples/data/plan_schema.tql and plan_data.tql.", + "invariants": [ + { + "name": "plan-count", + "query": "match $p isa Plan; get $p; count;", + "query_type": "get_aggregate", + "expected_value": 2 + }, + { + "name": "proposition-count", + "query": "match $p isa Proposition; get $p; count;", + "query_type": "get_aggregate", + "expected_value": 7 + }, + { + "name": "has-action-count", + "query": "match $r isa has_action; get $r; count;", + "query_type": "get_aggregate", + "expected_value": 6 + }, + { + "name": "action-preconditions-count", + "query": "match $r isa action_preconditions; get $r; count;", + "query_type": "get_aggregate", + "expected_value": 4 + }, + { + "name": "action-effects-count", + "query": "match $r isa action_effects; get $r; count;", + "query_type": "get_aggregate", + "expected_value": 4 + }, + { + "name": "collect-water-sample-sentinel-count", + "query": "match $a isa Action, has action_name \"collect-water-sample\"; get $a; count;", + "query_type": "get_aggregate", + "expected_value": 1 + } + ] +} diff --git a/ros_typedb_tools/ros_typedb_tools/profiles/test_data_mixed.json b/ros_typedb_tools/ros_typedb_tools/profiles/test_data_mixed.json new file mode 100644 index 0000000..fb194d9 --- /dev/null +++ b/ros_typedb_tools/ros_typedb_tools/profiles/test_data_mixed.json @@ -0,0 +1,33 @@ +{ + "description": "Mixed read/write profile for ros_typedb/test/typedb_test_data/schema.tql.", + "cleanup_query": "match $robot isa robot, has full-name \"{key}\"; delete $robot isa robot;", + "queries": [ + { + "query": "match $x isa entity; get $x;", + "query_type": "get" + }, + { + "query": "insert $robot isa robot, has full-name \"{key}\", has robot-type \"stress\", has age {cycle}, has alive true;", + "query_type": "insert", + "tracks_cleanup": true + }, + { + "query": "match $x isa relation; get $x;", + "query_type": "get" + }, + { + "query": "match $robot isa robot, has full-name \"{key}\", has age $age; delete $robot has $age; insert $robot has age {cycle_plus_one};", + "query_type": "update", + "tracks_cleanup": true + }, + { + "query": "match $x isa attribute; get $x;", + "query_type": "get" + }, + { + "query": "match $robot isa robot, has full-name \"{key}\"; delete $robot isa robot;", + "query_type": "delete", + "clears_cleanup": true + } + ] +} diff --git a/ros_typedb_tools/ros_typedb_tools/ros_typedb_stress_experiment.py b/ros_typedb_tools/ros_typedb_tools/ros_typedb_stress_experiment.py index 75a19bf..f7d8b32 100644 --- a/ros_typedb_tools/ros_typedb_tools/ros_typedb_stress_experiment.py +++ b/ros_typedb_tools/ros_typedb_tools/ros_typedb_stress_experiment.py @@ -16,15 +16,15 @@ from __future__ import annotations import argparse -import sys from pathlib import Path +import sys import rclpy -from ros_typedb_tools.stress_config import QUERY_TYPE_BY_NAME from ros_typedb_tools.stress_config import INVARIANT_PROFILE_NAMES +from ros_typedb_tools.stress_config import MIXED_PROFILE_NAMES +from ros_typedb_tools.stress_config import QUERY_TYPE_BY_NAME from ros_typedb_tools.stress_config import StressExperimentResult -from ros_typedb_tools.stress_config import build_query_specs from ros_typedb_tools.stress_output import default_timeout_output_path from ros_typedb_tools.stress_output import print_summary from ros_typedb_tools.stress_output import summarize_invariant_records @@ -110,10 +110,20 @@ def build_argument_parser() -> argparse.ArgumentParser: ) parser.add_argument( '--mode', - choices=('read',), + choices=('read', 'mixed'), default='read', help='Built-in query mix to use when --query is omitted.', ) + parser.add_argument( + '--mixed-profile', + choices=MIXED_PROFILE_NAMES, + default='auto', + help=( + 'Schema-specific mixed-mode profile. auto selects plan-schema ' + 'when --invariant-profile is plan-schema or plan-schema-mixed; ' + 'otherwise it selects test-data.' + ), + ) parser.add_argument( '--timeout-s', type=float, @@ -179,7 +189,6 @@ def _write_requested_outputs( ) -> None: output_path = None if args.output: - query_specs = build_query_specs(args) output_path = Path(args.output).expanduser() write_results( output_path, @@ -191,11 +200,14 @@ def _write_requested_outputs( clients=args.clients, duration_s=args.duration_s, mode=args.mode, - query_mix=query_specs, + mixed_profile=( + result.config.mixed_profile_name or args.mixed_profile + ), + query_mix=result.config.query_specs, request_gap_s=args.request_gap_s, - max_in_flight=args.max_in_flight, + max_in_flight=result.config.max_in_flight, executor=args.executor, - invariant_profile=args.invariant_profile, + invariant_profile=result.config.invariant_profile_name, invariant_period_s=args.invariant_period_s, invariant_records=result.invariant_records, ) diff --git a/ros_typedb_tools/ros_typedb_tools/stress_config.py b/ros_typedb_tools/ros_typedb_tools/stress_config.py index 4cfabdd..7033f51 100644 --- a/ros_typedb_tools/ros_typedb_tools/stress_config.py +++ b/ros_typedb_tools/ros_typedb_tools/stress_config.py @@ -16,13 +16,17 @@ from __future__ import annotations import argparse -import json from dataclasses import dataclass +import json from pathlib import Path from typing import Any +from typing import TYPE_CHECKING from ros_typedb_msgs.srv import Query +if TYPE_CHECKING: + from ros_typedb_tools.stress_mixed import MixedQueryProfile + QUERY_TYPE_BY_NAME = { 'insert': Query.Request.INSERT, @@ -50,12 +54,19 @@ ) INVARIANT_PROFILE_FILES = { + 'plan-schema-mixed': 'plan_schema_mixed_invariants.json', 'plan-schema': 'plan_schema_invariants.json', 'test-data': 'test_data_invariants.json', } INVARIANT_PROFILE_NAMES = ('none', *INVARIANT_PROFILE_FILES) INVARIANT_PROFILE_DIR = Path(__file__).with_name('profiles') +MIXED_PROFILE_FILES = { + 'plan-schema': 'plan_schema_mixed.json', + 'test-data': 'test_data_mixed.json', +} +MIXED_PROFILE_NAMES = ('auto', *MIXED_PROFILE_FILES) + @dataclass(frozen=True) class QuerySpec: @@ -63,6 +74,7 @@ class QuerySpec: query: str query_type: str + cleanup_key: str | None = None @dataclass(frozen=True) @@ -113,12 +125,32 @@ class InvariantRecord: actual_value: Any +@dataclass(frozen=True) +class ResolvedStressConfig: + """Concrete runtime configuration derived from CLI arguments. + + The CLI exposes convenient selectors such as ``--mixed-profile auto``. + The runner should not need to interpret those selectors while it is also + managing ROS futures. This object stores the selected query mix, invariant + checks, and mixed profile after all CLI-level choices have been resolved. + """ + + query_specs: list[QuerySpec] + invariant_specs: list[InvariantSpec] + mixed_profile: MixedQueryProfile | None + mixed_profile_name: str | None + invariant_profile_name: str + max_in_flight: int + debug_events_output: Path | None + + @dataclass(frozen=True) class StressExperimentResult: """Complete stress experiment result.""" records: list[RequestRecord] invariant_records: list[InvariantRecord] + config: ResolvedStressConfig def validate_experiment_args(args: argparse.Namespace) -> None: @@ -149,10 +181,18 @@ def validate_experiment_args(args: argparse.Namespace) -> None: raise ValueError( '--invariant-period-s must be greater than or equal to zero' ) + mixed_profile = getattr(args, 'mixed_profile', 'auto') + if mixed_profile not in MIXED_PROFILE_NAMES: + raise ValueError(f'unsupported mixed profile: {mixed_profile}') def build_query_specs(args: argparse.Namespace) -> list[QuerySpec]: - """Build the query sequence used by the stress experiment.""" + """Build fixed query specs for explicit-query or read-only modes. + + Mixed mode uses schema-specific templates and is resolved in + ``stress_resolution``. Keeping this helper fixed-query-only avoids hidden + profile loading from a generic config function. + """ if args.query and not args.query_type: raise ValueError('--query-type is required when --query is used') if args.query_type and not args.query: @@ -167,6 +207,9 @@ def build_query_specs(args: argparse.Namespace) -> list[QuerySpec]: for query, query_type in DEFAULT_READ_QUERY_SPECS ] + if args.mode == 'mixed': + raise ValueError('mixed mode requires a resolved mixed profile') + raise ValueError(f'unsupported mode: {args.mode}') diff --git a/ros_typedb_tools/ros_typedb_tools/stress_mixed.py b/ros_typedb_tools/ros_typedb_tools/stress_mixed.py new file mode 100644 index 0000000..a0ec6de --- /dev/null +++ b/ros_typedb_tools/ros_typedb_tools/stress_mixed.py @@ -0,0 +1,168 @@ +# Copyright 2026 Gustavo Rezende Silva +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +"""Mixed read/write profile loading for stress experiments.""" + +from __future__ import annotations + +from dataclasses import dataclass +import json +from pathlib import Path +from typing import Any + +from ros_typedb_tools.stress_config import MIXED_PROFILE_FILES +from ros_typedb_tools.stress_config import QUERY_TYPE_BY_NAME +from ros_typedb_tools.stress_config import QuerySpec + + +MIXED_PROFILE_DIR = Path(__file__).with_name('profiles') + + +@dataclass(frozen=True) +class MixedQueryTemplate: + """One query template from a mixed read/write profile.""" + + query: str + query_type: str + tracks_cleanup: bool = False + clears_cleanup: bool = False + + +@dataclass(frozen=True) +class MixedQueryProfile: + """Loaded mixed read/write profile.""" + + name: str + cleanup_query: str + templates: list[MixedQueryTemplate] + + +def resolve_mixed_profile_name(args: Any) -> str: + """Resolve the selected mixed profile name.""" + mixed_profile = getattr(args, 'mixed_profile', 'auto') + if mixed_profile != 'auto': + return mixed_profile + + invariant_profile = getattr(args, 'invariant_profile', 'none') + if invariant_profile == 'plan-schema-mixed': + return 'plan-schema' + if invariant_profile in MIXED_PROFILE_FILES: + return invariant_profile + return 'test-data' + + +def load_mixed_query_profile(args: Any) -> MixedQueryProfile: + """Load the mixed read/write profile selected by CLI arguments.""" + profile_name = resolve_mixed_profile_name(args) + profile_filename = MIXED_PROFILE_FILES.get(profile_name) + if profile_filename is None: + raise ValueError(f'unsupported mixed profile: {profile_name}') + return _load_mixed_query_profile( + profile_name, + MIXED_PROFILE_DIR / profile_filename, + ) + + +def _load_mixed_query_profile( + profile_name: str, + profile_path: Path, +) -> MixedQueryProfile: + """Load one packaged mixed read/write profile.""" + payload = json.loads(profile_path.read_text(encoding='utf-8')) + templates = [] + for query in payload.get('queries', []): + query_type = query['query_type'] + if query_type not in QUERY_TYPE_BY_NAME: + raise ValueError(f'unsupported mixed query type: {query_type}') + templates.append( + MixedQueryTemplate( + query=query['query'], + query_type=query_type, + tracks_cleanup=query.get('tracks_cleanup', False), + clears_cleanup=query.get('clears_cleanup', False), + ) + ) + if not templates: + raise ValueError(f'mixed profile has no queries: {profile_name}') + return MixedQueryProfile( + name=profile_name, + cleanup_query=payload['cleanup_query'], + templates=templates, + ) + + +def make_mixed_key( + *, + run_id: str, + client_id: int, + cycle: int, +) -> str: + """Build the unique key for one experiment-owned temporary entity.""" + return f'ros-typedb-stress-{run_id}-c{client_id}-n{cycle}' + + +def build_mixed_query_spec( + *, + profile: MixedQueryProfile, + run_id: str, + client_id: int, + client_request_index: int, +) -> QuerySpec: + """ + Build one mixed-mode query for a client's local request sequence. + + The profile decides which schema-specific entity is written. The generated + key is unique to this experiment run, so mixed mode does not touch baseline + fixture data. + """ + operation_index = client_request_index % len(profile.templates) + cycle = client_request_index // len(profile.templates) + template = profile.templates[operation_index] + key = make_mixed_key( + run_id=run_id, + client_id=client_id, + cycle=cycle, + ) + cleanup_key = ( + key + if template.tracks_cleanup or template.clears_cleanup + else None + ) + return QuerySpec( + query=_render_mixed_template(template.query, key=key, cycle=cycle), + query_type=template.query_type, + cleanup_key=cleanup_key, + ) + + +def build_mixed_cleanup_query_spec( + profile: MixedQueryProfile, + key: str, +) -> QuerySpec: + """Build a delete query for one experiment-owned temporary key.""" + return QuerySpec( + query=_render_mixed_template(profile.cleanup_query, key=key, cycle=0), + query_type='delete', + cleanup_key=key, + ) + + +def _render_mixed_template(template: str, *, key: str, cycle: int) -> str: + """Substitute the supported mixed-profile placeholders.""" + return template.format( + key=key, + cycle=cycle, + cycle_float=f'{float(cycle):.1f}', + cycle_plus_one=cycle + 1, + cycle_plus_one_float=f'{float(cycle + 1):.1f}', + ) diff --git a/ros_typedb_tools/ros_typedb_tools/stress_output.py b/ros_typedb_tools/ros_typedb_tools/stress_output.py index 3e9bcb9..5794a67 100644 --- a/ros_typedb_tools/ros_typedb_tools/stress_output.py +++ b/ros_typedb_tools/ros_typedb_tools/stress_output.py @@ -15,12 +15,12 @@ from __future__ import annotations -import json -import statistics -import time from collections.abc import Sequence from dataclasses import asdict +import json from pathlib import Path +import statistics +import time from typing import Any from ros_typedb_tools.stress_config import InvariantRecord @@ -99,6 +99,7 @@ def write_results( clients: int = 1, duration_s: float | None = None, mode: str = 'read', + mixed_profile: str = 'auto', query_mix: Sequence[QuerySpec] | None = None, request_gap_s: float = 0.0, max_in_flight: int | None = None, @@ -112,6 +113,7 @@ def write_results( payload = { 'service_name': service_name, 'mode': mode, + 'mixed_profile': mixed_profile, 'clients': clients, 'duration_s': duration_s, 'request_gap_s': request_gap_s, diff --git a/ros_typedb_tools/ros_typedb_tools/stress_resolution.py b/ros_typedb_tools/ros_typedb_tools/stress_resolution.py new file mode 100644 index 0000000..518917c --- /dev/null +++ b/ros_typedb_tools/ros_typedb_tools/stress_resolution.py @@ -0,0 +1,87 @@ +# Copyright 2026 Gustavo Rezende Silva +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +"""Resolve raw stress CLI arguments into concrete runtime configuration.""" + +from __future__ import annotations + +import argparse +from pathlib import Path + +from ros_typedb_tools.stress_config import build_invariant_specs +from ros_typedb_tools.stress_config import build_query_specs +from ros_typedb_tools.stress_config import QuerySpec +from ros_typedb_tools.stress_config import ResolvedStressConfig +from ros_typedb_tools.stress_config import validate_experiment_args +from ros_typedb_tools.stress_mixed import load_mixed_query_profile +from ros_typedb_tools.stress_mixed import MixedQueryProfile + + +def resolve_stress_config( + args: argparse.Namespace, +) -> ResolvedStressConfig: + """Build the concrete configuration used by the runner and output writer. + + This is the only place that combines the raw CLI flags with packaged + profiles. The runner receives ordinary query specs and an optional loaded + mixed profile; it does not need to know how ``auto`` profile selection + works. + """ + validate_experiment_args(args) + + invariant_specs = build_invariant_specs(args) + mixed_profile = _resolve_dynamic_mixed_profile(args) + query_specs = _build_runtime_query_specs(args, mixed_profile) + debug_events_output = ( + Path(args.debug_events_output).expanduser() + if args.debug_events_output + else None + ) + + return ResolvedStressConfig( + query_specs=query_specs, + invariant_specs=invariant_specs, + mixed_profile=mixed_profile, + mixed_profile_name=( + mixed_profile.name if mixed_profile is not None else None + ), + invariant_profile_name=args.invariant_profile, + max_in_flight=args.max_in_flight or args.clients, + debug_events_output=debug_events_output, + ) + + +def _resolve_dynamic_mixed_profile( + args: argparse.Namespace, +) -> MixedQueryProfile | None: + """Load a mixed profile only when the workload is profile-driven.""" + if args.mode == 'mixed' and args.query is None: + return load_mixed_query_profile(args) + return None + + +def _build_runtime_query_specs( + args: argparse.Namespace, + mixed_profile: MixedQueryProfile | None, +) -> list[QuerySpec]: + """Build fixed query specs or metadata specs for dynamic mixed mode.""" + if mixed_profile is None: + return build_query_specs(args) + + # Dynamic mixed requests are rendered per client and request index at + # runtime. These QuerySpecs preserve the selected profile's query mix for + # JSON output and debug readability. + return [ + QuerySpec(query=template.query, query_type=template.query_type) + for template in mixed_profile.templates + ] diff --git a/ros_typedb_tools/ros_typedb_tools/stress_runner.py b/ros_typedb_tools/ros_typedb_tools/stress_runner.py index a8b2203..78a3153 100644 --- a/ros_typedb_tools/ros_typedb_tools/stress_runner.py +++ b/ros_typedb_tools/ros_typedb_tools/stress_runner.py @@ -16,10 +16,10 @@ from __future__ import annotations import argparse -import time from dataclasses import dataclass -from pathlib import Path +import time from typing import Any +import uuid import rclpy from rclpy.executors import MultiThreadedExecutor @@ -27,18 +27,19 @@ from ros_typedb_msgs.srv import Query +from ros_typedb_tools.stress_config import build_query_request from ros_typedb_tools.stress_config import InvariantRecord from ros_typedb_tools.stress_config import InvariantSpec from ros_typedb_tools.stress_config import QuerySpec from ros_typedb_tools.stress_config import RequestRecord from ros_typedb_tools.stress_config import StressExperimentResult -from ros_typedb_tools.stress_config import build_invariant_specs -from ros_typedb_tools.stress_config import build_query_request -from ros_typedb_tools.stress_config import build_query_specs -from ros_typedb_tools.stress_config import validate_experiment_args from ros_typedb_tools.stress_invariants import build_timeout_invariant_record from ros_typedb_tools.stress_invariants import evaluate_invariant_response +from ros_typedb_tools.stress_mixed import build_mixed_cleanup_query_spec +from ros_typedb_tools.stress_mixed import build_mixed_query_spec +from ros_typedb_tools.stress_mixed import MixedQueryProfile from ros_typedb_tools.stress_output import DebugEventWriter +from ros_typedb_tools.stress_resolution import resolve_stress_config @dataclass @@ -57,6 +58,10 @@ class _RunState: invariant_records: list[InvariantRecord] pending_by_client: dict[int, _PendingRequest] last_started_by_client: list[float] + sent_count_by_client: list[int] + mixed_cleanup_keys: set[str] + mixed_profile: MixedQueryProfile | None + mixed_run_id: str next_index: int next_invariant_index: int deadline_s: float | None @@ -148,9 +153,25 @@ def _start_request( state: _RunState, timeout_s: float, started_at_s: float, + use_mixed_queries: bool, + mixed_profile: MixedQueryProfile | None, + mixed_run_id: str, ) -> _PendingRequest: - query_index = state.next_index % len(query_specs) - query_spec = query_specs[query_index] + if use_mixed_queries: + if mixed_profile is None: + raise RuntimeError('mixed profile is required for mixed mode') + query_index = state.sent_count_by_client[client_id] % len( + mixed_profile.templates + ) + query_spec = build_mixed_query_spec( + profile=mixed_profile, + run_id=mixed_run_id, + client_id=client_id, + client_request_index=state.sent_count_by_client[client_id], + ) + else: + query_index = state.next_index % len(query_specs) + query_spec = query_specs[query_index] request = build_query_request( query_spec.query, query_spec.query_type, @@ -167,6 +188,14 @@ def _start_request( ) state.pending_by_client[client_id] = pending state.last_started_by_client[client_id] = started_at_s + state.sent_count_by_client[client_id] += 1 + # Mixed workloads create experiment-owned temporary concepts. Track keys + # until a matching delete succeeds, then final cleanup can remove leftovers. + if ( + query_spec.cleanup_key is not None + and query_spec.query_type != 'delete' + ): + state.mixed_cleanup_keys.add(query_spec.cleanup_key) state.next_index += 1 return pending @@ -201,6 +230,9 @@ def _send_available_requests( state=state, timeout_s=args.timeout_s, started_at_s=now_s, + use_mixed_queries=args.mode == 'mixed' and args.query is None, + mixed_profile=state.mixed_profile, + mixed_run_id=state.mixed_run_id, ) debug_events.log( 'request_sent', @@ -300,6 +332,12 @@ def _record_done_future( return response = pending.future.result() + if ( + response.success + and pending.query_spec.cleanup_key is not None + and pending.query_spec.query_type == 'delete' + ): + state.mixed_cleanup_keys.discard(pending.query_spec.cleanup_key) state.records.append( _record_from_pending( pending, @@ -471,6 +509,111 @@ def _run_invariant_profile( state.next_invariant_index += 1 +def _run_cleanup_request( + *, + node: Any, + executor: Any | None, + client: Any, + state: _RunState, + query_spec: QuerySpec, + timeout_s: float, + debug_events: DebugEventWriter, +) -> None: + started_at_s = time.monotonic() + request = build_query_request( + query_spec.query, + query_spec.query_type, + timeout_s, + ) + future = client.call_async(request) + pending = _PendingRequest( + client_id=-1, + index=state.next_index, + query_index=-1, + query_spec=query_spec, + future=future, + started_at_s=started_at_s, + ) + state.next_index += 1 + debug_events.log( + 'cleanup_sent', + index=pending.index, + cleanup_key=query_spec.cleanup_key, + ) + + while True: + _spin_once(node, executor) + now_s = time.monotonic() + if future.done(): + _record_done_future( + pending=pending, + now_s=now_s, + pending_count=1, + state=state, + debug_events=debug_events, + ) + debug_events.log( + 'cleanup_complete', + index=pending.index, + cleanup_key=query_spec.cleanup_key, + ) + return + + if now_s - started_at_s >= timeout_s: + _record_timeout( + pending=pending, + now_s=now_s, + pending_count=1, + state=state, + debug_events=debug_events, + ) + debug_events.log( + 'cleanup_timeout', + index=pending.index, + cleanup_key=query_spec.cleanup_key, + ) + return + + +def _cleanup_mixed_data( + *, + args: argparse.Namespace, + node: Any, + executor: Any | None, + cleanup_client: Any | None, + mixed_profile: MixedQueryProfile | None, + state: _RunState, + debug_events: DebugEventWriter, +) -> None: + if args.mode != 'mixed' or args.query is not None: + return + if ( + cleanup_client is None + or mixed_profile is None + or not state.mixed_cleanup_keys + ): + return + + debug_events.log( + 'cleanup_started', + cleanup_keys=len(state.mixed_cleanup_keys), + ) + for key in sorted(state.mixed_cleanup_keys): + _run_cleanup_request( + node=node, + executor=executor, + client=cleanup_client, + state=state, + query_spec=build_mixed_cleanup_query_spec(mixed_profile, key), + timeout_s=args.timeout_s, + debug_events=debug_events, + ) + debug_events.log( + 'cleanup_finished', + cleanup_keys=len(state.mixed_cleanup_keys), + ) + + def _maybe_run_periodic_invariants( *, args: argparse.Namespace, @@ -504,15 +647,7 @@ def _maybe_run_periodic_invariants( def run_experiment(args: argparse.Namespace) -> StressExperimentResult: """Run the stress experiment.""" - validate_experiment_args(args) - query_specs = build_query_specs(args) - invariant_specs = build_invariant_specs(args) - max_in_flight = args.max_in_flight or args.clients - debug_events_output = ( - Path(args.debug_events_output).expanduser() - if args.debug_events_output - else None - ) + config = resolve_stress_config(args) node = rclpy.create_node('ros_typedb_stress_experiment') executor = _build_executor(args) @@ -524,7 +659,12 @@ def run_experiment(args: argparse.Namespace) -> StressExperimentResult: ] invariant_client = ( node.create_client(Query, args.service_name) - if invariant_specs + if config.invariant_specs + else None + ) + cleanup_client = ( + node.create_client(Query, args.service_name) + if config.mixed_profile is not None else None ) deadline_s = ( @@ -537,6 +677,10 @@ def run_experiment(args: argparse.Namespace) -> StressExperimentResult: invariant_records=[], pending_by_client={}, last_started_by_client=[float('-inf') for _ in range(args.clients)], + sent_count_by_client=[0 for _ in range(args.clients)], + mixed_cleanup_keys=set(), + mixed_profile=config.mixed_profile, + mixed_run_id=uuid.uuid4().hex[:12], next_index=0, next_invariant_index=0, deadline_s=deadline_s, @@ -545,18 +689,21 @@ def run_experiment(args: argparse.Namespace) -> StressExperimentResult: ) try: - with DebugEventWriter(debug_events_output) as debug_events: + with DebugEventWriter(config.debug_events_output) as debug_events: debug_events.log( 'experiment_started', clients=args.clients, executor=args.executor, - max_in_flight=max_in_flight, + max_in_flight=config.max_in_flight, request_gap_s=args.request_gap_s, - invariant_profile=args.invariant_profile, + invariant_profile=config.invariant_profile_name, invariant_period_s=args.invariant_period_s, + mixed_profile=config.mixed_profile_name, ) _wait_for_all_clients( - clients + ([invariant_client] if invariant_client else []), + clients + + ([invariant_client] if invariant_client else []) + + ([cleanup_client] if cleanup_client else []), service_name=args.service_name, wait_service_timeout_s=args.wait_service_timeout_s, debug_events=debug_events, @@ -592,9 +739,9 @@ def run_experiment(args: argparse.Namespace) -> StressExperimentResult: _send_available_requests( args=args, clients=clients, - query_specs=query_specs, + query_specs=config.query_specs, state=state, - max_in_flight=max_in_flight, + max_in_flight=config.max_in_flight, debug_events=debug_events, ) _maybe_run_periodic_invariants( @@ -602,18 +749,28 @@ def run_experiment(args: argparse.Namespace) -> StressExperimentResult: node=node, executor=executor, invariant_client=invariant_client, - invariant_specs=invariant_specs, + invariant_specs=config.invariant_specs, state=state, now_s=now_s, debug_events=debug_events, ) - if invariant_specs and invariant_client is not None: + _cleanup_mixed_data( + args=args, + node=node, + executor=executor, + cleanup_client=cleanup_client, + mixed_profile=config.mixed_profile, + state=state, + debug_events=debug_events, + ) + + if config.invariant_specs and invariant_client is not None: _run_invariant_profile( node=node, executor=executor, client=invariant_client, - specs=invariant_specs, + specs=config.invariant_specs, state=state, phase='final', timeout_s=args.timeout_s, @@ -641,4 +798,5 @@ def run_experiment(args: argparse.Namespace) -> StressExperimentResult: return StressExperimentResult( records=state.records, invariant_records=state.invariant_records, + config=config, ) diff --git a/ros_typedb_tools/test/test_ros_typedb_stress_experiment.py b/ros_typedb_tools/test/test_ros_typedb_stress_experiment.py index 0a2e2be..0cb8761 100644 --- a/ros_typedb_tools/test/test_ros_typedb_stress_experiment.py +++ b/ros_typedb_tools/test/test_ros_typedb_stress_experiment.py @@ -8,6 +8,9 @@ from rcl_interfaces.msg import ParameterType from rcl_interfaces.msg import ParameterValue +from ros_typedb_msgs.msg import Attribute +from ros_typedb_msgs.msg import QueryResult +from ros_typedb_msgs.msg import ResultTree from ros_typedb_tools.fake_query_service import ( build_fake_service_argument_parser, ) @@ -15,14 +18,18 @@ build_argument_parser, ) from ros_typedb_tools.stress_config import ( - QUERY_TYPE_BY_NAME, + build_invariant_specs, + build_query_specs, InvariantRecord, InvariantSpec, + QUERY_TYPE_BY_NAME, RequestRecord, - build_invariant_specs, - build_query_specs, ) from ros_typedb_tools.stress_invariants import evaluate_invariant_response +from ros_typedb_tools.stress_mixed import build_mixed_cleanup_query_spec +from ros_typedb_tools.stress_mixed import build_mixed_query_spec +from ros_typedb_tools.stress_mixed import load_mixed_query_profile +from ros_typedb_tools.stress_mixed import make_mixed_key from ros_typedb_tools.stress_output import ( DebugEventWriter, default_timeout_output_path, @@ -31,10 +38,8 @@ write_results, write_timeout_results, ) +from ros_typedb_tools.stress_resolution import resolve_stress_config from ros_typedb_tools.stress_runner import run_experiment -from ros_typedb_msgs.msg import Attribute -from ros_typedb_msgs.msg import QueryResult -from ros_typedb_msgs.msg import ResultTree def _record( @@ -198,6 +203,32 @@ def test_parser_accepts_stage_three_arguments(): assert args.invariant_period_s == 5 +def test_parser_accepts_stage_four_mixed_mode(): + """Check Stage 4 mixed read/write mode arguments.""" + parser = build_argument_parser() + + args = parser.parse_args( + [ + '--clients', + '10', + '--duration-s', + '120', + '--mode', + 'mixed', + '--mixed-profile', + 'plan-schema', + '--invariant-profile', + 'plan-schema-mixed', + ] + ) + + assert args.clients == 10 + assert args.duration_s == 120 + assert args.mode == 'mixed' + assert args.mixed_profile == 'plan-schema' + assert args.invariant_profile == 'plan-schema-mixed' + + def test_fake_service_parser_accepts_diagnostic_arguments(): """Check the fake service CLI options.""" parser = build_fake_service_argument_parser() @@ -288,6 +319,7 @@ def test_write_results_writes_summary_and_request_records(tmp_path: Path): clients=2, duration_s=10.0, mode='read', + mixed_profile='auto', query_mix=build_query_specs( SimpleNamespace(query=None, query_type=None, mode='read') ), @@ -309,6 +341,7 @@ def test_write_results_writes_summary_and_request_records(tmp_path: Path): assert payload['invariant_profile'] == 'test-data' assert payload['invariant_period_s'] == 5.0 assert payload['mode'] == 'read' + assert payload['mixed_profile'] == 'auto' assert payload['query_type'] == 'fetch' assert payload['query_mix'][0]['query_type'] in QUERY_TYPE_BY_NAME assert payload['summary']['successes'] == 1 @@ -403,6 +436,124 @@ def test_build_query_specs_uses_read_query_mix_by_default(): assert all(spec.query_type in QUERY_TYPE_BY_NAME for spec in specs) +def test_resolve_stress_config_uses_mixed_query_mix(): + """Check resolved mixed mode includes read and write query types.""" + config = resolve_stress_config( + SimpleNamespace( + requests=1, + clients=1, + duration_s=None, + timeout_s=5.0, + wait_service_timeout_s=1.0, + request_gap_s=0.0, + max_in_flight=None, + executor='global', + executor_threads=2, + debug_events_output=None, + invariant_profile='none', + invariant_period_s=10.0, + query=None, + query_type=None, + mode='mixed', + mixed_profile='test-data', + ) + ) + + query_types = {spec.query_type for spec in config.query_specs} + assert {'delete', 'get', 'insert', 'update'} <= query_types + assert config.mixed_profile_name == 'test-data' + assert config.max_in_flight == 1 + + +def test_build_mixed_query_spec_cycles_temporary_robot_operations(): + """Check mixed mode generates insert, update, and delete robot queries.""" + profile = load_mixed_query_profile( + SimpleNamespace(mixed_profile='test-data', invariant_profile='none') + ) + read_spec = build_mixed_query_spec( + profile=profile, + run_id='abc123', + client_id=2, + client_request_index=0, + ) + insert_spec = build_mixed_query_spec( + profile=profile, + run_id='abc123', + client_id=2, + client_request_index=1, + ) + update_spec = build_mixed_query_spec( + profile=profile, + run_id='abc123', + client_id=2, + client_request_index=3, + ) + delete_spec = build_mixed_query_spec( + profile=profile, + run_id='abc123', + client_id=2, + client_request_index=5, + ) + + key = make_mixed_key(run_id='abc123', client_id=2, cycle=0) + assert read_spec.query_type == 'get' + assert read_spec.cleanup_key is None + assert insert_spec.query_type == 'insert' + assert f'has full-name "{key}"' in insert_spec.query + assert 'isa robot' in insert_spec.query + assert insert_spec.cleanup_key == key + assert update_spec.query_type == 'update' + assert f'has full-name "{key}"' in update_spec.query + assert 'delete $robot has $age;' in update_spec.query + assert update_spec.cleanup_key == key + assert delete_spec.query_type == 'delete' + assert f'has full-name "{key}"' in delete_spec.query + assert delete_spec.cleanup_key == key + + +def test_build_mixed_cleanup_query_spec_deletes_exact_temp_key(): + """Check mixed cleanup targets only one generated robot key.""" + profile = load_mixed_query_profile( + SimpleNamespace(mixed_profile='test-data', invariant_profile='none') + ) + spec = build_mixed_cleanup_query_spec( + profile, + 'ros-typedb-stress-run-c0-n0', + ) + + assert spec.query_type == 'delete' + assert spec.cleanup_key == 'ros-typedb-stress-run-c0-n0' + assert 'isa robot' in spec.query + assert 'has full-name "ros-typedb-stress-run-c0-n0"' in spec.query + + +def test_build_mixed_query_spec_supports_plan_schema_profile(): + """Check plan-schema mixed mode writes temporary actions.""" + profile = load_mixed_query_profile( + SimpleNamespace(mixed_profile='plan-schema', invariant_profile='none') + ) + + insert_spec = build_mixed_query_spec( + profile=profile, + run_id='abc123', + client_id=1, + client_request_index=1, + ) + cleanup_spec = build_mixed_cleanup_query_spec( + profile, + insert_spec.cleanup_key, + ) + + assert insert_spec.query_type == 'insert' + assert 'isa Action' in insert_spec.query + assert 'has action_name "ros-typedb-stress-abc123-c1-n0"' in ( + insert_spec.query + ) + assert 'has action_duration 0.0' in insert_spec.query + assert cleanup_spec.query_type == 'delete' + assert 'isa Action' in cleanup_spec.query + + def test_build_query_specs_rejects_query_without_query_type(): """Check explicit queries still require an explicit query type.""" args = SimpleNamespace( @@ -446,6 +597,31 @@ def test_build_invariant_specs_uses_plan_schema_profile(): 'collect-water-sample-sentinel-count', ] assert all(spec.query_type == 'get_aggregate' for spec in specs) + assert { + spec.name: spec.expected_value + for spec in specs + }['action-preconditions-count'] == 4 + + +def test_build_invariant_specs_uses_plan_schema_mixed_profile(): + """Check the mixed-safe plan-schema invariant profile.""" + specs = build_invariant_specs( + SimpleNamespace(invariant_profile='plan-schema-mixed') + ) + + assert [spec.name for spec in specs] == [ + 'plan-count', + 'proposition-count', + 'has-action-count', + 'action-preconditions-count', + 'action-effects-count', + 'collect-water-sample-sentinel-count', + ] + assert all(spec.query_type == 'get_aggregate' for spec in specs) + assert { + spec.name: spec.expected_value + for spec in specs + }['action-preconditions-count'] == 4 def test_build_invariant_specs_returns_empty_list_for_none_profile(): From 5c38a7ba6b3e1449b68cd68bbedaf8a5b56d7d29 Mon Sep 17 00:00:00 2001 From: Gustavo Date: Sun, 28 Jun 2026 13:35:47 -0300 Subject: [PATCH 22/36] update .gitignore and dockerignore --- .dockerignore | 3 +++ .gitignore | 3 +++ 2 files changed, 6 insertions(+) diff --git a/.dockerignore b/.dockerignore index 06c423b..dc75620 100644 --- a/.dockerignore +++ b/.dockerignore @@ -1,3 +1,6 @@ +# folder for keep annotations to self +docs-self/ + .vscode docs/source/_autosummary/ docs/source/generated/ diff --git a/.gitignore b/.gitignore index 8b267c5..49041e9 100644 --- a/.gitignore +++ b/.gitignore @@ -1,3 +1,6 @@ +# folder for keep annotations to self +docs-self/ + .vscode docs/source/_autosummary/ docs/source/generated/ From 7eaf94705685fbb4dd5c0f012d807591c4be0dc2 Mon Sep 17 00:00:00 2001 From: Gustavo Date: Sun, 28 Jun 2026 15:04:09 -0300 Subject: [PATCH 23/36] add ros_typedb_examples/launch/test_data_example.launch.py --- .../launch/test_data_example.launch.py | 152 ++++++++++++++++++ 1 file changed, 152 insertions(+) create mode 100644 ros_typedb_examples/launch/test_data_example.launch.py diff --git a/ros_typedb_examples/launch/test_data_example.launch.py b/ros_typedb_examples/launch/test_data_example.launch.py new file mode 100644 index 0000000..7489a74 --- /dev/null +++ b/ros_typedb_examples/launch/test_data_example.launch.py @@ -0,0 +1,152 @@ +"""Launch ros_typedb with the bundled test schema and data.""" + +# Copyright 2026 Gustavo Rezende Silva +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +import os + +from ament_index_python.packages import get_package_share_directory + +from launch import LaunchDescription +from launch.actions import DeclareLaunchArgument +from launch.actions import EmitEvent +from launch.actions import RegisterEventHandler +from launch.events import matches_action +from launch.substitutions import LaunchConfiguration + +from launch_ros.actions import LifecycleNode +from launch_ros.event_handlers import OnStateTransition +from launch_ros.events.lifecycle import ChangeState + +import lifecycle_msgs + + +def generate_launch_description(): + """Launch ros_typedb with the bundled test schema and data.""" + package_share = get_package_share_directory('ros_typedb_examples') + default_schema_path = os.path.join( + package_share, + 'data', + 'typedb_test_data', + 'schema.tql', + ) + default_data_path = os.path.join( + package_share, + 'data', + 'typedb_test_data', + 'data.tql', + ) + + database_name = LaunchConfiguration('database_name') + address = LaunchConfiguration('address') + force_data = LaunchConfiguration('force_data') + force_database = LaunchConfiguration('force_database') + reload_schema = LaunchConfiguration('reload_schema') + infer = LaunchConfiguration('infer') + query_timeout_s = LaunchConfiguration('query_timeout_s') + + database_name_arg = DeclareLaunchArgument( + 'database_name', + default_value='ros_typedb_test_data_example', + description='database name', + ) + + address_arg = DeclareLaunchArgument( + 'address', + default_value='localhost:1729', + description='TypeDB server address', + ) + + force_data_arg = DeclareLaunchArgument( + 'force_data', + default_value='True', + description='reload test data on startup', + ) + + force_database_arg = DeclareLaunchArgument( + 'force_database', + default_value='True', + description='recreate the test database on startup', + ) + + reload_schema_arg = DeclareLaunchArgument( + 'reload_schema', + default_value='True', + description='reload test schema on startup', + ) + + infer_arg = DeclareLaunchArgument( + 'infer', + default_value='True', + description='use inference engine', + ) + + query_timeout_s_arg = DeclareLaunchArgument( + 'query_timeout_s', + default_value='5.0', + description='default TypeDB query timeout in seconds', + ) + + ros_typedb_node = LifecycleNode( + package='ros_typedb', + executable='ros_typedb', + name='ros_typedb_interface', + namespace='', + output='screen', + parameters=[{ + 'schema_path': [default_schema_path], + 'data_path': [default_data_path], + 'database_name': database_name, + 'address': address, + 'force_data': force_data, + 'force_database': force_database, + 'reload_schema': reload_schema, + 'infer': infer, + 'query_timeout_s': query_timeout_s, + }], + ) + + configure_event = EmitEvent( + event=ChangeState( + lifecycle_node_matcher=matches_action(ros_typedb_node), + transition_id=lifecycle_msgs.msg.Transition.TRANSITION_CONFIGURE, + ), + ) + + activate_event = RegisterEventHandler( + OnStateTransition( + target_lifecycle_node=ros_typedb_node, + goal_state='inactive', + entities=[ + EmitEvent(event=ChangeState( + lifecycle_node_matcher=matches_action(ros_typedb_node), + transition_id=( + lifecycle_msgs.msg.Transition.TRANSITION_ACTIVATE + ), + )), + ], + ), + ) + + return LaunchDescription([ + database_name_arg, + address_arg, + force_data_arg, + force_database_arg, + reload_schema_arg, + infer_arg, + query_timeout_s_arg, + ros_typedb_node, + configure_event, + activate_event, + ]) From 399c855f89cfa70f7ce80e2f170f067d2b759ecb Mon Sep 17 00:00:00 2001 From: Gustavo Date: Sun, 28 Jun 2026 15:11:39 -0300 Subject: [PATCH 24/36] ros_typedb_tools: add delete database fault stage --- ros_typedb_tools/README.md | 168 +++++++- .../ros_typedb_stress_experiment.py | 44 +- .../ros_typedb_tools/stress_config.py | 50 ++- .../ros_typedb_tools/stress_output.py | 53 +++ .../ros_typedb_tools/stress_resolution.py | 9 +- .../ros_typedb_tools/stress_runner.py | 389 ++++++++++++++---- .../test/test_ros_typedb_stress_experiment.py | 248 +++++++++++ 7 files changed, 866 insertions(+), 95 deletions(-) diff --git a/ros_typedb_tools/README.md b/ros_typedb_tools/README.md index ce86c8c..e652ad3 100644 --- a/ros_typedb_tools/README.md +++ b/ros_typedb_tools/README.md @@ -10,7 +10,7 @@ This package currently provides three console scripts: - `typedb_rule_diagram`: generate a rule dependency/read-write diagram for one or more schema files that contain rules. - `ros_typedb_stress_experiment`: send repeated requests to the real `ros_typedb` query service and report request metrics. -Both tools can write `.svg` or `.dot` output. +The diagram tools can write `.svg` or `.dot` output. ## Prerequisites @@ -108,6 +108,31 @@ Prerequisites: - `ros_typedb_interface` is running. - The lifecycle node has been configured so the query service exists. +### Experiment Stages + +The stress tool is organized as staged experiments. Earlier stages are useful +on their own; later stages build on the same result format and command-line +interface. + +| Stage | Status | Purpose | +| --- | --- | --- | +| 1. Minimal real service load | Implemented | Send repeated requests to the real ROS `Query` service and measure latency, success, errors, and timeouts. | +| 2. Concurrent client stress | Implemented | Run many ROS service clients concurrently to find service/executor transport limits before blaming TypeDB or driver code. | +| 3. Correctness invariants | Implemented | Check that expected baseline facts remain visible during and after load, not only that requests returned. | +| 4. Mixed read/write load | Implemented | Read while inserting, updating, deleting, and cleaning up experiment-owned temporary data. | +| 5. Delete-database fault injection | Implemented | Delete the database during load and measure whether the driver fails loudly, reloads correctly, and recovers invariants. | +| 6. TypeDB restart fault injection | Planned | Stop/restart TypeDB during load and measure outage, reconnect behavior, and post-recovery correctness. | +| 7. Lifecycle stress | Planned | Trigger ROS lifecycle cleanup/configure/activate transitions during load to check teardown races. | +| 8. External schema/data profiles | Planned | Run the same harness with application-specific schema/data, query mixes, invariants, setup, and cleanup profiles. | + +Use Stage 1 when validating a single query path. Use Stage 2 when probing ROS +service throughput and timeout behavior. Use Stage 3 when checking that load +does not corrupt or hide baseline data. Use Stage 4 when validating robustness +under real read/write activity. Use Stage 5 when validating recovery after a +database deletion fault. + +### Stage 1: Minimal Real Service Load + Example: ```bash @@ -119,6 +144,12 @@ ros2 run ros_typedb_tools ros_typedb_stress_experiment \ --output ~/results/ros_typedb_stress_results.json ``` +This is the smallest useful run. It is best for confirming that the service, +query type, timeout handling, result conversion, and JSON output work for one +known query before adding concurrency. + +### Stage 2: Concurrent Client Stress + Concurrent read stress example: ```bash @@ -142,6 +173,10 @@ ros2 run ros_typedb_tools ros_typedb_stress_experiment \ --output ~/results/ros_typedb_get_stress_results.json ``` +This stage is for separating ROS service transport behavior from database +behavior. It reports throughput and latency percentiles across concurrent +clients and can write debug JSONL events for request/future/executor analysis. + When `--output` is set, timeout-only records are also written next to the main result file using the suffix `_timeouts.json`. Use `--timeout-output` to choose a different path. Timeout records include request index, client id, query index, @@ -150,6 +185,8 @@ The command exits successfully when the experiment runs to completion, even if some requests fail. Use `--fail-on-failure` when a nonzero exit code is useful for CI or scripted thresholds. +### Stage 3: Correctness Invariants + An invariant is a correctness check that should remain true while the stress load is running. Request metrics show whether the service answered; invariants show whether the database still contains expected baseline facts. @@ -196,6 +233,125 @@ Checks run every 10 seconds by default and once at the end. Use reported separately from request failures and always make the command exit with status 1. +### Stage 4: Mixed Read/Write Load + +Mixed read/write stress for the bundled `ros_typedb` test schema: + +```bash +ros2 run ros_typedb_tools ros_typedb_stress_experiment \ + --clients 10 \ + --duration-s 120 \ + --timeout-s 10 \ + --mode mixed \ + --invariant-profile test-data \ + --request-gap-s 0.01 \ + --max-in-flight 10 \ + --output ~/results/ros_typedb_mixed_stress_results.json +``` + +The packaged `test-data` mixed profile reads the database while inserting, +updating, and deleting experiment-owned temporary `robot` entities. It is +intended for the schema in `ros_typedb/test/typedb_test_data/schema.tql` and +cleans up remaining temporary robots before the final invariant check. + +Mixed read/write stress for the `ros_typedb_examples` plan schema: + +```bash +ros2 run ros_typedb_tools ros_typedb_stress_experiment \ + --clients 10 \ + --duration-s 120 \ + --timeout-s 10 \ + --mode mixed \ + --mixed-profile plan-schema \ + --invariant-profile plan-schema-mixed \ + --request-gap-s 0.01 \ + --max-in-flight 10 \ + --output ~/results/ros_typedb_plan_mixed_stress_results.json +``` + +Mixed profiles are packaged JSON files under +`ros_typedb_tools/ros_typedb_tools/profiles/`. The `plan-schema` mixed profile +writes temporary `Action` entities, so use `plan-schema-mixed` invariants when +periodic invariant checks run during that mixed workload. That invariant +profile keeps stable checks for plans, propositions, relations, and the +`collect-water-sample` sentinel action without requiring an exact total +`Action` count while temporary actions may exist. + +### Stage 5: Delete-Database Fault Injection + +This stage calls the real `delete_database` service while normal query clients +are still active. It is meant to check that a missing database does not produce +silent success against an uninitialized empty database. After the delete call, +periodic invariant passes during the active load record recovery as soon as all +checks pass once. If periodic checks do not observe recovery, the runner uses a +post-load recovery check until invariants pass or `--fault-recovery-timeout-s` +expires. + +For a reproducible run, start the bundled test-data launch file in one terminal. +It configures and activates `/ros_typedb_interface`, loads +`ros_typedb/test/typedb_test_data/schema.tql` and `data.tql`, and leaves the +schema/data paths available for recovery: + +```bash +docker exec -it ros_typedb bash -lc ' +cd /home/ubuntu-user/typedb_ws && +source /opt/ros/humble/setup.bash && +source install/setup.bash && +ros2 launch ros_typedb_examples test_data_example.launch.py' +``` + +Then run the fault-injection experiment from another terminal: + +```bash +docker exec -it ros_typedb bash -lc ' +cd /home/ubuntu-user/typedb_ws && +source /opt/ros/humble/setup.bash && +source install/setup.bash && +ros2 run ros_typedb_tools ros_typedb_stress_experiment \ + --clients 10 \ + --duration-s 180 \ + --timeout-s 10 \ + --mode read \ + --fault delete-database \ + --fault-at-s 60 \ + --invariant-profile test-data \ + --request-gap-s 0.01 \ + --max-in-flight 10 \ + --output /tmp/stage5_delete_database.json \ + --debug-events-output /tmp/stage5_delete_database_debug.jsonl' +``` + +The result JSON always includes a `fault` object with: + +- `fault`: configured fault name, such as `delete-database` +- `fault_at_s`: configured trigger time +- `fault_triggered_at_s`: monotonic timestamp when the fault was triggered +- `fault_delete_success`, `fault_delete_error`, and `fault_delete_latency_s` +- `fault_recovered_at_s`: first timestamp when all recovery invariants passed +- `fault_recovery_s`: elapsed time from trigger to successful recovery + +If the fault is triggered but invariants do not recover, the CLI exits with +status 1. Keep `--invariant-profile test-data` paired with the bundled +test-data launch; use a matching invariant profile for any other schema/data +pair. Keep `--max-in-flight 10` and `--request-gap-s 0.01` for robustness +testing so the experiment stays inside the known stable ROS service envelope. + +### Planned Fault-Injection Stages + +Stages 6-8 are not implemented yet. They are placeholders for the next +robustness checks: + +- Stage 6, TypeDB restart fault injection: stop and restart the TypeDB server + or container during load, then measure outage and reconnect behavior. +- Stage 7, lifecycle stress: call lifecycle cleanup/configure/activate while + requests are in flight, then check that service callbacks and resource + teardown do not race. +- Stage 8, external profiles: load schema-specific query mixes, invariants, + setup, and cleanup from user-provided JSON/YAML profile files instead of only + the packaged examples. + +### Diagnostics and Boundaries + For ROS client/executor debugging, `--debug-events-output` writes JSONL events for request sends, `spin_once()` calls, completed futures, timeouts, and periodic pending-request snapshots. Use `--executor single` or `--executor multi` to @@ -257,14 +413,16 @@ Useful options: - `--requests`: number of requests to send when `--duration-s` is omitted - `--clients`: number of concurrent service clients - `--duration-s`: run for this many seconds instead of a fixed request count -- `--mode`: built-in query mix to use when `--query` is omitted; currently - supports `read` +- `--mode`: built-in query mix to use when `--query` is omitted; supports + `read` and `mixed` +- `--mixed-profile`: schema-specific mixed-mode profile, such as `test-data` + or `plan-schema`; `auto` derives it from the invariant profile when possible - `--request-gap-s`: minimum delay between requests from the same client - `--max-in-flight`: maximum number of outstanding requests across all clients - `--executor`: client executor mode, one of `global`, `single`, or `multi` - `--debug-events-output`: optional JSONL debug event path -- `--invariant-profile`: optional correctness profile, such as `test-data` or - `plan-schema` +- `--invariant-profile`: optional correctness profile, such as `test-data`, + `plan-schema`, or `plan-schema-mixed` - `--invariant-period-s`: seconds between periodic invariant checks - `--timeout-s`: per-request client and server timeout - `--output`: optional JSON results path diff --git a/ros_typedb_tools/ros_typedb_tools/ros_typedb_stress_experiment.py b/ros_typedb_tools/ros_typedb_tools/ros_typedb_stress_experiment.py index f7d8b32..7f0d356 100644 --- a/ros_typedb_tools/ros_typedb_tools/ros_typedb_stress_experiment.py +++ b/ros_typedb_tools/ros_typedb_tools/ros_typedb_stress_experiment.py @@ -180,6 +180,40 @@ def build_argument_parser() -> argparse.ArgumentParser: 'making the tool command fail.' ), ) + parser.add_argument( + '--fault', + choices=('none', 'delete-database'), + default='none', + help=( + 'Fault to inject during the experiment. ' + 'Defaults to none.' + ), + ) + parser.add_argument( + '--fault-at-s', + type=float, + help=( + 'Seconds into the experiment at which to trigger the fault. ' + 'Required when --fault is not none.' + ), + ) + parser.add_argument( + '--fault-recovery-timeout-s', + type=float, + default=30.0, + help=( + 'Seconds to wait for invariants to pass after a fault. ' + 'Defaults to 30.' + ), + ) + parser.add_argument( + '--delete-database-service-name', + help=( + 'delete_database service name. Defaults to the --service-name ' + 'prefix with /delete_database appended ' + '(e.g. /ros_typedb_interface/delete_database).' + ), + ) return parser @@ -210,6 +244,7 @@ def _write_requested_outputs( invariant_profile=result.config.invariant_profile_name, invariant_period_s=args.invariant_period_s, invariant_records=result.invariant_records, + fault_result=result.fault_result, ) print(f' output: {output_path}') @@ -239,8 +274,15 @@ def main(argv: list[str] | None = None) -> int: invariant_summary = summarize_invariant_records( result.invariant_records ) - print_summary(summary, invariant_summary) + print_summary(summary, invariant_summary, result.fault_result) _write_requested_outputs(args, result) + fault_result = result.fault_result + if ( + fault_result.fault != 'none' + and fault_result.fault_triggered_at_s is not None + and fault_result.fault_recovered_at_s is None + ): + return 1 if invariant_summary['failures'] > 0: return 1 if args.fail_on_failure and summary['failures'] > 0: diff --git a/ros_typedb_tools/ros_typedb_tools/stress_config.py b/ros_typedb_tools/ros_typedb_tools/stress_config.py index 7033f51..d6c4fe1 100644 --- a/ros_typedb_tools/ros_typedb_tools/stress_config.py +++ b/ros_typedb_tools/ros_typedb_tools/stress_config.py @@ -127,7 +127,8 @@ class InvariantRecord: @dataclass(frozen=True) class ResolvedStressConfig: - """Concrete runtime configuration derived from CLI arguments. + """ + Concrete runtime configuration derived from CLI arguments. The CLI exposes convenient selectors such as ``--mixed-profile auto``. The runner should not need to interpret those selectors while it is also @@ -142,6 +143,23 @@ class ResolvedStressConfig: invariant_profile_name: str max_in_flight: int debug_events_output: Path | None + fault: str + fault_at_s: float | None + fault_recovery_timeout_s: float + + +@dataclass(frozen=True) +class FaultResult: + """Recorded timing and outcome of a fault injection.""" + + fault: str + fault_at_s: float | None + fault_triggered_at_s: float | None + fault_delete_success: bool | None + fault_delete_error: str | None + fault_delete_latency_s: float | None + fault_recovered_at_s: float | None + fault_recovery_s: float | None @dataclass(frozen=True) @@ -151,6 +169,7 @@ class StressExperimentResult: records: list[RequestRecord] invariant_records: list[InvariantRecord] config: ResolvedStressConfig + fault_result: FaultResult def validate_experiment_args(args: argparse.Namespace) -> None: @@ -184,10 +203,37 @@ def validate_experiment_args(args: argparse.Namespace) -> None: mixed_profile = getattr(args, 'mixed_profile', 'auto') if mixed_profile not in MIXED_PROFILE_NAMES: raise ValueError(f'unsupported mixed profile: {mixed_profile}') + fault = getattr(args, 'fault', 'none') + fault_at_s = getattr(args, 'fault_at_s', None) + fault_recovery_timeout_s = getattr(args, 'fault_recovery_timeout_s', 30.0) + if fault == 'delete-database': + if args.invariant_profile == 'none': + raise ValueError( + '--fault delete-database requires --invariant-profile to be set' + ) + if args.duration_s is None: + raise ValueError( + '--fault delete-database requires --duration-s' + ) + if fault_at_s is None: + raise ValueError( + '--fault delete-database requires --fault-at-s' + ) + if fault_at_s <= 0: + raise ValueError('--fault-at-s must be greater than zero') + if fault_at_s >= args.duration_s: + raise ValueError( + '--fault-at-s must be less than --duration-s' + ) + if fault_recovery_timeout_s <= 0: + raise ValueError( + '--fault-recovery-timeout-s must be greater than zero' + ) def build_query_specs(args: argparse.Namespace) -> list[QuerySpec]: - """Build fixed query specs for explicit-query or read-only modes. + """ + Build fixed query specs for explicit-query or read-only modes. Mixed mode uses schema-specific templates and is resolved in ``stress_resolution``. Keeping this helper fixed-query-only avoids hidden diff --git a/ros_typedb_tools/ros_typedb_tools/stress_output.py b/ros_typedb_tools/ros_typedb_tools/stress_output.py index 5794a67..3dc3a05 100644 --- a/ros_typedb_tools/ros_typedb_tools/stress_output.py +++ b/ros_typedb_tools/ros_typedb_tools/stress_output.py @@ -23,6 +23,7 @@ import time from typing import Any +from ros_typedb_tools.stress_config import FaultResult from ros_typedb_tools.stress_config import InvariantRecord from ros_typedb_tools.stress_config import QuerySpec from ros_typedb_tools.stress_config import RequestRecord @@ -88,6 +89,31 @@ def summarize_invariant_records( } +def _fault_payload(fault_result: FaultResult | None) -> dict[str, Any]: + """Build a stable JSON object for fault result fields.""" + if fault_result is None: + fault_result = FaultResult( + fault='none', + fault_at_s=None, + fault_triggered_at_s=None, + fault_delete_success=None, + fault_delete_error=None, + fault_delete_latency_s=None, + fault_recovered_at_s=None, + fault_recovery_s=None, + ) + return { + 'fault': fault_result.fault, + 'fault_at_s': fault_result.fault_at_s, + 'fault_triggered_at_s': fault_result.fault_triggered_at_s, + 'fault_delete_success': fault_result.fault_delete_success, + 'fault_delete_error': fault_result.fault_delete_error, + 'fault_delete_latency_s': fault_result.fault_delete_latency_s, + 'fault_recovered_at_s': fault_result.fault_recovered_at_s, + 'fault_recovery_s': fault_result.fault_recovery_s, + } + + def write_results( output_path: Path, *, @@ -107,6 +133,7 @@ def write_results( invariant_profile: str = 'none', invariant_period_s: float = 10.0, invariant_records: list[InvariantRecord] | None = None, + fault_result: FaultResult | None = None, ) -> None: """Write experiment results to a JSON file.""" invariants = invariant_records or [] @@ -129,6 +156,7 @@ def write_results( 'timeout_s': timeout_s, 'summary': summarize_records(records), 'invariant_summary': summarize_invariant_records(invariants), + 'fault': _fault_payload(fault_result), 'requests': [asdict(record) for record in records], 'invariants': [asdict(record) for record in invariants], } @@ -210,6 +238,7 @@ def log(self, event: str, **fields: Any) -> None: def print_summary( summary: dict[str, Any], invariant_summary: dict[str, Any] | None = None, + fault_result: FaultResult | None = None, ) -> None: """Print compact experiment summary.""" latency = summary['latency_s'] @@ -231,3 +260,27 @@ def print_summary( print(f" invariants_successes: {invariant_summary['successes']}") print(f" invariants_failures: {invariant_summary['failures']}") print(f" invariants_timeouts: {invariant_summary['timeouts']}") + if fault_result is not None and fault_result.fault != 'none': + triggered_at = ( + f'{fault_result.fault_triggered_at_s:.2f}s' + if fault_result.fault_triggered_at_s is not None + else 'not triggered' + ) + delete_latency = ( + f'{fault_result.fault_delete_latency_s:.3f}s' + if fault_result.fault_delete_latency_s is not None + else 'n/a' + ) + delete_status = ( + 'ok' if fault_result.fault_delete_success else 'FAILED' + ) + recovery_str = ( + f'{fault_result.fault_recovery_s:.2f}s' + if fault_result.fault_recovery_s is not None + else 'not recovered' + ) + print( + f' fault: {fault_result.fault} triggered at {triggered_at} | ' + f'delete: {delete_status} ({delete_latency}) | ' + f'recovery: {recovery_str}' + ) diff --git a/ros_typedb_tools/ros_typedb_tools/stress_resolution.py b/ros_typedb_tools/ros_typedb_tools/stress_resolution.py index 518917c..93865a8 100644 --- a/ros_typedb_tools/ros_typedb_tools/stress_resolution.py +++ b/ros_typedb_tools/ros_typedb_tools/stress_resolution.py @@ -30,7 +30,8 @@ def resolve_stress_config( args: argparse.Namespace, ) -> ResolvedStressConfig: - """Build the concrete configuration used by the runner and output writer. + """ + Build the concrete configuration used by the runner and output writer. This is the only place that combines the raw CLI flags with packaged profiles. The runner receives ordinary query specs and an optional loaded @@ -47,6 +48,9 @@ def resolve_stress_config( if args.debug_events_output else None ) + fault = getattr(args, 'fault', 'none') + fault_at_s = getattr(args, 'fault_at_s', None) + fault_recovery_timeout_s = getattr(args, 'fault_recovery_timeout_s', 30.0) return ResolvedStressConfig( query_specs=query_specs, @@ -58,6 +62,9 @@ def resolve_stress_config( invariant_profile_name=args.invariant_profile, max_in_flight=args.max_in_flight or args.clients, debug_events_output=debug_events_output, + fault=fault, + fault_at_s=fault_at_s, + fault_recovery_timeout_s=fault_recovery_timeout_s, ) diff --git a/ros_typedb_tools/ros_typedb_tools/stress_runner.py b/ros_typedb_tools/ros_typedb_tools/stress_runner.py index 78a3153..6975c94 100644 --- a/ros_typedb_tools/ros_typedb_tools/stress_runner.py +++ b/ros_typedb_tools/ros_typedb_tools/stress_runner.py @@ -28,6 +28,7 @@ from ros_typedb_msgs.srv import Query from ros_typedb_tools.stress_config import build_query_request +from ros_typedb_tools.stress_config import FaultResult from ros_typedb_tools.stress_config import InvariantRecord from ros_typedb_tools.stress_config import InvariantSpec from ros_typedb_tools.stress_config import QuerySpec @@ -40,6 +41,7 @@ from ros_typedb_tools.stress_mixed import MixedQueryProfile from ros_typedb_tools.stress_output import DebugEventWriter from ros_typedb_tools.stress_resolution import resolve_stress_config +from std_srvs.srv import Empty @dataclass @@ -67,6 +69,12 @@ class _RunState: deadline_s: float | None next_snapshot_s: float next_invariant_s: float + fault_triggered: bool = False + fault_triggered_at_s: float | None = None + fault_delete_success: bool | None = None + fault_delete_error: str | None = None + fault_delete_latency_s: float | None = None + fault_recovered_at_s: float | None = None def _build_executor(args: argparse.Namespace) -> Any | None: @@ -405,6 +413,28 @@ def _process_pending_requests( return timed_out_this_cycle +def _spin_until_future_done( + future: Any, + node: Any, + executor: Any | None, + timeout_s: float, + started_at_s: float, +) -> tuple[bool, float]: + """ + Spin until future completes or timeout expires. + + Returns (timed_out, ended_at_s). Cancels future on timeout. + """ + while True: + _spin_once(node, executor) + now_s = time.monotonic() + if future.done(): + return False, now_s + if now_s - started_at_s >= timeout_s: + future.cancel() + return True, now_s + + def _run_invariant_check( *, node: Any, @@ -430,57 +460,54 @@ def _run_invariant_check( phase=phase, ) - while True: - _spin_once(node, executor) - now_s = time.monotonic() - if future.done(): - exception = future.exception() - if exception is not None: - record = evaluate_invariant_response( - index=index, - phase=phase, - spec=spec, - started_at_s=started_at_s, - ended_at_s=now_s, - response=None, - exception_message=str(exception), - ) - else: - record = evaluate_invariant_response( - index=index, - phase=phase, - spec=spec, - started_at_s=started_at_s, - ended_at_s=now_s, - response=future.result(), - ) - debug_events.log( - 'invariant_complete', - index=index, - name=spec.name, - phase=phase, - success=record.success, - latency_s=record.latency_s, - error_message=record.error_message, - ) - return record - - if now_s - started_at_s >= timeout_s: - future.cancel() - record = build_timeout_invariant_record( - index=index, - phase=phase, - spec=spec, - started_at_s=started_at_s, - ) - debug_events.log( - 'invariant_timeout', - index=index, - name=spec.name, - phase=phase, - latency_s=record.latency_s, - ) - return record + timed_out, now_s = _spin_until_future_done( + future, node, executor, timeout_s, started_at_s + ) + if timed_out: + record = build_timeout_invariant_record( + index=index, + phase=phase, + spec=spec, + started_at_s=started_at_s, + ) + debug_events.log( + 'invariant_timeout', + index=index, + name=spec.name, + phase=phase, + latency_s=record.latency_s, + ) + return record + exception = future.exception() + if exception is not None: + record = evaluate_invariant_response( + index=index, + phase=phase, + spec=spec, + started_at_s=started_at_s, + ended_at_s=now_s, + response=None, + exception_message=str(exception), + ) + else: + record = evaluate_invariant_response( + index=index, + phase=phase, + spec=spec, + started_at_s=started_at_s, + ended_at_s=now_s, + response=future.result(), + ) + debug_events.log( + 'invariant_complete', + index=index, + name=spec.name, + phase=phase, + success=record.success, + latency_s=record.latency_s, + error_message=record.error_message, + ) + return record def _run_invariant_profile( @@ -493,7 +520,8 @@ def _run_invariant_profile( phase: str, timeout_s: float, debug_events: DebugEventWriter, -) -> None: +) -> list[InvariantRecord]: + records: list[InvariantRecord] = [] for spec in specs: record = _run_invariant_check( node=node, @@ -507,6 +535,30 @@ def _run_invariant_profile( ) state.invariant_records.append(record) state.next_invariant_index += 1 + records.append(record) + return records + + +def _mark_fault_recovered_from_invariant_pass( + *, + state: _RunState, + records: list[InvariantRecord], + debug_events: DebugEventWriter, +) -> bool: + """Record fault recovery when one full invariant pass succeeds.""" + if state.fault_triggered_at_s is None: + return False + if state.fault_recovered_at_s is not None: + return False + if not records or not all(record.success for record in records): + return False + + state.fault_recovered_at_s = time.monotonic() + debug_events.log( + 'recovery_complete', + recovery_s=state.fault_recovered_at_s - state.fault_triggered_at_s, + ) + return True def _run_cleanup_request( @@ -541,38 +593,36 @@ def _run_cleanup_request( cleanup_key=query_spec.cleanup_key, ) - while True: - _spin_once(node, executor) - now_s = time.monotonic() - if future.done(): - _record_done_future( - pending=pending, - now_s=now_s, - pending_count=1, - state=state, - debug_events=debug_events, - ) - debug_events.log( - 'cleanup_complete', - index=pending.index, - cleanup_key=query_spec.cleanup_key, - ) - return - - if now_s - started_at_s >= timeout_s: - _record_timeout( - pending=pending, - now_s=now_s, - pending_count=1, - state=state, - debug_events=debug_events, - ) - debug_events.log( - 'cleanup_timeout', - index=pending.index, - cleanup_key=query_spec.cleanup_key, - ) - return + timed_out, now_s = _spin_until_future_done( + future, node, executor, timeout_s, started_at_s + ) + if timed_out: + _record_timeout( + pending=pending, + now_s=now_s, + pending_count=1, + state=state, + debug_events=debug_events, + ) + debug_events.log( + 'cleanup_timeout', + index=pending.index, + cleanup_key=query_spec.cleanup_key, + ) + return + _record_done_future( + pending=pending, + now_s=now_s, + pending_count=1, + state=state, + debug_events=debug_events, + ) + debug_events.log( + 'cleanup_complete', + index=pending.index, + cleanup_key=query_spec.cleanup_key, + ) + return def _cleanup_mixed_data( @@ -614,6 +664,13 @@ def _cleanup_mixed_data( ) +def _derive_delete_db_service_name(service_name: str) -> str: + """Derive delete_database service name from the query service name.""" + parts = service_name.rstrip('/').rsplit('/', 1) + prefix = parts[0] if len(parts) > 1 else '' + return prefix + '/delete_database' + + def _maybe_run_periodic_invariants( *, args: argparse.Namespace, @@ -632,7 +689,7 @@ def _maybe_run_periodic_invariants( if now_s < state.next_invariant_s: return - _run_invariant_profile( + records = _run_invariant_profile( node=node, executor=executor, client=invariant_client, @@ -642,9 +699,105 @@ def _maybe_run_periodic_invariants( timeout_s=args.timeout_s, debug_events=debug_events, ) + _mark_fault_recovered_from_invariant_pass( + state=state, + records=records, + debug_events=debug_events, + ) state.next_invariant_s = time.monotonic() + args.invariant_period_s +def _trigger_delete_database_fault( + *, + node: Any, + executor: Any | None, + delete_db_client: Any, + state: _RunState, + timeout_s: float, + debug_events: DebugEventWriter, +) -> None: + """Call the delete_database service and record fault timing.""" + started_at_s = time.monotonic() + pending_count = len(state.pending_by_client) + state.fault_triggered = True + state.fault_triggered_at_s = started_at_s + debug_events.log( + 'fault_triggered', + fault='delete-database', + triggered_at_s=started_at_s, + pending_count_at_fault=pending_count, + ) + request = Empty.Request() + future = delete_db_client.call_async(request) + timed_out, ended_at_s = _spin_until_future_done( + future, node, executor, timeout_s, started_at_s + ) + state.fault_delete_latency_s = ended_at_s - started_at_s + if timed_out: + state.fault_delete_success = False + state.fault_delete_error = ( + f'delete_database service timed out after {timeout_s:g}s' + ) + debug_events.log( + 'fault_delete_timeout', + latency_s=state.fault_delete_latency_s, + ) + else: + exc = future.exception() + if exc is not None: + state.fault_delete_success = False + state.fault_delete_error = str(exc) + debug_events.log( + 'fault_delete_error', + error=state.fault_delete_error, + latency_s=state.fault_delete_latency_s, + ) + else: + state.fault_delete_success = True + debug_events.log( + 'fault_delete_complete', + latency_s=state.fault_delete_latency_s, + ) + + +def _wait_for_recovery( + *, + node: Any, + executor: Any | None, + client: Any, + specs: list[InvariantSpec], + state: _RunState, + recovery_timeout_s: float, + invariant_timeout_s: float, + debug_events: DebugEventWriter, +) -> None: + """Poll invariants until all pass or recovery_timeout_s expires.""" + if not specs or state.fault_triggered_at_s is None: + return + if state.fault_recovered_at_s is not None: + return + recovery_deadline_s = state.fault_triggered_at_s + recovery_timeout_s + debug_events.log('recovery_started', recovery_timeout_s=recovery_timeout_s) + while time.monotonic() < recovery_deadline_s: + records = _run_invariant_profile( + node=node, + executor=executor, + client=client, + specs=specs, + state=state, + phase='recovery', + timeout_s=invariant_timeout_s, + debug_events=debug_events, + ) + if _mark_fault_recovered_from_invariant_pass( + state=state, + records=records, + debug_events=debug_events, + ): + return + debug_events.log('recovery_timeout', recovery_timeout_s=recovery_timeout_s) + + def run_experiment(args: argparse.Namespace) -> StressExperimentResult: """Run the stress experiment.""" config = resolve_stress_config(args) @@ -667,6 +820,16 @@ def run_experiment(args: argparse.Namespace) -> StressExperimentResult: if config.mixed_profile is not None else None ) + delete_db_service_name = ( + args.delete_database_service_name + if getattr(args, 'delete_database_service_name', None) + else _derive_delete_db_service_name(args.service_name) + ) + delete_db_client = ( + node.create_client(Empty, delete_db_service_name) + if getattr(args, 'fault', 'none') != 'none' + else None + ) deadline_s = ( time.monotonic() + args.duration_s if args.duration_s is not None @@ -703,11 +866,13 @@ def run_experiment(args: argparse.Namespace) -> StressExperimentResult: _wait_for_all_clients( clients + ([invariant_client] if invariant_client else []) - + ([cleanup_client] if cleanup_client else []), + + ([cleanup_client] if cleanup_client else []) + + ([delete_db_client] if delete_db_client is not None else []), service_name=args.service_name, wait_service_timeout_s=args.wait_service_timeout_s, debug_events=debug_events, ) + experiment_started_at_s = time.monotonic() while ( _should_send_request(state, requests=args.requests) @@ -754,7 +919,41 @@ def run_experiment(args: argparse.Namespace) -> StressExperimentResult: now_s=now_s, debug_events=debug_events, ) + if ( + delete_db_client is not None + and not state.fault_triggered + and config.fault_at_s is not None + and time.monotonic() - experiment_started_at_s + >= config.fault_at_s + ): + _trigger_delete_database_fault( + node=node, + executor=executor, + delete_db_client=delete_db_client, + state=state, + timeout_s=args.timeout_s, + debug_events=debug_events, + ) + + # 1. Recovery (must come before cleanup — cleanup queries trigger DB + # recreation, which would silently advance recovery unobserved) + if ( + state.fault_triggered + and config.invariant_specs + and invariant_client is not None + ): + _wait_for_recovery( + node=node, + executor=executor, + client=invariant_client, + specs=config.invariant_specs, + state=state, + recovery_timeout_s=config.fault_recovery_timeout_s, + invariant_timeout_s=args.timeout_s, + debug_events=debug_events, + ) + # 2. Mixed cleanup (existing) _cleanup_mixed_data( args=args, node=node, @@ -765,6 +964,7 @@ def run_experiment(args: argparse.Namespace) -> StressExperimentResult: debug_events=debug_events, ) + # 3. Final invariants (existing) if config.invariant_specs and invariant_client is not None: _run_invariant_profile( node=node, @@ -795,8 +995,25 @@ def run_experiment(args: argparse.Namespace) -> StressExperimentResult: executor.remove_node(node) executor.shutdown() node.destroy_node() + fault_recovery_s = ( + state.fault_recovered_at_s - state.fault_triggered_at_s + if state.fault_triggered_at_s is not None + and state.fault_recovered_at_s is not None + else None + ) + fault_result = FaultResult( + fault=config.fault, + fault_at_s=config.fault_at_s, + fault_triggered_at_s=state.fault_triggered_at_s, + fault_delete_success=state.fault_delete_success, + fault_delete_error=state.fault_delete_error, + fault_delete_latency_s=state.fault_delete_latency_s, + fault_recovered_at_s=state.fault_recovered_at_s, + fault_recovery_s=fault_recovery_s, + ) return StressExperimentResult( records=state.records, invariant_records=state.invariant_records, config=config, + fault_result=fault_result, ) diff --git a/ros_typedb_tools/test/test_ros_typedb_stress_experiment.py b/ros_typedb_tools/test/test_ros_typedb_stress_experiment.py index 0cb8761..48f442f 100644 --- a/ros_typedb_tools/test/test_ros_typedb_stress_experiment.py +++ b/ros_typedb_tools/test/test_ros_typedb_stress_experiment.py @@ -14,16 +14,21 @@ from ros_typedb_tools.fake_query_service import ( build_fake_service_argument_parser, ) +import ros_typedb_tools.ros_typedb_stress_experiment as stress_cli from ros_typedb_tools.ros_typedb_stress_experiment import ( build_argument_parser, ) from ros_typedb_tools.stress_config import ( build_invariant_specs, build_query_specs, + FaultResult, InvariantRecord, InvariantSpec, QUERY_TYPE_BY_NAME, RequestRecord, + ResolvedStressConfig, + StressExperimentResult, + validate_experiment_args, ) from ros_typedb_tools.stress_invariants import evaluate_invariant_response from ros_typedb_tools.stress_mixed import build_mixed_cleanup_query_spec @@ -39,6 +44,10 @@ write_timeout_results, ) from ros_typedb_tools.stress_resolution import resolve_stress_config +from ros_typedb_tools.stress_runner import ( + _mark_fault_recovered_from_invariant_pass, +) +from ros_typedb_tools.stress_runner import _RunState from ros_typedb_tools.stress_runner import run_experiment @@ -89,6 +98,28 @@ def _invariant_record( ) +def _run_state(**overrides) -> _RunState: + """Return minimal run state for helper tests.""" + state = _RunState( + records=[], + invariant_records=[], + pending_by_client={}, + last_started_by_client=[], + sent_count_by_client=[], + mixed_cleanup_keys=set(), + mixed_profile=None, + mixed_run_id='test', + next_index=0, + next_invariant_index=0, + deadline_s=None, + next_snapshot_s=0.0, + next_invariant_s=0.0, + ) + for name, value in overrides.items(): + setattr(state, name, value) + return state + + def _aggregate_count_response(value: int, *, success: bool = True): parameter_value = ParameterValue() parameter_value.type = ParameterType.PARAMETER_INTEGER @@ -304,6 +335,47 @@ def test_summarize_invariant_records_counts_failures_and_timeouts(): assert summary['timeouts'] == 1 +def test_fault_recovery_is_marked_from_successful_invariant_pass(): + """Check a successful post-fault invariant pass records recovery.""" + state = _run_state( + fault_triggered=True, + fault_triggered_at_s=1.0, + ) + + marked = _mark_fault_recovered_from_invariant_pass( + state=state, + records=[ + _invariant_record(0, success=True), + _invariant_record(1, success=True), + ], + debug_events=DebugEventWriter(None), + ) + + assert marked is True + assert state.fault_recovered_at_s is not None + assert state.fault_recovered_at_s > state.fault_triggered_at_s + + +def test_fault_recovery_ignores_failed_invariant_pass(): + """Check failed invariant passes do not record recovery.""" + state = _run_state( + fault_triggered=True, + fault_triggered_at_s=1.0, + ) + + marked = _mark_fault_recovered_from_invariant_pass( + state=state, + records=[ + _invariant_record(0, success=True), + _invariant_record(1, success=False), + ], + debug_events=DebugEventWriter(None), + ) + + assert marked is False + assert state.fault_recovered_at_s is None + + def test_write_results_writes_summary_and_request_records(tmp_path: Path): """Check JSON output includes metadata, summary, and records.""" output_path = tmp_path / 'stress' / 'results.json' @@ -329,6 +401,16 @@ def test_write_results_writes_summary_and_request_records(tmp_path: Path): invariant_profile='test-data', invariant_period_s=5.0, invariant_records=[_invariant_record(0, success=True)], + fault_result=FaultResult( + fault='delete-database', + fault_at_s=10.0, + fault_triggered_at_s=10.25, + fault_delete_success=True, + fault_delete_error=None, + fault_delete_latency_s=0.125, + fault_recovered_at_s=12.75, + fault_recovery_s=2.5, + ), ) payload = json.loads(output_path.read_text(encoding='utf-8')) @@ -355,6 +437,42 @@ def test_write_results_writes_summary_and_request_records(tmp_path: Path): assert payload['requests'][0]['success'] is True assert payload['invariant_summary']['successes'] == 1 assert payload['invariants'][0]['name'] == 'person-count' + assert payload['fault'] == { + 'fault': 'delete-database', + 'fault_at_s': 10.0, + 'fault_triggered_at_s': 10.25, + 'fault_delete_success': True, + 'fault_delete_error': None, + 'fault_delete_latency_s': 0.125, + 'fault_recovered_at_s': 12.75, + 'fault_recovery_s': 2.5, + } + + +def test_write_results_writes_default_fault_payload(tmp_path: Path): + """Check JSON output always includes fault metadata.""" + output_path = tmp_path / 'results.json' + + write_results( + output_path, + service_name='/ros_typedb_interface/query', + query_type='fetch', + query='match $x isa entity; fetch $x: attribute;', + timeout_s=5.0, + records=[], + ) + + payload = json.loads(output_path.read_text(encoding='utf-8')) + assert payload['fault'] == { + 'fault': 'none', + 'fault_at_s': None, + 'fault_triggered_at_s': None, + 'fault_delete_success': None, + 'fault_delete_error': None, + 'fault_delete_latency_s': None, + 'fault_recovered_at_s': None, + 'fault_recovery_s': None, + } def test_default_timeout_output_path_uses_result_stem(): @@ -722,3 +840,133 @@ def test_run_experiment_rejects_invalid_client_count(): with pytest.raises(ValueError, match='--clients'): run_experiment(args) + + +def _base_args(**overrides): + """Return a SimpleNamespace with valid base args for validation tests.""" + base = { + 'requests': 1, + 'clients': 1, + 'duration_s': 60.0, + 'timeout_s': 5.0, + 'wait_service_timeout_s': 1.0, + 'request_gap_s': 0.0, + 'max_in_flight': None, + 'executor': 'global', + 'executor_threads': 2, + 'debug_events_output': None, + 'invariant_profile': 'none', + 'invariant_period_s': 10.0, + 'query': None, + 'query_type': None, + 'mode': 'read', + 'mixed_profile': 'auto', + 'fault': 'none', + 'fault_at_s': None, + 'fault_recovery_timeout_s': 30.0, + } + base.update(overrides) + return SimpleNamespace(**base) + + +def test_validate_args_fault_delete_database_requires_invariant_profile(): + """Check --fault delete-database requires --invariant-profile to be set.""" + args = _base_args( + fault='delete-database', + invariant_profile='none', + fault_at_s=10.0, + ) + + with pytest.raises(ValueError, match='--invariant-profile'): + validate_experiment_args(args) + + +def test_validate_args_fault_delete_database_requires_duration_s(): + """Check --fault delete-database requires --duration-s.""" + args = _base_args( + fault='delete-database', + invariant_profile='test-data', + duration_s=None, + fault_at_s=10.0, + ) + + with pytest.raises(ValueError, match='--duration-s'): + validate_experiment_args(args) + + +def test_validate_args_fault_delete_database_requires_fault_at_s(): + """Check --fault delete-database requires --fault-at-s.""" + args = _base_args( + fault='delete-database', + invariant_profile='test-data', + duration_s=60.0, + fault_at_s=None, + ) + + with pytest.raises(ValueError, match='--fault-at-s'): + validate_experiment_args(args) + + +def test_validate_args_fault_at_s_must_be_less_than_duration_s(): + """Check --fault-at-s must be less than --duration-s.""" + args = _base_args( + fault='delete-database', + invariant_profile='test-data', + duration_s=30.0, + fault_at_s=30.0, + ) + + with pytest.raises(ValueError, match='--fault-at-s must be less than'): + validate_experiment_args(args) + + +def test_validate_args_fault_recovery_timeout_s_must_be_positive(): + """Check --fault-recovery-timeout-s must be greater than zero.""" + args = _base_args(fault_recovery_timeout_s=0.0) + + with pytest.raises(ValueError, match='--fault-recovery-timeout-s'): + validate_experiment_args(args) + + +def test_validate_args_fault_none_passes_without_fault_args(): + """Check --fault none (default) passes validation with no fault args.""" + args = _base_args() + + validate_experiment_args(args) # should not raise + + +def test_main_fails_when_triggered_fault_does_not_recover(monkeypatch): + """Check CLI exits nonzero for unrecovered triggered faults.""" + fault_result = FaultResult( + fault='delete-database', + fault_at_s=10.0, + fault_triggered_at_s=10.0, + fault_delete_success=True, + fault_delete_error=None, + fault_delete_latency_s=0.1, + fault_recovered_at_s=None, + fault_recovery_s=None, + ) + result = StressExperimentResult( + records=[], + invariant_records=[], + config=ResolvedStressConfig( + query_specs=[], + invariant_specs=[], + mixed_profile=None, + mixed_profile_name=None, + invariant_profile_name='test-data', + max_in_flight=1, + debug_events_output=None, + fault='delete-database', + fault_at_s=10.0, + fault_recovery_timeout_s=30.0, + ), + fault_result=fault_result, + ) + + monkeypatch.setattr(stress_cli.rclpy, 'init', lambda args=None: None) + monkeypatch.setattr(stress_cli.rclpy, 'shutdown', lambda: None) + monkeypatch.setattr(stress_cli, 'run_experiment', lambda args: result) + + assert stress_cli.main([]) == 1 From 2869515de3c27218d80a7aad656775e10f211959 Mon Sep 17 00:00:00 2001 From: Gustavo Date: Sun, 28 Jun 2026 16:07:25 -0300 Subject: [PATCH 25/36] ros_typedb_tools: add TypeDB restart fault stage --- ros_typedb_tools/README.md | 66 +++- .../ros_typedb_stress_experiment.py | 90 ++++- .../ros_typedb_tools/stress_config.py | 35 +- .../ros_typedb_tools/stress_output.py | 48 ++- .../ros_typedb_tools/stress_resolution.py | 16 + .../ros_typedb_tools/stress_runner.py | 267 ++++++++++++- .../test/test_ros_typedb_stress_experiment.py | 371 +++++++++++++++++- 7 files changed, 873 insertions(+), 20 deletions(-) diff --git a/ros_typedb_tools/README.md b/ros_typedb_tools/README.md index e652ad3..1e4f78d 100644 --- a/ros_typedb_tools/README.md +++ b/ros_typedb_tools/README.md @@ -121,7 +121,7 @@ interface. | 3. Correctness invariants | Implemented | Check that expected baseline facts remain visible during and after load, not only that requests returned. | | 4. Mixed read/write load | Implemented | Read while inserting, updating, deleting, and cleaning up experiment-owned temporary data. | | 5. Delete-database fault injection | Implemented | Delete the database during load and measure whether the driver fails loudly, reloads correctly, and recovers invariants. | -| 6. TypeDB restart fault injection | Planned | Stop/restart TypeDB during load and measure outage, reconnect behavior, and post-recovery correctness. | +| 6. TypeDB restart fault injection | Implemented | Stop/restart TypeDB during load and measure outage, reconnect behavior, and post-recovery correctness. | | 7. Lifecycle stress | Planned | Trigger ROS lifecycle cleanup/configure/activate transitions during load to check teardown races. | | 8. External schema/data profiles | Planned | Run the same harness with application-specific schema/data, query mixes, invariants, setup, and cleanup profiles. | @@ -129,7 +129,8 @@ Use Stage 1 when validating a single query path. Use Stage 2 when probing ROS service throughput and timeout behavior. Use Stage 3 when checking that load does not corrupt or hide baseline data. Use Stage 4 when validating robustness under real read/write activity. Use Stage 5 when validating recovery after a -database deletion fault. +database deletion fault. Use Stage 6 when validating driver reconnect behavior +after the TypeDB server process or container is restarted. ### Stage 1: Minimal Real Service Load @@ -336,13 +337,68 @@ test-data launch; use a matching invariant profile for any other schema/data pair. Keep `--max-in-flight 10` and `--request-gap-s 0.01` for robustness testing so the experiment stays inside the known stable ROS service envelope. +### Stage 6: TypeDB Restart Fault Injection + +This stage restarts the TypeDB server while normal query clients continue +sending requests. It is meant to check that server outages produce bounded, +explicit failures and that later requests reconnect to TypeDB without +restarting the ROS lifecycle node. Recovery is still correctness-based: the +first complete post-fault invariant pass marks `fault_recovered_at_s`. + +Run TypeDB separately from the ROS node and stress process. Do not pass the +same Docker container that is running `ros_typedb_interface` or the experiment +to `--typedb-container`, because stopping that container stops the test itself. +By default, the process fault uses `pkill -f "typedb/core/server"` to stop +TypeDB and `typedb server` to start it again: + +```bash +ros2 run ros_typedb_tools ros_typedb_stress_experiment \ + --clients 10 \ + --duration-s 240 \ + --timeout-s 10 \ + --mode read \ + --fault restart-typedb \ + --typedb-restart-delay-s 2 \ + --fault-at-s 60 \ + --invariant-profile test-data \ + --request-gap-s 0.01 \ + --max-in-flight 10 \ + --output /tmp/stage6_restart_typedb.json \ + --debug-events-output /tmp/stage6_restart_typedb_debug.jsonl +``` + +Override the process commands with `--typedb-stop-command` and +`--typedb-start-command` when needed. For a TypeDB-only Docker container, pass +`--typedb-container`; it overrides the process commands: + +```bash +--typedb-container typedb_server +``` + +The restart controller runs in a background thread, so the main experiment loop +keeps spinning ROS futures and sending requests during the outage. The result +JSON adds restart fields to the `fault` object: + +- `fault_restart_stop_success`, `fault_restart_stop_error`, and + `fault_restart_stop_latency_s` +- `fault_restart_start_success`, `fault_restart_start_error`, and + `fault_restart_start_latency_s` +- `fault_restart_delay_s`: configured delay between stop and start +- `fault_restart_outage_s`: elapsed controller time from stop start to start + completion +- `fault_observed_outage_s`: elapsed time from the first failed post-fault + request result to the first later successful request result + +If the fault is triggered but invariants do not recover, the CLI exits with +status 1. Request failures during the outage remain recorded in the JSON output; +use `--fail-on-failure` only when you want any request failure to fail the +command even if invariants later recover. + ### Planned Fault-Injection Stages -Stages 6-8 are not implemented yet. They are placeholders for the next +Stages 7-8 are not implemented yet. They are placeholders for the next robustness checks: -- Stage 6, TypeDB restart fault injection: stop and restart the TypeDB server - or container during load, then measure outage and reconnect behavior. - Stage 7, lifecycle stress: call lifecycle cleanup/configure/activate while requests are in flight, then check that service callbacks and resource teardown do not race. diff --git a/ros_typedb_tools/ros_typedb_tools/ros_typedb_stress_experiment.py b/ros_typedb_tools/ros_typedb_tools/ros_typedb_stress_experiment.py index 7f0d356..3d66ed5 100644 --- a/ros_typedb_tools/ros_typedb_tools/ros_typedb_stress_experiment.py +++ b/ros_typedb_tools/ros_typedb_tools/ros_typedb_stress_experiment.py @@ -18,12 +18,16 @@ import argparse from pathlib import Path import sys +from typing import Any import rclpy from ros_typedb_tools.stress_config import INVARIANT_PROFILE_NAMES from ros_typedb_tools.stress_config import MIXED_PROFILE_NAMES from ros_typedb_tools.stress_config import QUERY_TYPE_BY_NAME +from ros_typedb_tools.stress_config import DEFAULT_TYPEDB_START_COMMAND +from ros_typedb_tools.stress_config import DEFAULT_TYPEDB_STOP_COMMAND +from ros_typedb_tools.stress_config import InvariantRecord from ros_typedb_tools.stress_config import StressExperimentResult from ros_typedb_tools.stress_output import default_timeout_output_path from ros_typedb_tools.stress_output import print_summary @@ -182,7 +186,7 @@ def build_argument_parser() -> argparse.ArgumentParser: ) parser.add_argument( '--fault', - choices=('none', 'delete-database'), + choices=('none', 'delete-database', 'restart-typedb'), default='none', help=( 'Fault to inject during the experiment. ' @@ -214,6 +218,43 @@ def build_argument_parser() -> argparse.ArgumentParser: '(e.g. /ros_typedb_interface/delete_database).' ), ) + parser.add_argument( + '--typedb-container', + help=( + 'Docker container that runs only the TypeDB server. Used with ' + '--fault restart-typedb to run docker stop/start.' + ), + ) + parser.add_argument( + '--typedb-stop-command', + default=DEFAULT_TYPEDB_STOP_COMMAND, + help=( + 'Command used with --fault restart-typedb to stop TypeDB. ' + 'Defaults to pkill -f "typedb/core/server". Ignored when ' + '--typedb-container is set.' + ), + ) + parser.add_argument( + '--typedb-start-command', + default=DEFAULT_TYPEDB_START_COMMAND, + help=( + 'Command used with --fault restart-typedb to start TypeDB. ' + 'Defaults to typedb server. Ignored when --typedb-container is ' + 'set.' + ), + ) + parser.add_argument( + '--typedb-restart-delay-s', + type=float, + default=2.0, + help='Seconds to wait between TypeDB stop and start. Defaults to 2.', + ) + parser.add_argument( + '--fault-command-timeout-s', + type=float, + default=30.0, + help='Seconds to wait for each fault command. Defaults to 30.', + ) return parser @@ -262,6 +303,46 @@ def _write_requested_outputs( print(f' timeout_output: {timeout_output_path}') +def _fault_controller_failed(fault_result: Any) -> bool: + """Return True when the configured fault action itself failed.""" + if fault_result.fault == 'restart-typedb': + return ( + fault_result.fault_triggered_at_s is not None + and ( + fault_result.fault_restart_stop_success is not True + or fault_result.fault_restart_start_success is not True + ) + ) + return False + + +def _invariant_failures_should_fail( + records: list[InvariantRecord], + fault_result: Any, +) -> bool: + """Return True when invariant failures should make the CLI fail.""" + if not any(not record.success for record in records): + return False + if ( + fault_result.fault == 'none' + or fault_result.fault_triggered_at_s is None + or fault_result.fault_recovered_at_s is None + ): + return True + + for record in records: + if record.success: + continue + if ( + record.phase in ('periodic', 'recovery') + and record.started_at_s >= fault_result.fault_triggered_at_s + and record.started_at_s <= fault_result.fault_recovered_at_s + ): + continue + return True + return False + + def main(argv: list[str] | None = None) -> int: """Run the stress experiment CLI.""" parser = build_argument_parser() @@ -283,7 +364,12 @@ def main(argv: list[str] | None = None) -> int: and fault_result.fault_recovered_at_s is None ): return 1 - if invariant_summary['failures'] > 0: + if _fault_controller_failed(fault_result): + return 1 + if _invariant_failures_should_fail( + result.invariant_records, + fault_result, + ): return 1 if args.fail_on_failure and summary['failures'] > 0: return 1 diff --git a/ros_typedb_tools/ros_typedb_tools/stress_config.py b/ros_typedb_tools/ros_typedb_tools/stress_config.py index d6c4fe1..04ea996 100644 --- a/ros_typedb_tools/ros_typedb_tools/stress_config.py +++ b/ros_typedb_tools/ros_typedb_tools/stress_config.py @@ -67,6 +67,9 @@ } MIXED_PROFILE_NAMES = ('auto', *MIXED_PROFILE_FILES) +DEFAULT_TYPEDB_STOP_COMMAND = 'pkill -f "typedb/core/server"' +DEFAULT_TYPEDB_START_COMMAND = 'typedb server' + @dataclass(frozen=True) class QuerySpec: @@ -146,6 +149,11 @@ class ResolvedStressConfig: fault: str fault_at_s: float | None fault_recovery_timeout_s: float + typedb_container: str | None = None + typedb_stop_command: str | None = DEFAULT_TYPEDB_STOP_COMMAND + typedb_start_command: str | None = DEFAULT_TYPEDB_START_COMMAND + typedb_restart_delay_s: float = 2.0 + fault_command_timeout_s: float = 30.0 @dataclass(frozen=True) @@ -160,6 +168,15 @@ class FaultResult: fault_delete_latency_s: float | None fault_recovered_at_s: float | None fault_recovery_s: float | None + fault_restart_stop_success: bool | None = None + fault_restart_stop_error: str | None = None + fault_restart_stop_latency_s: float | None = None + fault_restart_start_success: bool | None = None + fault_restart_start_error: str | None = None + fault_restart_start_latency_s: float | None = None + fault_restart_delay_s: float | None = None + fault_restart_outage_s: float | None = None + fault_observed_outage_s: float | None = None @dataclass(frozen=True) @@ -206,18 +223,18 @@ def validate_experiment_args(args: argparse.Namespace) -> None: fault = getattr(args, 'fault', 'none') fault_at_s = getattr(args, 'fault_at_s', None) fault_recovery_timeout_s = getattr(args, 'fault_recovery_timeout_s', 30.0) - if fault == 'delete-database': + if fault != 'none': if args.invariant_profile == 'none': raise ValueError( - '--fault delete-database requires --invariant-profile to be set' + f'--fault {fault} requires --invariant-profile to be set' ) if args.duration_s is None: raise ValueError( - '--fault delete-database requires --duration-s' + f'--fault {fault} requires --duration-s' ) if fault_at_s is None: raise ValueError( - '--fault delete-database requires --fault-at-s' + f'--fault {fault} requires --fault-at-s' ) if fault_at_s <= 0: raise ValueError('--fault-at-s must be greater than zero') @@ -225,6 +242,16 @@ def validate_experiment_args(args: argparse.Namespace) -> None: raise ValueError( '--fault-at-s must be less than --duration-s' ) + if fault == 'restart-typedb': + if getattr(args, 'typedb_restart_delay_s', 0.0) < 0: + raise ValueError( + '--typedb-restart-delay-s must be greater than or equal to ' + 'zero' + ) + if getattr(args, 'fault_command_timeout_s', 0.0) <= 0: + raise ValueError( + '--fault-command-timeout-s must be greater than zero' + ) if fault_recovery_timeout_s <= 0: raise ValueError( '--fault-recovery-timeout-s must be greater than zero' diff --git a/ros_typedb_tools/ros_typedb_tools/stress_output.py b/ros_typedb_tools/ros_typedb_tools/stress_output.py index 3dc3a05..650cf19 100644 --- a/ros_typedb_tools/ros_typedb_tools/stress_output.py +++ b/ros_typedb_tools/ros_typedb_tools/stress_output.py @@ -20,6 +20,7 @@ import json from pathlib import Path import statistics +import threading import time from typing import Any @@ -111,6 +112,23 @@ def _fault_payload(fault_result: FaultResult | None) -> dict[str, Any]: 'fault_delete_latency_s': fault_result.fault_delete_latency_s, 'fault_recovered_at_s': fault_result.fault_recovered_at_s, 'fault_recovery_s': fault_result.fault_recovery_s, + 'fault_restart_stop_success': ( + fault_result.fault_restart_stop_success + ), + 'fault_restart_stop_error': fault_result.fault_restart_stop_error, + 'fault_restart_stop_latency_s': ( + fault_result.fault_restart_stop_latency_s + ), + 'fault_restart_start_success': ( + fault_result.fault_restart_start_success + ), + 'fault_restart_start_error': fault_result.fault_restart_start_error, + 'fault_restart_start_latency_s': ( + fault_result.fault_restart_start_latency_s + ), + 'fault_restart_delay_s': fault_result.fault_restart_delay_s, + 'fault_restart_outage_s': fault_result.fault_restart_outage_s, + 'fault_observed_outage_s': fault_result.fault_observed_outage_s, } @@ -210,6 +228,7 @@ def __init__(self, output_path: Path | None) -> None: """Create an event writer for the optional output path.""" self._output_path = output_path self._stream = None + self._lock = threading.Lock() def __enter__(self) -> 'DebugEventWriter': """Open the event stream when debug output is enabled.""" @@ -232,7 +251,8 @@ def log(self, event: str, **fields: Any) -> None: 'monotonic_s': time.monotonic(), **fields, } - self._stream.write(json.dumps(payload, sort_keys=True) + '\n') + with self._lock: + self._stream.write(json.dumps(payload, sort_keys=True) + '\n') def print_summary( @@ -279,6 +299,32 @@ def print_summary( if fault_result.fault_recovery_s is not None else 'not recovered' ) + if fault_result.fault == 'restart-typedb': + restart_status = ( + 'ok' + if ( + fault_result.fault_restart_stop_success + and fault_result.fault_restart_start_success + ) + else 'FAILED' + ) + restart_outage = ( + f'{fault_result.fault_restart_outage_s:.3f}s' + if fault_result.fault_restart_outage_s is not None + else 'n/a' + ) + observed_outage = ( + f'{fault_result.fault_observed_outage_s:.3f}s' + if fault_result.fault_observed_outage_s is not None + else 'n/a' + ) + print( + f' fault: {fault_result.fault} triggered at ' + f'{triggered_at} | restart: {restart_status} ' + f'({restart_outage}) | observed outage: ' + f'{observed_outage} | recovery: {recovery_str}' + ) + return print( f' fault: {fault_result.fault} triggered at {triggered_at} | ' f'delete: {delete_status} ({delete_latency}) | ' diff --git a/ros_typedb_tools/ros_typedb_tools/stress_resolution.py b/ros_typedb_tools/ros_typedb_tools/stress_resolution.py index 93865a8..d754606 100644 --- a/ros_typedb_tools/ros_typedb_tools/stress_resolution.py +++ b/ros_typedb_tools/ros_typedb_tools/stress_resolution.py @@ -20,6 +20,8 @@ from ros_typedb_tools.stress_config import build_invariant_specs from ros_typedb_tools.stress_config import build_query_specs +from ros_typedb_tools.stress_config import DEFAULT_TYPEDB_START_COMMAND +from ros_typedb_tools.stress_config import DEFAULT_TYPEDB_STOP_COMMAND from ros_typedb_tools.stress_config import QuerySpec from ros_typedb_tools.stress_config import ResolvedStressConfig from ros_typedb_tools.stress_config import validate_experiment_args @@ -51,6 +53,15 @@ def resolve_stress_config( fault = getattr(args, 'fault', 'none') fault_at_s = getattr(args, 'fault_at_s', None) fault_recovery_timeout_s = getattr(args, 'fault_recovery_timeout_s', 30.0) + typedb_container = getattr(args, 'typedb_container', None) + typedb_stop_command = getattr( + args, 'typedb_stop_command', DEFAULT_TYPEDB_STOP_COMMAND + ) or DEFAULT_TYPEDB_STOP_COMMAND + typedb_start_command = getattr( + args, 'typedb_start_command', DEFAULT_TYPEDB_START_COMMAND + ) or DEFAULT_TYPEDB_START_COMMAND + typedb_restart_delay_s = getattr(args, 'typedb_restart_delay_s', 2.0) + fault_command_timeout_s = getattr(args, 'fault_command_timeout_s', 30.0) return ResolvedStressConfig( query_specs=query_specs, @@ -65,6 +76,11 @@ def resolve_stress_config( fault=fault, fault_at_s=fault_at_s, fault_recovery_timeout_s=fault_recovery_timeout_s, + typedb_container=typedb_container, + typedb_stop_command=typedb_stop_command, + typedb_start_command=typedb_start_command, + typedb_restart_delay_s=typedb_restart_delay_s, + fault_command_timeout_s=fault_command_timeout_s, ) diff --git a/ros_typedb_tools/ros_typedb_tools/stress_runner.py b/ros_typedb_tools/ros_typedb_tools/stress_runner.py index 6975c94..a6f6c99 100644 --- a/ros_typedb_tools/ros_typedb_tools/stress_runner.py +++ b/ros_typedb_tools/ros_typedb_tools/stress_runner.py @@ -17,6 +17,9 @@ import argparse from dataclasses import dataclass +import shlex +import subprocess +import threading import time from typing import Any import uuid @@ -28,6 +31,8 @@ from ros_typedb_msgs.srv import Query from ros_typedb_tools.stress_config import build_query_request +from ros_typedb_tools.stress_config import DEFAULT_TYPEDB_START_COMMAND +from ros_typedb_tools.stress_config import DEFAULT_TYPEDB_STOP_COMMAND from ros_typedb_tools.stress_config import FaultResult from ros_typedb_tools.stress_config import InvariantRecord from ros_typedb_tools.stress_config import InvariantSpec @@ -75,6 +80,15 @@ class _RunState: fault_delete_error: str | None = None fault_delete_latency_s: float | None = None fault_recovered_at_s: float | None = None + fault_restart_stop_success: bool | None = None + fault_restart_stop_error: str | None = None + fault_restart_stop_latency_s: float | None = None + fault_restart_start_success: bool | None = None + fault_restart_start_error: str | None = None + fault_restart_start_latency_s: float | None = None + fault_restart_started_at_s: float | None = None + fault_restart_ended_at_s: float | None = None + fault_restart_delay_s: float | None = None def _build_executor(args: argparse.Namespace) -> Any | None: @@ -198,7 +212,7 @@ def _start_request( state.last_started_by_client[client_id] = started_at_s state.sent_count_by_client[client_id] += 1 # Mixed workloads create experiment-owned temporary concepts. Track keys - # until a matching delete succeeds, then final cleanup can remove leftovers. + # until a matching delete succeeds. Final cleanup removes leftovers. if ( query_spec.cleanup_key is not None and query_spec.query_type != 'delete' @@ -760,6 +774,214 @@ def _trigger_delete_database_fault( ) +def _build_typedb_restart_commands( + *, + typedb_container: str | None, + typedb_stop_command: str | None, + typedb_start_command: str | None, +) -> tuple[list[str], list[str]]: + """Build stop/start command argv for a TypeDB restart fault.""" + if typedb_container: + return ( + ['docker', 'stop', typedb_container], + ['docker', 'start', typedb_container], + ) + return ( + shlex.split(typedb_stop_command or DEFAULT_TYPEDB_STOP_COMMAND), + shlex.split(typedb_start_command or DEFAULT_TYPEDB_START_COMMAND), + ) + + +def _run_fault_command( + command: list[str], + *, + timeout_s: float, +) -> tuple[bool, str | None, float]: + """Run one TypeDB restart command and return success, error, latency.""" + started_at_s = time.monotonic() + try: + completed = subprocess.run( + command, + capture_output=True, + check=False, + text=True, + timeout=timeout_s, + ) + except subprocess.TimeoutExpired: + return ( + False, + f'command timed out after {timeout_s:g}s: {command!r}', + time.monotonic() - started_at_s, + ) + except Exception as exc: # noqa: B902 + return False, str(exc), time.monotonic() - started_at_s + + if completed.returncode == 0: + return True, None, time.monotonic() - started_at_s + error = completed.stderr.strip() or completed.stdout.strip() + if not error: + error = f'command exited with status {completed.returncode}' + return False, error, time.monotonic() - started_at_s + + +def _start_fault_command( + command: list[str], + *, + observation_s: float = 1.0, +) -> tuple[bool, str | None, float]: + """Start a TypeDB process command that may keep running in foreground.""" + started_at_s = time.monotonic() + try: + process = subprocess.Popen( # noqa: S603 + command, + stdout=subprocess.DEVNULL, + stderr=subprocess.DEVNULL, + ) + except Exception as exc: # noqa: B902 + return False, str(exc), time.monotonic() - started_at_s + + try: + returncode = process.wait(timeout=observation_s) + except subprocess.TimeoutExpired: + return True, None, time.monotonic() - started_at_s + + if returncode == 0: + return True, None, time.monotonic() - started_at_s + return ( + False, + f'command exited with status {returncode}', + time.monotonic() - started_at_s, + ) + + +def _restart_typedb_fault_worker( + *, + stop_command: list[str], + start_command: list[str], + start_command_waits_for_exit: bool, + restart_delay_s: float, + command_timeout_s: float, + state: _RunState, + debug_events: DebugEventWriter, +) -> None: + """Stop and restart TypeDB while the main experiment loop continues.""" + state.fault_restart_started_at_s = time.monotonic() + state.fault_restart_delay_s = restart_delay_s + debug_events.log( + 'fault_restart_stop_started', + command=stop_command, + command_timeout_s=command_timeout_s, + ) + stop_success, stop_error, stop_latency_s = _run_fault_command( + stop_command, + timeout_s=command_timeout_s, + ) + state.fault_restart_stop_success = stop_success + state.fault_restart_stop_error = stop_error + state.fault_restart_stop_latency_s = stop_latency_s + debug_events.log( + 'fault_restart_stop_complete', + success=stop_success, + error=stop_error, + latency_s=stop_latency_s, + ) + + if restart_delay_s > 0: + time.sleep(restart_delay_s) + + debug_events.log( + 'fault_restart_start_started', + command=start_command, + command_timeout_s=command_timeout_s, + ) + if start_command_waits_for_exit: + start_success, start_error, start_latency_s = _run_fault_command( + start_command, + timeout_s=command_timeout_s, + ) + else: + start_success, start_error, start_latency_s = _start_fault_command( + start_command, + ) + state.fault_restart_start_success = start_success + state.fault_restart_start_error = start_error + state.fault_restart_start_latency_s = start_latency_s + state.fault_restart_ended_at_s = time.monotonic() + debug_events.log( + 'fault_restart_start_complete', + success=start_success, + error=start_error, + latency_s=start_latency_s, + controller_outage_s=( + state.fault_restart_ended_at_s + - state.fault_restart_started_at_s + ), + ) + + +def _trigger_restart_typedb_fault( + *, + config: Any, + state: _RunState, + debug_events: DebugEventWriter, +) -> threading.Thread: + """Start a background TypeDB restart fault controller.""" + started_at_s = time.monotonic() + pending_count = len(state.pending_by_client) + state.fault_triggered = True + state.fault_triggered_at_s = started_at_s + stop_command, start_command = _build_typedb_restart_commands( + typedb_container=config.typedb_container, + typedb_stop_command=config.typedb_stop_command, + typedb_start_command=config.typedb_start_command, + ) + start_command_waits_for_exit = config.typedb_container is not None + debug_events.log( + 'fault_triggered', + fault='restart-typedb', + triggered_at_s=started_at_s, + pending_count_at_fault=pending_count, + stop_command=stop_command, + start_command=start_command, + ) + thread = threading.Thread( + target=_restart_typedb_fault_worker, + kwargs={ + 'stop_command': stop_command, + 'start_command': start_command, + 'start_command_waits_for_exit': start_command_waits_for_exit, + 'restart_delay_s': config.typedb_restart_delay_s, + 'command_timeout_s': config.fault_command_timeout_s, + 'state': state, + 'debug_events': debug_events, + }, + daemon=True, + ) + thread.start() + return thread + + +def _compute_observed_fault_outage_s( + records: list[RequestRecord], + *, + fault_triggered_at_s: float | None, +) -> float | None: + """Measure observed outage from request failures and later success.""" + if fault_triggered_at_s is None: + return None + first_failure_ended_at_s = None + for record in sorted(records, key=lambda item: item.ended_at_s): + if record.ended_at_s < fault_triggered_at_s: + continue + if first_failure_ended_at_s is None: + if not record.success: + first_failure_ended_at_s = record.ended_at_s + continue + if record.success: + return record.ended_at_s - first_failure_ended_at_s + return None + + def _wait_for_recovery( *, node: Any, @@ -827,7 +1049,7 @@ def run_experiment(args: argparse.Namespace) -> StressExperimentResult: ) delete_db_client = ( node.create_client(Empty, delete_db_service_name) - if getattr(args, 'fault', 'none') != 'none' + if getattr(args, 'fault', 'none') == 'delete-database' else None ) deadline_s = ( @@ -850,6 +1072,7 @@ def run_experiment(args: argparse.Namespace) -> StressExperimentResult: next_snapshot_s=time.monotonic() + 1.0, next_invariant_s=time.monotonic() + args.invariant_period_s, ) + restart_fault_thread = None try: with DebugEventWriter(config.debug_events_output) as debug_events: @@ -920,7 +1143,8 @@ def run_experiment(args: argparse.Namespace) -> StressExperimentResult: debug_events=debug_events, ) if ( - delete_db_client is not None + config.fault == 'delete-database' + and delete_db_client is not None and not state.fault_triggered and config.fault_at_s is not None and time.monotonic() - experiment_started_at_s @@ -934,9 +1158,24 @@ def run_experiment(args: argparse.Namespace) -> StressExperimentResult: timeout_s=args.timeout_s, debug_events=debug_events, ) + if ( + config.fault == 'restart-typedb' + and not state.fault_triggered + and config.fault_at_s is not None + and time.monotonic() - experiment_started_at_s + >= config.fault_at_s + ): + restart_fault_thread = _trigger_restart_typedb_fault( + config=config, + state=state, + debug_events=debug_events, + ) + + if restart_fault_thread is not None: + restart_fault_thread.join() - # 1. Recovery (must come before cleanup — cleanup queries trigger DB - # recreation, which would silently advance recovery unobserved) + # 1. Recovery must come before cleanup. Cleanup queries trigger DB + # recreation, silently advancing recovery unobserved. if ( state.fault_triggered and config.invariant_specs @@ -1001,6 +1240,12 @@ def run_experiment(args: argparse.Namespace) -> StressExperimentResult: and state.fault_recovered_at_s is not None else None ) + fault_restart_outage_s = ( + state.fault_restart_ended_at_s - state.fault_restart_started_at_s + if state.fault_restart_started_at_s is not None + and state.fault_restart_ended_at_s is not None + else None + ) fault_result = FaultResult( fault=config.fault, fault_at_s=config.fault_at_s, @@ -1010,6 +1255,18 @@ def run_experiment(args: argparse.Namespace) -> StressExperimentResult: fault_delete_latency_s=state.fault_delete_latency_s, fault_recovered_at_s=state.fault_recovered_at_s, fault_recovery_s=fault_recovery_s, + fault_restart_stop_success=state.fault_restart_stop_success, + fault_restart_stop_error=state.fault_restart_stop_error, + fault_restart_stop_latency_s=state.fault_restart_stop_latency_s, + fault_restart_start_success=state.fault_restart_start_success, + fault_restart_start_error=state.fault_restart_start_error, + fault_restart_start_latency_s=state.fault_restart_start_latency_s, + fault_restart_delay_s=state.fault_restart_delay_s, + fault_restart_outage_s=fault_restart_outage_s, + fault_observed_outage_s=_compute_observed_fault_outage_s( + state.records, + fault_triggered_at_s=state.fault_triggered_at_s, + ), ) return StressExperimentResult( records=state.records, diff --git a/ros_typedb_tools/test/test_ros_typedb_stress_experiment.py b/ros_typedb_tools/test/test_ros_typedb_stress_experiment.py index 48f442f..41f1f6b 100644 --- a/ros_typedb_tools/test/test_ros_typedb_stress_experiment.py +++ b/ros_typedb_tools/test/test_ros_typedb_stress_experiment.py @@ -2,6 +2,7 @@ import json from pathlib import Path +import subprocess from types import SimpleNamespace import pytest @@ -21,6 +22,8 @@ from ros_typedb_tools.stress_config import ( build_invariant_specs, build_query_specs, + DEFAULT_TYPEDB_START_COMMAND, + DEFAULT_TYPEDB_STOP_COMMAND, FaultResult, InvariantRecord, InvariantSpec, @@ -45,7 +48,11 @@ ) from ros_typedb_tools.stress_resolution import resolve_stress_config from ros_typedb_tools.stress_runner import ( + _build_typedb_restart_commands, + _compute_observed_fault_outage_s, _mark_fault_recovered_from_invariant_pass, + _run_fault_command, + _start_fault_command, ) from ros_typedb_tools.stress_runner import _RunState from ros_typedb_tools.stress_runner import run_experiment @@ -80,15 +87,18 @@ def _invariant_record( *, success: bool, timed_out: bool = False, + phase: str = 'final', + started_at_s: float | None = None, ) -> InvariantRecord: + started_at_s = 20.0 + index if started_at_s is None else started_at_s return InvariantRecord( index=index, - phase='final', + phase=phase, name='person-count', query_type='get_aggregate', query='match $p isa person; get $p; count;', - started_at_s=20.0 + index, - ended_at_s=20.1 + index, + started_at_s=started_at_s, + ended_at_s=started_at_s + 0.1, latency_s=0.1, success=success, timed_out=timed_out, @@ -260,6 +270,42 @@ def test_parser_accepts_stage_four_mixed_mode(): assert args.invariant_profile == 'plan-schema-mixed' +def test_parser_accepts_stage_six_restart_fault_arguments(): + """Check Stage 6 TypeDB restart fault arguments.""" + parser = build_argument_parser() + + args = parser.parse_args( + [ + '--clients', + '10', + '--duration-s', + '240', + '--mode', + 'read', + '--fault', + 'restart-typedb', + '--typedb-container', + 'typedb_server', + '--fault-at-s', + '60', + '--typedb-restart-delay-s', + '3', + '--fault-command-timeout-s', + '15', + '--invariant-profile', + 'test-data', + ] + ) + + assert args.fault == 'restart-typedb' + assert args.typedb_container == 'typedb_server' + assert args.typedb_stop_command == DEFAULT_TYPEDB_STOP_COMMAND + assert args.typedb_start_command == DEFAULT_TYPEDB_START_COMMAND + assert args.fault_at_s == 60 + assert args.typedb_restart_delay_s == 3 + assert args.fault_command_timeout_s == 15 + + def test_fake_service_parser_accepts_diagnostic_arguments(): """Check the fake service CLI options.""" parser = build_fake_service_argument_parser() @@ -446,6 +492,15 @@ def test_write_results_writes_summary_and_request_records(tmp_path: Path): 'fault_delete_latency_s': 0.125, 'fault_recovered_at_s': 12.75, 'fault_recovery_s': 2.5, + 'fault_restart_stop_success': None, + 'fault_restart_stop_error': None, + 'fault_restart_stop_latency_s': None, + 'fault_restart_start_success': None, + 'fault_restart_start_error': None, + 'fault_restart_start_latency_s': None, + 'fault_restart_delay_s': None, + 'fault_restart_outage_s': None, + 'fault_observed_outage_s': None, } @@ -472,9 +527,128 @@ def test_write_results_writes_default_fault_payload(tmp_path: Path): 'fault_delete_latency_s': None, 'fault_recovered_at_s': None, 'fault_recovery_s': None, + 'fault_restart_stop_success': None, + 'fault_restart_stop_error': None, + 'fault_restart_stop_latency_s': None, + 'fault_restart_start_success': None, + 'fault_restart_start_error': None, + 'fault_restart_start_latency_s': None, + 'fault_restart_delay_s': None, + 'fault_restart_outage_s': None, + 'fault_observed_outage_s': None, } +def test_build_typedb_restart_commands_uses_container(): + """Check container restart commands override process hooks.""" + stop_command, start_command = _build_typedb_restart_commands( + typedb_container='typedb_server', + typedb_stop_command='custom stop', + typedb_start_command='custom start', + ) + + assert stop_command == ['docker', 'stop', 'typedb_server'] + assert start_command == ['docker', 'start', 'typedb_server'] + + +def test_build_typedb_restart_commands_uses_default_process_hooks(): + """Check default process restart commands.""" + stop_command, start_command = _build_typedb_restart_commands( + typedb_container=None, + typedb_stop_command=None, + typedb_start_command=None, + ) + + assert stop_command == ['pkill', '-f', 'typedb/core/server'] + assert start_command == ['typedb', 'server'] + + +def test_build_typedb_restart_commands_uses_command_hooks(): + """Check restart command hooks are parsed into argv.""" + stop_command, start_command = _build_typedb_restart_commands( + typedb_container=None, + typedb_stop_command='bash -lc "typedb server stop"', + typedb_start_command='bash -lc "typedb server start"', + ) + + assert stop_command == ['bash', '-lc', 'typedb server stop'] + assert start_command == ['bash', '-lc', 'typedb server start'] + + +def test_run_fault_command_reports_nonzero_exit(monkeypatch): + """Check fault command execution reports stderr on failure.""" + class Completed: + returncode = 2 + stdout = '' + stderr = 'failed to stop' + + def fake_run(command, **kwargs): + assert command == ['typedb', 'server', 'stop'] + assert kwargs['timeout'] == 5.0 + return Completed() + + monkeypatch.setattr( + 'ros_typedb_tools.stress_runner.subprocess.run', + fake_run, + ) + + success, error, latency_s = _run_fault_command( + ['typedb', 'server', 'stop'], + timeout_s=5.0, + ) + + assert success is False + assert error == 'failed to stop' + assert latency_s >= 0.0 + + +def test_start_fault_command_treats_running_process_as_success(monkeypatch): + """Check foreground server commands succeed once they stay running.""" + class RunningProcess: + def wait(self, timeout): + assert timeout == 0.25 + raise subprocess.TimeoutExpired(['typedb', 'server'], timeout) + + def fake_popen(command, **kwargs): + assert command == ['typedb', 'server'] + assert kwargs['stdout'] == subprocess.DEVNULL + assert kwargs['stderr'] == subprocess.DEVNULL + return RunningProcess() + + monkeypatch.setattr( + 'ros_typedb_tools.stress_runner.subprocess.Popen', + fake_popen, + ) + + success, error, latency_s = _start_fault_command( + ['typedb', 'server'], + observation_s=0.25, + ) + + assert success is True + assert error is None + assert latency_s >= 0.0 + + +def test_compute_observed_fault_outage_s_uses_failure_to_success_window(): + """Check observed outage is measured from failure to later success.""" + records = [ + _record(0, latency_s=0.1, success=True), + _record(1, latency_s=0.2, success=False), + _record(2, latency_s=0.4, success=False, timed_out=True), + _record(3, latency_s=0.3, success=True), + ] + + outage_s = _compute_observed_fault_outage_s( + records, + fault_triggered_at_s=10.5, + ) + + assert outage_s == pytest.approx( + records[3].ended_at_s - records[1].ended_at_s + ) + + def test_default_timeout_output_path_uses_result_stem(): """Check timeout output path is derived from the main output path.""" assert default_timeout_output_path( @@ -864,6 +1038,11 @@ def _base_args(**overrides): 'fault': 'none', 'fault_at_s': None, 'fault_recovery_timeout_s': 30.0, + 'typedb_container': None, + 'typedb_stop_command': DEFAULT_TYPEDB_STOP_COMMAND, + 'typedb_start_command': DEFAULT_TYPEDB_START_COMMAND, + 'typedb_restart_delay_s': 2.0, + 'fault_command_timeout_s': 30.0, } base.update(overrides) return SimpleNamespace(**base) @@ -920,6 +1099,46 @@ def test_validate_args_fault_at_s_must_be_less_than_duration_s(): validate_experiment_args(args) +def test_validate_args_fault_restart_typedb_accepts_default_controller(): + """Check restart-typedb accepts default process commands.""" + args = _base_args( + fault='restart-typedb', + invariant_profile='test-data', + duration_s=60.0, + fault_at_s=10.0, + ) + + validate_experiment_args(args) + + +def test_validate_args_fault_restart_typedb_accepts_container_override(): + """Check restart-typedb accepts container overriding process commands.""" + args = _base_args( + fault='restart-typedb', + invariant_profile='test-data', + duration_s=60.0, + fault_at_s=10.0, + typedb_container='typedb_server', + typedb_stop_command='typedb server stop', + typedb_start_command='typedb server start', + ) + + validate_experiment_args(args) + + +def test_validate_args_fault_restart_typedb_accepts_container(): + """Check restart-typedb accepts a TypeDB container controller.""" + args = _base_args( + fault='restart-typedb', + invariant_profile='test-data', + duration_s=60.0, + fault_at_s=10.0, + typedb_container='typedb_server', + ) + + validate_experiment_args(args) + + def test_validate_args_fault_recovery_timeout_s_must_be_positive(): """Check --fault-recovery-timeout-s must be greater than zero.""" args = _base_args(fault_recovery_timeout_s=0.0) @@ -970,3 +1189,149 @@ def test_main_fails_when_triggered_fault_does_not_recover(monkeypatch): monkeypatch.setattr(stress_cli, 'run_experiment', lambda args: result) assert stress_cli.main([]) == 1 + + +def test_main_fails_when_restart_fault_controller_fails(monkeypatch): + """Check CLI exits nonzero when restart stop/start fails.""" + fault_result = FaultResult( + fault='restart-typedb', + fault_at_s=10.0, + fault_triggered_at_s=10.0, + fault_delete_success=None, + fault_delete_error=None, + fault_delete_latency_s=None, + fault_recovered_at_s=12.0, + fault_recovery_s=2.0, + fault_restart_stop_success=False, + fault_restart_stop_error='failed to stop', + fault_restart_start_success=True, + ) + result = StressExperimentResult( + records=[], + invariant_records=[], + config=ResolvedStressConfig( + query_specs=[], + invariant_specs=[], + mixed_profile=None, + mixed_profile_name=None, + invariant_profile_name='test-data', + max_in_flight=1, + debug_events_output=None, + fault='restart-typedb', + fault_at_s=10.0, + fault_recovery_timeout_s=30.0, + ), + fault_result=fault_result, + ) + + monkeypatch.setattr(stress_cli.rclpy, 'init', lambda args=None: None) + monkeypatch.setattr(stress_cli.rclpy, 'shutdown', lambda: None) + monkeypatch.setattr(stress_cli, 'run_experiment', lambda args: result) + + assert stress_cli.main([]) == 1 + + +def test_main_allows_periodic_invariant_failures_during_recovered_fault( + monkeypatch, +): + """Check recovered fault runs tolerate outage-window invariant failures.""" + fault_result = FaultResult( + fault='restart-typedb', + fault_at_s=10.0, + fault_triggered_at_s=10.0, + fault_delete_success=None, + fault_delete_error=None, + fault_delete_latency_s=None, + fault_recovered_at_s=20.0, + fault_recovery_s=10.0, + fault_restart_stop_success=True, + fault_restart_start_success=True, + ) + result = StressExperimentResult( + records=[], + invariant_records=[ + _invariant_record( + 0, + success=False, + phase='periodic', + started_at_s=12.0, + ), + _invariant_record( + 1, + success=False, + phase='recovery', + started_at_s=15.0, + ), + _invariant_record( + 2, + success=True, + phase='final', + started_at_s=21.0, + ), + ], + config=ResolvedStressConfig( + query_specs=[], + invariant_specs=[], + mixed_profile=None, + mixed_profile_name=None, + invariant_profile_name='test-data', + max_in_flight=1, + debug_events_output=None, + fault='restart-typedb', + fault_at_s=10.0, + fault_recovery_timeout_s=30.0, + ), + fault_result=fault_result, + ) + + monkeypatch.setattr(stress_cli.rclpy, 'init', lambda args=None: None) + monkeypatch.setattr(stress_cli.rclpy, 'shutdown', lambda: None) + monkeypatch.setattr(stress_cli, 'run_experiment', lambda args: result) + + assert stress_cli.main([]) == 0 + + +def test_main_fails_final_invariant_failure_after_recovered_fault(monkeypatch): + """Check final invariant failures still fail recovered fault runs.""" + fault_result = FaultResult( + fault='restart-typedb', + fault_at_s=10.0, + fault_triggered_at_s=10.0, + fault_delete_success=None, + fault_delete_error=None, + fault_delete_latency_s=None, + fault_recovered_at_s=20.0, + fault_recovery_s=10.0, + fault_restart_stop_success=True, + fault_restart_start_success=True, + ) + result = StressExperimentResult( + records=[], + invariant_records=[ + _invariant_record( + 0, + success=False, + phase='final', + started_at_s=21.0, + ), + ], + config=ResolvedStressConfig( + query_specs=[], + invariant_specs=[], + mixed_profile=None, + mixed_profile_name=None, + invariant_profile_name='test-data', + max_in_flight=1, + debug_events_output=None, + fault='restart-typedb', + fault_at_s=10.0, + fault_recovery_timeout_s=30.0, + ), + fault_result=fault_result, + ) + + monkeypatch.setattr(stress_cli.rclpy, 'init', lambda args=None: None) + monkeypatch.setattr(stress_cli.rclpy, 'shutdown', lambda: None) + monkeypatch.setattr(stress_cli, 'run_experiment', lambda args: result) + + assert stress_cli.main([]) == 1 From 7a3982d6821ff130432f09dfad2ff7930c55bcba Mon Sep 17 00:00:00 2001 From: Gustavo Date: Sun, 28 Jun 2026 19:05:15 -0300 Subject: [PATCH 26/36] ros_typedb: gate services by lifecycle state --- ros_typedb/ros_typedb/ros_typedb_interface.py | 58 ++++++--- .../test/test_ros_typedb_interface_unit.py | 117 +++++++++++++++--- 2 files changed, 143 insertions(+), 32 deletions(-) diff --git a/ros_typedb/ros_typedb/ros_typedb_interface.py b/ros_typedb/ros_typedb/ros_typedb_interface.py index 413d871..95666f8 100644 --- a/ros_typedb/ros_typedb/ros_typedb_interface.py +++ b/ros_typedb/ros_typedb/ros_typedb_interface.py @@ -13,6 +13,7 @@ # limitations under the License. """ros_typedb_interface - python interface to interact with typedb via ROS.""" +from functools import wraps import threading import traceback @@ -503,6 +504,20 @@ def query_result_to_ros_msg( return response +def lifecycle_state_is_active(node): + """Return True when the lifecycle node is in the active state.""" + return node._state_machine.current_state[1] == 'active' + + +def when_lifecycle_active(func): + """Decorate a method to run only when its lifecycle node is active.""" + @wraps(func) + def inner(self, *args, **kwargs): + if lifecycle_state_is_active(self): + return func(self, *args, **kwargs) + return inner + + class ROSTypeDBInterface(Node): """ROS lifecycle node to interact with typedb.""" @@ -529,6 +544,18 @@ def __init__(self, node_name: str, **kwargs): self.query_cb_group = MutuallyExclusiveCallbackGroup() self._service_callback_lock = threading.Lock() + self.query_service = self.create_service( + Query, + self.get_name() + '/query', + self.query_service_cb, + callback_group=self.query_cb_group) + + self.delete_db_service = self.create_service( + Empty, + self.get_name() + '/delete_database', + self.delete_db_cb, + callback_group=self.query_cb_group) + def _get_service_callback_lock(self): """Return the lock used to serialize service callbacks with cleanup.""" lock = getattr(self, '_service_callback_lock', None) @@ -595,6 +622,7 @@ def close_typedb_interface(self) -> None: if typedb_interface is not None: self.typedb_interface = None + @when_lifecycle_active def publish_data_event(self, event_type: str) -> None: """ Publish message in the `/event` topic. @@ -643,18 +671,6 @@ def on_configure(self, state: State) -> TransitionCallbackReturn: self.get_name() + '/events', 10, callback_group=ReentrantCallbackGroup()) - - self.query_service = self.create_service( - Query, - self.get_name() + '/query', - self.query_service_cb, - callback_group=self.query_cb_group) - - self.delete_db_service = self.create_service( - Empty, - self.get_name() + '/delete_database', - self.delete_db_cb, - callback_group=self.query_cb_group) except Exception as exc: try: self.close_typedb_interface() @@ -677,12 +693,6 @@ def on_cleanup(self, state: State) -> TransitionCallbackReturn: :return: transition result """ - for service_name in ('query_service', 'delete_db_service'): - service = getattr(self, service_name, None) - if service is not None: - self.destroy_service(service) - delattr(self, service_name) - service_callback_lock = self._get_service_callback_lock() with service_callback_lock: if getattr(self, 'event_pub', None) is not None: @@ -710,6 +720,14 @@ def query_service_cb( """ service_callback_lock = self._get_service_callback_lock() with service_callback_lock: + if not lifecycle_state_is_active(self): + response.success = False + response.error_message = 'Node is not active' + return response + if getattr(self, 'typedb_interface', None) is None: + response.success = False + response.error_message = 'Node is not configured' + return response if req.query_type == Query.Request.INSERT: query_func = self.typedb_interface.insert_database elif req.query_type == Query.Request.DELETE: @@ -766,5 +784,9 @@ def delete_db_cb( """ service_callback_lock = self._get_service_callback_lock() with service_callback_lock: + if not lifecycle_state_is_active(self): + return response + if getattr(self, 'typedb_interface', None) is None: + return response self.typedb_interface.delete_database() return response diff --git a/ros_typedb/test/test_ros_typedb_interface_unit.py b/ros_typedb/test/test_ros_typedb_interface_unit.py index cae0c8a..fbc43ad 100644 --- a/ros_typedb/test/test_ros_typedb_interface_unit.py +++ b/ros_typedb/test/test_ros_typedb_interface_unit.py @@ -95,6 +95,7 @@ def fetch_database(self, query, timeout=None): node = ROSTypeDBInterface.__new__(ROSTypeDBInterface) node.typedb_interface = FakeTypeDBInterface() + node._state_machine = SimpleNamespace(current_state=(1, 'active')) req = Query.Request() req.query_type = Query.Request.FETCH @@ -120,6 +121,7 @@ def fetch_database(self, query, timeout=None): node = ROSTypeDBInterface.__new__(ROSTypeDBInterface) node.typedb_interface = FakeTypeDBInterface() + node._state_machine = SimpleNamespace(current_state=(1, 'active')) req = Query.Request() req.query_type = Query.Request.FETCH @@ -146,6 +148,7 @@ def raise_conversion_error(query_type, query_result): node = ROSTypeDBInterface.__new__(ROSTypeDBInterface) node.typedb_interface = FakeTypeDBInterface() + node._state_machine = SimpleNamespace(current_state=(1, 'active')) logger = MagicMock() node.get_logger = MagicMock(return_value=logger) monkeypatch.setattr( @@ -224,21 +227,16 @@ def test_close_typedb_interface_closes_driver_and_clears_interface(): assert node.typedb_interface is None -def test_on_cleanup_destroys_ros_entities_and_closes_typedb_driver(): +def test_on_cleanup_destroys_publisher_and_closes_typedb_driver(): driver = MagicMock() typedb_interface = SimpleNamespace(driver=driver) event_pub = MagicMock() - query_service = MagicMock() - delete_db_service = MagicMock() node = SimpleNamespace( event_pub=event_pub, - query_service=query_service, - delete_db_service=delete_db_service, typedb_interface=typedb_interface, destroy_publisher=MagicMock(return_value=True), - destroy_service=MagicMock(return_value=True), get_logger=MagicMock(return_value=MagicMock()), get_name=MagicMock(return_value='ros_typedb'), close_typedb_interface=MagicMock( @@ -252,14 +250,9 @@ def test_on_cleanup_destroys_ros_entities_and_closes_typedb_driver(): assert result == TransitionCallbackReturn.SUCCESS node.destroy_publisher.assert_called_once_with(event_pub) - node.destroy_service.assert_any_call(query_service) - node.destroy_service.assert_any_call(delete_db_service) - assert node.destroy_service.call_count == 2 assert node.close_typedb_interface.call_count == 1 driver.close.assert_called_once_with() assert not hasattr(node, 'event_pub') - assert not hasattr(node, 'query_service') - assert not hasattr(node, 'delete_db_service') assert node.typedb_interface is None @@ -271,12 +264,9 @@ def test_on_cleanup_waits_for_active_service_callback_before_closing_driver(): node = SimpleNamespace( event_pub=MagicMock(), - query_service=MagicMock(), - delete_db_service=MagicMock(), typedb_interface=typedb_interface, _service_callback_lock=service_callback_lock, destroy_publisher=MagicMock(return_value=True), - destroy_service=MagicMock(return_value=True), get_logger=MagicMock(return_value=MagicMock()), get_name=MagicMock(return_value='ros_typedb'), close_typedb_interface=MagicMock( @@ -870,3 +860,102 @@ def test_fetch_result_to_ros_result_tree(): assert result_tree.results[6] == employee_var_thing3_result assert result_tree.results[7] == employment_var_thing3_result assert expected_tree == result_tree + + +def test_lifecycle_state_is_active_returns_true_when_active(): + """lifecycle_state_is_active returns True when state label is 'active'.""" + node = SimpleNamespace( + _state_machine=SimpleNamespace(current_state=(1, 'active'))) + assert ros_typedb_interface.lifecycle_state_is_active(node) is True + + +def test_lifecycle_state_is_active_returns_false_for_non_active_states(): + """lifecycle_state_is_active returns False for non-active states.""" + for label in ('inactive', 'unconfigured', 'finalized'): + node = SimpleNamespace( + _state_machine=SimpleNamespace(current_state=(1, label))) + assert ros_typedb_interface.lifecycle_state_is_active(node) is False + + +def test_when_lifecycle_active_skips_call_when_not_active(): + """when_lifecycle_active returns None without calling func when inactive.""" + called = [] + + @ros_typedb_interface.when_lifecycle_active + def example_method(self): + """Return an example result.""" + called.append(True) + return 'result' + + node = SimpleNamespace( + _state_machine=SimpleNamespace(current_state=(1, 'inactive'))) + result = example_method(node) + + assert result is None + assert called == [] + + +def test_query_service_cb_returns_not_active_when_node_is_inactive(): + """query_service_cb returns success=False when node is not active.""" + node = ROSTypeDBInterface.__new__(ROSTypeDBInterface) + node._state_machine = SimpleNamespace(current_state=(1, 'inactive')) + node.typedb_interface = MagicMock() + + req = Query.Request() + req.query_type = Query.Request.FETCH + req.query = 'match $x isa thing; fetch $x;' + + response = Query.Response() + result = node.query_service_cb(req, response) + + assert result.success is False + assert 'not active' in result.error_message + + +def test_query_service_cb_returns_not_configured_when_typedb_interface_is_none(): + """query_service_cb returns success=False when typedb_interface is None.""" + node = ROSTypeDBInterface.__new__(ROSTypeDBInterface) + node._state_machine = SimpleNamespace(current_state=(1, 'active')) + node.typedb_interface = None + + req = Query.Request() + req.query_type = Query.Request.FETCH + req.query = 'match $x isa thing; fetch $x;' + + response = Query.Response() + result = node.query_service_cb(req, response) + + assert result.success is False + assert 'not configured' in result.error_message + + +def test_delete_db_cb_skips_deletion_when_node_is_inactive(): + """delete_db_cb skips database deletion when node is not active.""" + from std_srvs.srv import Empty + + node = ROSTypeDBInterface.__new__(ROSTypeDBInterface) + node._state_machine = SimpleNamespace(current_state=(1, 'inactive')) + typedb_mock = MagicMock() + node.typedb_interface = typedb_mock + + result = ROSTypeDBInterface.delete_db_cb( + node, Empty.Request(), Empty.Response()) + + assert result is not None + typedb_mock.delete_database.assert_not_called() + + +def test_delete_db_cb_skips_deletion_when_typedb_interface_is_none(): + """ + Check delete_db_cb skips deletion when typedb_interface is None. + + If the guard were absent, None.delete_database() would raise AttributeError. + """ + from std_srvs.srv import Empty + + node = ROSTypeDBInterface.__new__(ROSTypeDBInterface) + node._state_machine = SimpleNamespace(current_state=(1, 'active')) + node.typedb_interface = None + + ROSTypeDBInterface.delete_db_cb( + node, Empty.Request(), Empty.Response()) From 95fe3394c0eaa04daa9955f628d509a2304317da Mon Sep 17 00:00:00 2001 From: Gustavo Date: Sun, 28 Jun 2026 19:05:40 -0300 Subject: [PATCH 27/36] ros_typedb: control lifecycle reactivation in launch --- ros_typedb/launch/ros_typedb.launch.py | 34 ++++++++++++++++++++++++-- 1 file changed, 32 insertions(+), 2 deletions(-) diff --git a/ros_typedb/launch/ros_typedb.launch.py b/ros_typedb/launch/ros_typedb.launch.py index c8f238e..57f8269 100644 --- a/ros_typedb/launch/ros_typedb.launch.py +++ b/ros_typedb/launch/ros_typedb.launch.py @@ -15,6 +15,7 @@ from launch.actions import DeclareLaunchArgument from launch.actions import EmitEvent from launch.actions import RegisterEventHandler +from launch.conditions import IfCondition from launch.events import matches_action from launch.substitutions import LaunchConfiguration from launch_ros.actions import LifecycleNode @@ -31,6 +32,7 @@ def generate_launch_description(): force_data = LaunchConfiguration('force_data') force_database = LaunchConfiguration('force_database') infer = LaunchConfiguration('infer') + reactivate_on_deactivate = LaunchConfiguration('reactivate_on_deactivate') schema_path_arg = DeclareLaunchArgument( 'schema_path', @@ -74,6 +76,12 @@ def generate_launch_description(): description='use inference engine' ) + reactivate_on_deactivate_arg = DeclareLaunchArgument( + 'reactivate_on_deactivate', + default_value='True', + description='reactivate the lifecycle node after deactivate' + ) + ros_typedb_node = LifecycleNode( package='ros_typedb', executable='ros_typedb', @@ -101,16 +109,36 @@ def generate_launch_description(): ros_typedb_node_activate_event = RegisterEventHandler( OnStateTransition( target_lifecycle_node=ros_typedb_node, + start_state='configuring', goal_state='inactive', entities=[ EmitEvent(event=ChangeState( lifecycle_node_matcher=matches_action(ros_typedb_node), - transition_id=lifecycle_msgs.msg.Transition.TRANSITION_ACTIVATE, + transition_id=( + lifecycle_msgs.msg.Transition.TRANSITION_ACTIVATE + ), )), ], ) ) + ros_typedb_node_reactivate_event = RegisterEventHandler( + OnStateTransition( + target_lifecycle_node=ros_typedb_node, + start_state='deactivating', + goal_state='inactive', + entities=[ + EmitEvent(event=ChangeState( + lifecycle_node_matcher=matches_action(ros_typedb_node), + transition_id=( + lifecycle_msgs.msg.Transition.TRANSITION_ACTIVATE + ), + )), + ], + ), + condition=IfCondition(reactivate_on_deactivate), + ) + return LaunchDescription([ schema_path_arg, data_path_arg, @@ -119,7 +147,9 @@ def generate_launch_description(): force_data_arg, force_database_arg, infer_arg, + reactivate_on_deactivate_arg, ros_typedb_node, ros_typedb_node_config_event, - ros_typedb_node_activate_event + ros_typedb_node_activate_event, + ros_typedb_node_reactivate_event, ]) From fcb7ccaa1e786b179c4fc799a95c30f246a671dc Mon Sep 17 00:00:00 2001 From: Gustavo Date: Sun, 28 Jun 2026 19:06:04 -0300 Subject: [PATCH 28/36] ros_typedb: log TypeDB driver connection --- ros_typedb/ros_typedb/typedb_interface.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/ros_typedb/ros_typedb/typedb_interface.py b/ros_typedb/ros_typedb/typedb_interface.py index 4308898..acdf426 100644 --- a/ros_typedb/ros_typedb/typedb_interface.py +++ b/ros_typedb/ros_typedb/typedb_interface.py @@ -411,6 +411,8 @@ def close_late_driver(): succeeded, result = result_queue.get_nowait() if succeeded: + self.logger.warning( + 'Connected TypeDB driver to %s', self._address) self.driver = result else: raise result From 3156a257667913dfe00d643c01ab5ecf4e786585 Mon Sep 17 00:00:00 2001 From: Gustavo Date: Sun, 28 Jun 2026 19:12:56 -0300 Subject: [PATCH 29/36] update ros_typedb_examples --- ros_typedb_examples/data/plan_data.tql | 1 + .../launch/plan_example.launch.py | 28 +++++++++++++++++++ .../launch/test_data_example.launch.py | 28 +++++++++++++++++++ ros_typedb_examples/setup.py | 13 ++++++++- 4 files changed, 69 insertions(+), 1 deletion(-) diff --git a/ros_typedb_examples/data/plan_data.tql b/ros_typedb_examples/data/plan_data.tql index 31051bc..72f1adf 100644 --- a/ros_typedb_examples/data/plan_data.tql +++ b/ros_typedb_examples/data/plan_data.tql @@ -20,6 +20,7 @@ $inspect_hull isa Action, $collect_water_sample isa Action, has action_name "collect-water-sample", + has action_name "dual_name", has action_duration 240.0, has start_time 2026-06-26T10:05:00.000; diff --git a/ros_typedb_examples/launch/plan_example.launch.py b/ros_typedb_examples/launch/plan_example.launch.py index 0a6569f..3aa3348 100644 --- a/ros_typedb_examples/launch/plan_example.launch.py +++ b/ros_typedb_examples/launch/plan_example.launch.py @@ -21,6 +21,7 @@ from launch.actions import DeclareLaunchArgument from launch.actions import EmitEvent from launch.actions import RegisterEventHandler +from launch.conditions import IfCondition from launch.events import matches_action from launch.substitutions import LaunchConfiguration @@ -52,6 +53,7 @@ def generate_launch_description(): reload_schema = LaunchConfiguration('reload_schema') infer = LaunchConfiguration('infer') query_timeout_s = LaunchConfiguration('query_timeout_s') + reactivate_on_deactivate = LaunchConfiguration('reactivate_on_deactivate') database_name_arg = DeclareLaunchArgument( 'database_name', @@ -95,6 +97,12 @@ def generate_launch_description(): description='default TypeDB query timeout in seconds', ) + reactivate_on_deactivate_arg = DeclareLaunchArgument( + 'reactivate_on_deactivate', + default_value='True', + description='reactivate the lifecycle node after deactivate', + ) + ros_typedb_node = LifecycleNode( package='ros_typedb', executable='ros_typedb', @@ -124,6 +132,23 @@ def generate_launch_description(): activate_event = RegisterEventHandler( OnStateTransition( target_lifecycle_node=ros_typedb_node, + start_state='configuring', + goal_state='inactive', + entities=[ + EmitEvent(event=ChangeState( + lifecycle_node_matcher=matches_action(ros_typedb_node), + transition_id=( + lifecycle_msgs.msg.Transition.TRANSITION_ACTIVATE + ), + )), + ], + ), + ) + + reactivate_event = RegisterEventHandler( + OnStateTransition( + target_lifecycle_node=ros_typedb_node, + start_state='deactivating', goal_state='inactive', entities=[ EmitEvent(event=ChangeState( @@ -134,6 +159,7 @@ def generate_launch_description(): )), ], ), + condition=IfCondition(reactivate_on_deactivate), ) return LaunchDescription([ @@ -144,7 +170,9 @@ def generate_launch_description(): reload_schema_arg, infer_arg, query_timeout_s_arg, + reactivate_on_deactivate_arg, ros_typedb_node, configure_event, activate_event, + reactivate_event, ]) diff --git a/ros_typedb_examples/launch/test_data_example.launch.py b/ros_typedb_examples/launch/test_data_example.launch.py index 7489a74..e555614 100644 --- a/ros_typedb_examples/launch/test_data_example.launch.py +++ b/ros_typedb_examples/launch/test_data_example.launch.py @@ -21,6 +21,7 @@ from launch.actions import DeclareLaunchArgument from launch.actions import EmitEvent from launch.actions import RegisterEventHandler +from launch.conditions import IfCondition from launch.events import matches_action from launch.substitutions import LaunchConfiguration @@ -54,6 +55,7 @@ def generate_launch_description(): reload_schema = LaunchConfiguration('reload_schema') infer = LaunchConfiguration('infer') query_timeout_s = LaunchConfiguration('query_timeout_s') + reactivate_on_deactivate = LaunchConfiguration('reactivate_on_deactivate') database_name_arg = DeclareLaunchArgument( 'database_name', @@ -97,6 +99,12 @@ def generate_launch_description(): description='default TypeDB query timeout in seconds', ) + reactivate_on_deactivate_arg = DeclareLaunchArgument( + 'reactivate_on_deactivate', + default_value='True', + description='reactivate the lifecycle node after deactivate', + ) + ros_typedb_node = LifecycleNode( package='ros_typedb', executable='ros_typedb', @@ -126,6 +134,23 @@ def generate_launch_description(): activate_event = RegisterEventHandler( OnStateTransition( target_lifecycle_node=ros_typedb_node, + start_state='configuring', + goal_state='inactive', + entities=[ + EmitEvent(event=ChangeState( + lifecycle_node_matcher=matches_action(ros_typedb_node), + transition_id=( + lifecycle_msgs.msg.Transition.TRANSITION_ACTIVATE + ), + )), + ], + ), + ) + + reactivate_event = RegisterEventHandler( + OnStateTransition( + target_lifecycle_node=ros_typedb_node, + start_state='deactivating', goal_state='inactive', entities=[ EmitEvent(event=ChangeState( @@ -136,6 +161,7 @@ def generate_launch_description(): )), ], ), + condition=IfCondition(reactivate_on_deactivate), ) return LaunchDescription([ @@ -146,7 +172,9 @@ def generate_launch_description(): reload_schema_arg, infer_arg, query_timeout_s_arg, + reactivate_on_deactivate_arg, ros_typedb_node, configure_event, activate_event, + reactivate_event, ]) diff --git a/ros_typedb_examples/setup.py b/ros_typedb_examples/setup.py index 85279c9..6219b90 100644 --- a/ros_typedb_examples/setup.py +++ b/ros_typedb_examples/setup.py @@ -1,12 +1,18 @@ """Setup for the ros_typedb_examples package.""" -import os from glob import glob +import os from setuptools import find_packages from setuptools import setup package_name = 'ros_typedb_examples' +typedb_test_data_path = os.path.join( + '..', + 'ros_typedb', + 'test', + 'typedb_test_data', +) setup( name=package_name, @@ -20,6 +26,11 @@ glob('launch/*launch.[pxy][yma]*')), (os.path.join('share', package_name, 'data'), glob('data/*.tql')), + (os.path.join('share', package_name, 'data', 'typedb_test_data'), + [ + os.path.join(typedb_test_data_path, 'schema.tql'), + os.path.join(typedb_test_data_path, 'data.tql'), + ]), ], install_requires=['setuptools'], zip_safe=True, From 4f3c17f58c4cbd3b37a5136f8a71e7d884bdd057 Mon Sep 17 00:00:00 2001 From: Gustavo Date: Sun, 28 Jun 2026 19:13:19 -0300 Subject: [PATCH 30/36] add stress test for changing lifecycle state when running --- ros_typedb_tools/README.md | 84 +++- .../ros_typedb_stress_experiment.py | 60 ++- .../ros_typedb_tools/stress_config.py | 22 + .../ros_typedb_tools/stress_output.py | 65 +++ .../ros_typedb_tools/stress_resolution.py | 16 + .../ros_typedb_tools/stress_runner.py | 388 +++++++++++++++++- .../test/test_ros_typedb_stress_experiment.py | 223 ++++++++++ 7 files changed, 849 insertions(+), 9 deletions(-) diff --git a/ros_typedb_tools/README.md b/ros_typedb_tools/README.md index 1e4f78d..7304a6b 100644 --- a/ros_typedb_tools/README.md +++ b/ros_typedb_tools/README.md @@ -122,7 +122,7 @@ interface. | 4. Mixed read/write load | Implemented | Read while inserting, updating, deleting, and cleaning up experiment-owned temporary data. | | 5. Delete-database fault injection | Implemented | Delete the database during load and measure whether the driver fails loudly, reloads correctly, and recovers invariants. | | 6. TypeDB restart fault injection | Implemented | Stop/restart TypeDB during load and measure outage, reconnect behavior, and post-recovery correctness. | -| 7. Lifecycle stress | Planned | Trigger ROS lifecycle cleanup/configure/activate transitions during load to check teardown races. | +| 7. Lifecycle stress | Implemented | Trigger ROS lifecycle cleanup/configure/activate transitions during load to check teardown races. | | 8. External schema/data profiles | Planned | Run the same harness with application-specific schema/data, query mixes, invariants, setup, and cleanup profiles. | Use Stage 1 when validating a single query path. Use Stage 2 when probing ROS @@ -130,7 +130,10 @@ service throughput and timeout behavior. Use Stage 3 when checking that load does not corrupt or hide baseline data. Use Stage 4 when validating robustness under real read/write activity. Use Stage 5 when validating recovery after a database deletion fault. Use Stage 6 when validating driver reconnect behavior -after the TypeDB server process or container is restarted. +after the TypeDB server process or container is restarted. Use Stage 7 when +validating that ROS lifecycle cleanup waits for active query callbacks before +tearing down TypeDB resources and that the node can be configured and activated +again afterward. ### Stage 1: Minimal Real Service Load @@ -394,14 +397,81 @@ status 1. Request failures during the outage remain recorded in the JSON output; use `--fail-on-failure` only when you want any request failure to fail the command even if invariants later recover. +### Stage 7: Lifecycle Stress + +This stage calls the ROS lifecycle `change_state` service while normal query +clients continue sending requests. The default sequence is deactivate, cleanup, +configure, then activate. Cleanup destroys the query and delete services and +waits for active service callbacks before closing TypeDB resources; the +experiment verifies that this does not race with in-flight requests and that +the node recovers after reactivation. The controller checks `get_state` after +each transition and only sends the next transition after the expected lifecycle +state is observed. + +When using a launch file that automatically reactivates the node after every +deactivate, disable that behavior for this experiment. The bundled test-data +launch file defaults to normal auto-reactivation, so start it for Stage 7 with: + +```bash +ros2 launch ros_typedb_examples test_data_example.launch.py \ + reactivate_on_deactivate:=False +``` + +Then run the stress experiment: + +```bash +ros2 run ros_typedb_tools ros_typedb_stress_experiment \ + --clients 10 \ + --duration-s 180 \ + --timeout-s 10 \ + --mode read \ + --fault lifecycle-cleanup \ + --fault-at-s 60 \ + --invariant-profile test-data \ + --request-gap-s 0.01 \ + --max-in-flight 10 \ + --output /tmp/stage7_lifecycle_cleanup.json \ + --debug-events-output /tmp/stage7_lifecycle_cleanup_debug.jsonl +``` + +The lifecycle `change_state` and `get_state` services default to the +`--service-name` prefix with `/change_state` and `/get_state` appended. For +example, the default query service `/ros_typedb_interface/query` maps to +`/ros_typedb_interface/change_state` and `/ros_typedb_interface/get_state`. +Override them with `--lifecycle-change-state-service-name` and +`--lifecycle-get-state-service-name` when the launched lifecycle node uses a +different name, such as `/ros_typedb/change_state` and +`/ros_typedb/get_state`. + +By default the node is configured and activated again after cleanup. Use +`--no-lifecycle-reactivate` only when you intentionally want to leave the node +cleaned up; in that mode recovery invariants are not expected to pass unless +something else reactivates the node. + +The result JSON adds lifecycle fields to the `fault` object: + +- `fault_lifecycle_deactivate_success`, `fault_lifecycle_deactivate_error`, + and `fault_lifecycle_deactivate_latency_s` +- `fault_lifecycle_cleanup_success`, `fault_lifecycle_cleanup_error`, and + `fault_lifecycle_cleanup_latency_s` +- `fault_lifecycle_configure_success`, `fault_lifecycle_configure_error`, and + `fault_lifecycle_configure_latency_s` +- `fault_lifecycle_activate_success`, `fault_lifecycle_activate_error`, and + `fault_lifecycle_activate_latency_s` +- `fault_lifecycle_reactivate`: whether configure/activate was requested +- `fault_observed_outage_s`: elapsed time from the first failed post-fault + request result to the first later successful request result + +If a lifecycle transition is rejected or times out, the CLI exits with status +1. If transitions succeed but invariants do not recover, the CLI also exits +with status 1. Request failures while the lifecycle node is inactive or cleaned +up remain recorded in the JSON output. + ### Planned Fault-Injection Stages -Stages 7-8 are not implemented yet. They are placeholders for the next -robustness checks: +Stage 8 is not implemented yet. It is a placeholder for the next robustness +check: -- Stage 7, lifecycle stress: call lifecycle cleanup/configure/activate while - requests are in flight, then check that service callbacks and resource - teardown do not race. - Stage 8, external profiles: load schema-specific query mixes, invariants, setup, and cleanup from user-provided JSON/YAML profile files instead of only the packaged examples. diff --git a/ros_typedb_tools/ros_typedb_tools/ros_typedb_stress_experiment.py b/ros_typedb_tools/ros_typedb_tools/ros_typedb_stress_experiment.py index 3d66ed5..720e8aa 100644 --- a/ros_typedb_tools/ros_typedb_tools/ros_typedb_stress_experiment.py +++ b/ros_typedb_tools/ros_typedb_tools/ros_typedb_stress_experiment.py @@ -186,7 +186,12 @@ def build_argument_parser() -> argparse.ArgumentParser: ) parser.add_argument( '--fault', - choices=('none', 'delete-database', 'restart-typedb'), + choices=( + 'none', + 'delete-database', + 'restart-typedb', + 'lifecycle-cleanup', + ), default='none', help=( 'Fault to inject during the experiment. ' @@ -255,6 +260,45 @@ def build_argument_parser() -> argparse.ArgumentParser: default=30.0, help='Seconds to wait for each fault command. Defaults to 30.', ) + parser.add_argument( + '--lifecycle-change-state-service-name', + help=( + 'Lifecycle change_state service name. Defaults to the ' + '--service-name prefix with /change_state appended ' + '(e.g. /ros_typedb_interface/change_state).' + ), + ) + parser.add_argument( + '--lifecycle-get-state-service-name', + help=( + 'Lifecycle get_state service name. Defaults to the ' + '--service-name prefix with /get_state appended ' + '(e.g. /ros_typedb_interface/get_state).' + ), + ) + lifecycle_reactivate_group = parser.add_mutually_exclusive_group() + lifecycle_reactivate_group.add_argument( + '--lifecycle-reactivate', + dest='lifecycle_reactivate', + action='store_true', + help=( + 'Configure and activate the node after lifecycle cleanup. This ' + 'is the default.' + ), + ) + lifecycle_reactivate_group.add_argument( + '--no-lifecycle-reactivate', + dest='lifecycle_reactivate', + action='store_false', + help='Leave the node cleaned up after the lifecycle fault.', + ) + parser.set_defaults(lifecycle_reactivate=True) + parser.add_argument( + '--lifecycle-transition-timeout-s', + type=float, + default=10.0, + help='Seconds to wait for each lifecycle transition. Defaults to 10.', + ) return parser @@ -313,6 +357,20 @@ def _fault_controller_failed(fault_result: Any) -> bool: or fault_result.fault_restart_start_success is not True ) ) + if fault_result.fault == 'lifecycle-cleanup': + required_successes = [ + fault_result.fault_lifecycle_deactivate_success, + fault_result.fault_lifecycle_cleanup_success, + ] + if fault_result.fault_lifecycle_reactivate is not False: + required_successes.extend([ + fault_result.fault_lifecycle_configure_success, + fault_result.fault_lifecycle_activate_success, + ]) + return ( + fault_result.fault_triggered_at_s is not None + and any(success is not True for success in required_successes) + ) return False diff --git a/ros_typedb_tools/ros_typedb_tools/stress_config.py b/ros_typedb_tools/ros_typedb_tools/stress_config.py index 04ea996..8bc3f0e 100644 --- a/ros_typedb_tools/ros_typedb_tools/stress_config.py +++ b/ros_typedb_tools/ros_typedb_tools/stress_config.py @@ -154,6 +154,10 @@ class ResolvedStressConfig: typedb_start_command: str | None = DEFAULT_TYPEDB_START_COMMAND typedb_restart_delay_s: float = 2.0 fault_command_timeout_s: float = 30.0 + lifecycle_change_state_service_name: str | None = None + lifecycle_get_state_service_name: str | None = None + lifecycle_reactivate: bool = True + lifecycle_transition_timeout_s: float = 10.0 @dataclass(frozen=True) @@ -177,6 +181,19 @@ class FaultResult: fault_restart_delay_s: float | None = None fault_restart_outage_s: float | None = None fault_observed_outage_s: float | None = None + fault_lifecycle_deactivate_success: bool | None = None + fault_lifecycle_deactivate_error: str | None = None + fault_lifecycle_deactivate_latency_s: float | None = None + fault_lifecycle_cleanup_success: bool | None = None + fault_lifecycle_cleanup_error: str | None = None + fault_lifecycle_cleanup_latency_s: float | None = None + fault_lifecycle_configure_success: bool | None = None + fault_lifecycle_configure_error: str | None = None + fault_lifecycle_configure_latency_s: float | None = None + fault_lifecycle_activate_success: bool | None = None + fault_lifecycle_activate_error: str | None = None + fault_lifecycle_activate_latency_s: float | None = None + fault_lifecycle_reactivate: bool | None = None @dataclass(frozen=True) @@ -252,6 +269,11 @@ def validate_experiment_args(args: argparse.Namespace) -> None: raise ValueError( '--fault-command-timeout-s must be greater than zero' ) + if fault == 'lifecycle-cleanup': + if getattr(args, 'lifecycle_transition_timeout_s', 0.0) <= 0: + raise ValueError( + '--lifecycle-transition-timeout-s must be greater than zero' + ) if fault_recovery_timeout_s <= 0: raise ValueError( '--fault-recovery-timeout-s must be greater than zero' diff --git a/ros_typedb_tools/ros_typedb_tools/stress_output.py b/ros_typedb_tools/ros_typedb_tools/stress_output.py index 650cf19..a466a28 100644 --- a/ros_typedb_tools/ros_typedb_tools/stress_output.py +++ b/ros_typedb_tools/ros_typedb_tools/stress_output.py @@ -129,6 +129,45 @@ def _fault_payload(fault_result: FaultResult | None) -> dict[str, Any]: 'fault_restart_delay_s': fault_result.fault_restart_delay_s, 'fault_restart_outage_s': fault_result.fault_restart_outage_s, 'fault_observed_outage_s': fault_result.fault_observed_outage_s, + 'fault_lifecycle_deactivate_success': ( + fault_result.fault_lifecycle_deactivate_success + ), + 'fault_lifecycle_deactivate_error': ( + fault_result.fault_lifecycle_deactivate_error + ), + 'fault_lifecycle_deactivate_latency_s': ( + fault_result.fault_lifecycle_deactivate_latency_s + ), + 'fault_lifecycle_cleanup_success': ( + fault_result.fault_lifecycle_cleanup_success + ), + 'fault_lifecycle_cleanup_error': ( + fault_result.fault_lifecycle_cleanup_error + ), + 'fault_lifecycle_cleanup_latency_s': ( + fault_result.fault_lifecycle_cleanup_latency_s + ), + 'fault_lifecycle_configure_success': ( + fault_result.fault_lifecycle_configure_success + ), + 'fault_lifecycle_configure_error': ( + fault_result.fault_lifecycle_configure_error + ), + 'fault_lifecycle_configure_latency_s': ( + fault_result.fault_lifecycle_configure_latency_s + ), + 'fault_lifecycle_activate_success': ( + fault_result.fault_lifecycle_activate_success + ), + 'fault_lifecycle_activate_error': ( + fault_result.fault_lifecycle_activate_error + ), + 'fault_lifecycle_activate_latency_s': ( + fault_result.fault_lifecycle_activate_latency_s + ), + 'fault_lifecycle_reactivate': ( + fault_result.fault_lifecycle_reactivate + ), } @@ -325,6 +364,32 @@ def print_summary( f'{observed_outage} | recovery: {recovery_str}' ) return + if fault_result.fault == 'lifecycle-cleanup': + lifecycle_steps = [ + fault_result.fault_lifecycle_deactivate_success, + fault_result.fault_lifecycle_cleanup_success, + ] + if fault_result.fault_lifecycle_reactivate is not False: + lifecycle_steps.extend([ + fault_result.fault_lifecycle_configure_success, + fault_result.fault_lifecycle_activate_success, + ]) + lifecycle_status = ( + 'ok' if all(step is True for step in lifecycle_steps) + else 'FAILED' + ) + observed_outage = ( + f'{fault_result.fault_observed_outage_s:.3f}s' + if fault_result.fault_observed_outage_s is not None + else 'n/a' + ) + print( + f' fault: {fault_result.fault} triggered at ' + f'{triggered_at} | lifecycle: {lifecycle_status} ' + f'| observed outage: {observed_outage} | recovery: ' + f'{recovery_str}' + ) + return print( f' fault: {fault_result.fault} triggered at {triggered_at} | ' f'delete: {delete_status} ({delete_latency}) | ' diff --git a/ros_typedb_tools/ros_typedb_tools/stress_resolution.py b/ros_typedb_tools/ros_typedb_tools/stress_resolution.py index d754606..e2bb7af 100644 --- a/ros_typedb_tools/ros_typedb_tools/stress_resolution.py +++ b/ros_typedb_tools/ros_typedb_tools/stress_resolution.py @@ -62,6 +62,16 @@ def resolve_stress_config( ) or DEFAULT_TYPEDB_START_COMMAND typedb_restart_delay_s = getattr(args, 'typedb_restart_delay_s', 2.0) fault_command_timeout_s = getattr(args, 'fault_command_timeout_s', 30.0) + lifecycle_change_state_service_name = getattr( + args, 'lifecycle_change_state_service_name', None + ) + lifecycle_get_state_service_name = getattr( + args, 'lifecycle_get_state_service_name', None + ) + lifecycle_reactivate = getattr(args, 'lifecycle_reactivate', True) + lifecycle_transition_timeout_s = getattr( + args, 'lifecycle_transition_timeout_s', 10.0 + ) return ResolvedStressConfig( query_specs=query_specs, @@ -81,6 +91,12 @@ def resolve_stress_config( typedb_start_command=typedb_start_command, typedb_restart_delay_s=typedb_restart_delay_s, fault_command_timeout_s=fault_command_timeout_s, + lifecycle_change_state_service_name=( + lifecycle_change_state_service_name + ), + lifecycle_get_state_service_name=lifecycle_get_state_service_name, + lifecycle_reactivate=lifecycle_reactivate, + lifecycle_transition_timeout_s=lifecycle_transition_timeout_s, ) diff --git a/ros_typedb_tools/ros_typedb_tools/stress_runner.py b/ros_typedb_tools/ros_typedb_tools/stress_runner.py index a6f6c99..4fb02c3 100644 --- a/ros_typedb_tools/ros_typedb_tools/stress_runner.py +++ b/ros_typedb_tools/ros_typedb_tools/stress_runner.py @@ -28,6 +28,10 @@ from rclpy.executors import MultiThreadedExecutor from rclpy.executors import SingleThreadedExecutor +from lifecycle_msgs.msg import State +from lifecycle_msgs.msg import Transition +from lifecycle_msgs.srv import ChangeState +from lifecycle_msgs.srv import GetState from ros_typedb_msgs.srv import Query from ros_typedb_tools.stress_config import build_query_request @@ -89,6 +93,19 @@ class _RunState: fault_restart_started_at_s: float | None = None fault_restart_ended_at_s: float | None = None fault_restart_delay_s: float | None = None + fault_lifecycle_deactivate_success: bool | None = None + fault_lifecycle_deactivate_error: str | None = None + fault_lifecycle_deactivate_latency_s: float | None = None + fault_lifecycle_cleanup_success: bool | None = None + fault_lifecycle_cleanup_error: str | None = None + fault_lifecycle_cleanup_latency_s: float | None = None + fault_lifecycle_configure_success: bool | None = None + fault_lifecycle_configure_error: str | None = None + fault_lifecycle_configure_latency_s: float | None = None + fault_lifecycle_activate_success: bool | None = None + fault_lifecycle_activate_error: str | None = None + fault_lifecycle_activate_latency_s: float | None = None + fault_lifecycle_reactivate: bool | None = None def _build_executor(args: argparse.Namespace) -> Any | None: @@ -685,6 +702,20 @@ def _derive_delete_db_service_name(service_name: str) -> str: return prefix + '/delete_database' +def _derive_lifecycle_change_state_service_name(service_name: str) -> str: + """Derive lifecycle change_state service name from the query service.""" + parts = service_name.rstrip('/').rsplit('/', 1) + prefix = parts[0] if len(parts) > 1 else '' + return prefix + '/change_state' + + +def _derive_lifecycle_get_state_service_name(service_name: str) -> str: + """Derive lifecycle get_state service name from the query service.""" + parts = service_name.rstrip('/').rsplit('/', 1) + prefix = parts[0] if len(parts) > 1 else '' + return prefix + '/get_state' + + def _maybe_run_periodic_invariants( *, args: argparse.Namespace, @@ -961,6 +992,279 @@ def _trigger_restart_typedb_fault( return thread +def _call_lifecycle_transition( + *, + client: Any, + transition_id: int, + transition_name: str, + timeout_s: float, +) -> tuple[bool, str | None, float]: + """Request one lifecycle transition and wait for the main loop to spin.""" + started_at_s = time.monotonic() + request = ChangeState.Request() + request.transition.id = transition_id + future = client.call_async(request) + deadline_s = started_at_s + timeout_s + + while not future.done(): + if time.monotonic() >= deadline_s: + return ( + False, + ( + f'lifecycle {transition_name} transition timed out ' + f'after {timeout_s:g}s' + ), + time.monotonic() - started_at_s, + ) + time.sleep(0.01) + + latency_s = time.monotonic() - started_at_s + exc = future.exception() + if exc is not None: + return False, str(exc), latency_s + + result = future.result() + if not getattr(result, 'success', False): + return ( + False, + f'lifecycle {transition_name} transition rejected', + latency_s, + ) + return True, None, latency_s + + +def _call_lifecycle_get_state( + *, + client: Any, + timeout_s: float, +) -> tuple[bool, str | None, float, int | None, str | None]: + """Request the current lifecycle state.""" + started_at_s = time.monotonic() + future = client.call_async(GetState.Request()) + deadline_s = started_at_s + timeout_s + + while not future.done(): + if time.monotonic() >= deadline_s: + return ( + False, + f'lifecycle get_state timed out after {timeout_s:g}s', + time.monotonic() - started_at_s, + None, + None, + ) + time.sleep(0.01) + + latency_s = time.monotonic() - started_at_s + exc = future.exception() + if exc is not None: + return False, str(exc), latency_s, None, None + + current_state = future.result().current_state + return ( + True, + None, + latency_s, + current_state.id, + current_state.label, + ) + + +def _wait_for_lifecycle_state( + *, + client: Any, + expected_state_id: int, + expected_state_label: str, + timeout_s: float, + debug_events: DebugEventWriter, +) -> tuple[bool, str | None, float]: + """Poll lifecycle get_state until the expected primary state is reached.""" + started_at_s = time.monotonic() + deadline_s = started_at_s + timeout_s + last_state_id = None + last_state_label = None + last_error = None + + while time.monotonic() < deadline_s: + remaining_s = max(0.0, deadline_s - time.monotonic()) + success, error, _, state_id, state_label = _call_lifecycle_get_state( + client=client, + timeout_s=min(1.0, remaining_s), + ) + debug_events.log( + 'fault_lifecycle_state_observed', + expected_state=expected_state_label, + state_id=state_id, + state_label=state_label, + success=success, + error=error, + ) + if success and state_id == expected_state_id: + return True, None, time.monotonic() - started_at_s + last_state_id = state_id + last_state_label = state_label + last_error = error + time.sleep(0.05) + + if last_error: + detail = last_error + else: + detail = f'last state was {last_state_label} ({last_state_id})' + return ( + False, + ( + f'lifecycle did not reach {expected_state_label} after ' + f'{timeout_s:g}s; {detail}' + ), + time.monotonic() - started_at_s, + ) + + +def _record_lifecycle_transition_result( + *, + state: _RunState, + transition_name: str, + success: bool, + error: str | None, + latency_s: float, +) -> None: + """Store one lifecycle transition result on the run state.""" + setattr(state, f'fault_lifecycle_{transition_name}_success', success) + setattr(state, f'fault_lifecycle_{transition_name}_error', error) + setattr( + state, + f'fault_lifecycle_{transition_name}_latency_s', + latency_s, + ) + + +def _lifecycle_cleanup_fault_worker( + *, + lifecycle_change_state_client: Any, + lifecycle_get_state_client: Any, + reactivate: bool, + transition_timeout_s: float, + state: _RunState, + debug_events: DebugEventWriter, +) -> None: + """Cleanup and optionally reactivate the lifecycle node during load.""" + state.fault_lifecycle_reactivate = reactivate + transitions = [ + ( + 'deactivate', + Transition.TRANSITION_DEACTIVATE, + State.PRIMARY_STATE_INACTIVE, + 'inactive', + ), + ( + 'cleanup', + Transition.TRANSITION_CLEANUP, + State.PRIMARY_STATE_UNCONFIGURED, + 'unconfigured', + ), + ] + if reactivate: + transitions.extend([ + ( + 'configure', + Transition.TRANSITION_CONFIGURE, + State.PRIMARY_STATE_INACTIVE, + 'inactive', + ), + ( + 'activate', + Transition.TRANSITION_ACTIVATE, + State.PRIMARY_STATE_ACTIVE, + 'active', + ), + ]) + + for ( + transition_name, + transition_id, + expected_state_id, + expected_state_label, + ) in transitions: + debug_events.log( + 'fault_lifecycle_transition_started', + transition=transition_name, + transition_id=transition_id, + expected_state=expected_state_label, + transition_timeout_s=transition_timeout_s, + ) + success, error, latency_s = _call_lifecycle_transition( + client=lifecycle_change_state_client, + transition_id=transition_id, + transition_name=transition_name, + timeout_s=transition_timeout_s, + ) + if success: + state_success, state_error, state_latency_s = ( + _wait_for_lifecycle_state( + client=lifecycle_get_state_client, + expected_state_id=expected_state_id, + expected_state_label=expected_state_label, + timeout_s=transition_timeout_s, + debug_events=debug_events, + ) + ) + success = state_success + error = state_error + latency_s += state_latency_s + _record_lifecycle_transition_result( + state=state, + transition_name=transition_name, + success=success, + error=error, + latency_s=latency_s, + ) + debug_events.log( + 'fault_lifecycle_transition_complete', + transition=transition_name, + success=success, + error=error, + latency_s=latency_s, + ) + if not success: + return + + +def _trigger_lifecycle_cleanup_fault( + *, + lifecycle_change_state_client: Any, + lifecycle_get_state_client: Any, + config: Any, + state: _RunState, + debug_events: DebugEventWriter, +) -> threading.Thread: + """Start a background lifecycle cleanup fault controller.""" + started_at_s = time.monotonic() + pending_count = len(state.pending_by_client) + state.fault_triggered = True + state.fault_triggered_at_s = started_at_s + state.fault_lifecycle_reactivate = config.lifecycle_reactivate + debug_events.log( + 'fault_triggered', + fault='lifecycle-cleanup', + triggered_at_s=started_at_s, + pending_count_at_fault=pending_count, + lifecycle_reactivate=config.lifecycle_reactivate, + ) + thread = threading.Thread( + target=_lifecycle_cleanup_fault_worker, + kwargs={ + 'lifecycle_change_state_client': lifecycle_change_state_client, + 'lifecycle_get_state_client': lifecycle_get_state_client, + 'reactivate': config.lifecycle_reactivate, + 'transition_timeout_s': config.lifecycle_transition_timeout_s, + 'state': state, + 'debug_events': debug_events, + }, + daemon=True, + ) + thread.start() + return thread + + def _compute_observed_fault_outage_s( records: list[RequestRecord], *, @@ -1052,6 +1356,26 @@ def run_experiment(args: argparse.Namespace) -> StressExperimentResult: if getattr(args, 'fault', 'none') == 'delete-database' else None ) + lifecycle_change_state_service_name = ( + config.lifecycle_change_state_service_name + if config.lifecycle_change_state_service_name + else _derive_lifecycle_change_state_service_name(args.service_name) + ) + lifecycle_get_state_service_name = ( + config.lifecycle_get_state_service_name + if config.lifecycle_get_state_service_name + else _derive_lifecycle_get_state_service_name(args.service_name) + ) + lifecycle_change_state_client = ( + node.create_client(ChangeState, lifecycle_change_state_service_name) + if config.fault == 'lifecycle-cleanup' + else None + ) + lifecycle_get_state_client = ( + node.create_client(GetState, lifecycle_get_state_service_name) + if config.fault == 'lifecycle-cleanup' + else None + ) deadline_s = ( time.monotonic() + args.duration_s if args.duration_s is not None @@ -1073,6 +1397,7 @@ def run_experiment(args: argparse.Namespace) -> StressExperimentResult: next_invariant_s=time.monotonic() + args.invariant_period_s, ) restart_fault_thread = None + lifecycle_fault_thread = None try: with DebugEventWriter(config.debug_events_output) as debug_events: @@ -1090,7 +1415,17 @@ def run_experiment(args: argparse.Namespace) -> StressExperimentResult: clients + ([invariant_client] if invariant_client else []) + ([cleanup_client] if cleanup_client else []) - + ([delete_db_client] if delete_db_client is not None else []), + + ([delete_db_client] if delete_db_client is not None else []) + + ( + [lifecycle_change_state_client] + if lifecycle_change_state_client is not None + else [] + ) + + ( + [lifecycle_get_state_client] + if lifecycle_get_state_client is not None + else [] + ), service_name=args.service_name, wait_service_timeout_s=args.wait_service_timeout_s, debug_events=debug_events, @@ -1170,9 +1505,29 @@ def run_experiment(args: argparse.Namespace) -> StressExperimentResult: state=state, debug_events=debug_events, ) + if ( + config.fault == 'lifecycle-cleanup' + and lifecycle_change_state_client is not None + and lifecycle_get_state_client is not None + and not state.fault_triggered + and config.fault_at_s is not None + and time.monotonic() - experiment_started_at_s + >= config.fault_at_s + ): + lifecycle_fault_thread = _trigger_lifecycle_cleanup_fault( + lifecycle_change_state_client=( + lifecycle_change_state_client + ), + lifecycle_get_state_client=lifecycle_get_state_client, + config=config, + state=state, + debug_events=debug_events, + ) if restart_fault_thread is not None: restart_fault_thread.join() + if lifecycle_fault_thread is not None: + lifecycle_fault_thread.join() # 1. Recovery must come before cleanup. Cleanup queries trigger DB # recreation, silently advancing recovery unobserved. @@ -1267,6 +1622,37 @@ def run_experiment(args: argparse.Namespace) -> StressExperimentResult: state.records, fault_triggered_at_s=state.fault_triggered_at_s, ), + fault_lifecycle_deactivate_success=( + state.fault_lifecycle_deactivate_success + ), + fault_lifecycle_deactivate_error=( + state.fault_lifecycle_deactivate_error + ), + fault_lifecycle_deactivate_latency_s=( + state.fault_lifecycle_deactivate_latency_s + ), + fault_lifecycle_cleanup_success=state.fault_lifecycle_cleanup_success, + fault_lifecycle_cleanup_error=state.fault_lifecycle_cleanup_error, + fault_lifecycle_cleanup_latency_s=( + state.fault_lifecycle_cleanup_latency_s + ), + fault_lifecycle_configure_success=( + state.fault_lifecycle_configure_success + ), + fault_lifecycle_configure_error=( + state.fault_lifecycle_configure_error + ), + fault_lifecycle_configure_latency_s=( + state.fault_lifecycle_configure_latency_s + ), + fault_lifecycle_activate_success=( + state.fault_lifecycle_activate_success + ), + fault_lifecycle_activate_error=state.fault_lifecycle_activate_error, + fault_lifecycle_activate_latency_s=( + state.fault_lifecycle_activate_latency_s + ), + fault_lifecycle_reactivate=state.fault_lifecycle_reactivate, ) return StressExperimentResult( records=state.records, diff --git a/ros_typedb_tools/test/test_ros_typedb_stress_experiment.py b/ros_typedb_tools/test/test_ros_typedb_stress_experiment.py index 41f1f6b..d562070 100644 --- a/ros_typedb_tools/test/test_ros_typedb_stress_experiment.py +++ b/ros_typedb_tools/test/test_ros_typedb_stress_experiment.py @@ -49,7 +49,10 @@ from ros_typedb_tools.stress_resolution import resolve_stress_config from ros_typedb_tools.stress_runner import ( _build_typedb_restart_commands, + _call_lifecycle_transition, _compute_observed_fault_outage_s, + _derive_lifecycle_change_state_service_name, + _derive_lifecycle_get_state_service_name, _mark_fault_recovered_from_invariant_pass, _run_fault_command, _start_fault_command, @@ -130,6 +133,35 @@ def _run_state(**overrides) -> _RunState: return state +class _DoneFuture: + """Minimal completed future for lifecycle helper tests.""" + + def __init__(self, *, result=None, exception=None): + self._result = result + self._exception = exception + + def done(self): + return True + + def exception(self): + return self._exception + + def result(self): + return self._result + + +class _CapturingClient: + """Minimal service client that captures the last async request.""" + + def __init__(self, future): + self.future = future + self.request = None + + def call_async(self, request): + self.request = request + return self.future + + def _aggregate_count_response(value: int, *, success: bool = True): parameter_value = ParameterValue() parameter_value.type = ParameterType.PARAMETER_INTEGER @@ -306,6 +338,44 @@ def test_parser_accepts_stage_six_restart_fault_arguments(): assert args.fault_command_timeout_s == 15 +def test_parser_accepts_stage_seven_lifecycle_fault_arguments(): + """Check Stage 7 lifecycle cleanup fault arguments.""" + parser = build_argument_parser() + + args = parser.parse_args( + [ + '--clients', + '10', + '--duration-s', + '180', + '--mode', + 'read', + '--fault', + 'lifecycle-cleanup', + '--fault-at-s', + '60', + '--lifecycle-change-state-service-name', + '/ros_typedb/change_state', + '--lifecycle-get-state-service-name', + '/ros_typedb/get_state', + '--no-lifecycle-reactivate', + '--lifecycle-transition-timeout-s', + '12', + '--invariant-profile', + 'test-data', + ] + ) + + assert args.fault == 'lifecycle-cleanup' + assert args.fault_at_s == 60 + assert args.lifecycle_change_state_service_name == ( + '/ros_typedb/change_state' + ) + assert args.lifecycle_get_state_service_name == '/ros_typedb/get_state' + assert args.lifecycle_reactivate is False + assert args.lifecycle_transition_timeout_s == 12 + + def test_fake_service_parser_accepts_diagnostic_arguments(): """Check the fake service CLI options.""" parser = build_fake_service_argument_parser() @@ -501,6 +571,19 @@ def test_write_results_writes_summary_and_request_records(tmp_path: Path): 'fault_restart_delay_s': None, 'fault_restart_outage_s': None, 'fault_observed_outage_s': None, + 'fault_lifecycle_deactivate_success': None, + 'fault_lifecycle_deactivate_error': None, + 'fault_lifecycle_deactivate_latency_s': None, + 'fault_lifecycle_cleanup_success': None, + 'fault_lifecycle_cleanup_error': None, + 'fault_lifecycle_cleanup_latency_s': None, + 'fault_lifecycle_configure_success': None, + 'fault_lifecycle_configure_error': None, + 'fault_lifecycle_configure_latency_s': None, + 'fault_lifecycle_activate_success': None, + 'fault_lifecycle_activate_error': None, + 'fault_lifecycle_activate_latency_s': None, + 'fault_lifecycle_reactivate': None, } @@ -536,6 +619,19 @@ def test_write_results_writes_default_fault_payload(tmp_path: Path): 'fault_restart_delay_s': None, 'fault_restart_outage_s': None, 'fault_observed_outage_s': None, + 'fault_lifecycle_deactivate_success': None, + 'fault_lifecycle_deactivate_error': None, + 'fault_lifecycle_deactivate_latency_s': None, + 'fault_lifecycle_cleanup_success': None, + 'fault_lifecycle_cleanup_error': None, + 'fault_lifecycle_cleanup_latency_s': None, + 'fault_lifecycle_configure_success': None, + 'fault_lifecycle_configure_error': None, + 'fault_lifecycle_configure_latency_s': None, + 'fault_lifecycle_activate_success': None, + 'fault_lifecycle_activate_error': None, + 'fault_lifecycle_activate_latency_s': None, + 'fault_lifecycle_reactivate': None, } @@ -575,6 +671,61 @@ def test_build_typedb_restart_commands_uses_command_hooks(): assert start_command == ['bash', '-lc', 'typedb server start'] +def test_derive_lifecycle_change_state_service_name_from_query_service(): + """Check lifecycle service name follows the query service prefix.""" + assert _derive_lifecycle_change_state_service_name( + '/ros_typedb_interface/query' + ) == '/ros_typedb_interface/change_state' + assert _derive_lifecycle_change_state_service_name( + '/query' + ) == '/change_state' + + +def test_derive_lifecycle_get_state_service_name_from_query_service(): + """Check lifecycle get_state service name follows the query prefix.""" + assert _derive_lifecycle_get_state_service_name( + '/ros_typedb_interface/query' + ) == '/ros_typedb_interface/get_state' + assert _derive_lifecycle_get_state_service_name( + '/query' + ) == '/get_state' + + +def test_call_lifecycle_transition_reports_success(): + """Check lifecycle transition helper builds a ChangeState request.""" + response = SimpleNamespace(success=True) + client = _CapturingClient(_DoneFuture(result=response)) + + success, error, latency_s = _call_lifecycle_transition( + client=client, + transition_id=3, + transition_name='activate', + timeout_s=1.0, + ) + + assert success is True + assert error is None + assert latency_s >= 0.0 + assert client.request.transition.id == 3 + + +def test_call_lifecycle_transition_reports_rejected_transition(): + """Check lifecycle transition helper treats success=False as failure.""" + response = SimpleNamespace(success=False) + client = _CapturingClient(_DoneFuture(result=response)) + + success, error, latency_s = _call_lifecycle_transition( + client=client, + transition_id=2, + transition_name='cleanup', + timeout_s=1.0, + ) + + assert success is False + assert error == 'lifecycle cleanup transition rejected' + assert latency_s >= 0.0 + + def test_run_fault_command_reports_nonzero_exit(monkeypatch): """Check fault command execution reports stderr on failure.""" class Completed: @@ -604,6 +755,7 @@ def fake_run(command, **kwargs): def test_start_fault_command_treats_running_process_as_success(monkeypatch): """Check foreground server commands succeed once they stay running.""" + class RunningProcess: def wait(self, timeout): assert timeout == 0.25 @@ -1043,6 +1195,10 @@ def _base_args(**overrides): 'typedb_start_command': DEFAULT_TYPEDB_START_COMMAND, 'typedb_restart_delay_s': 2.0, 'fault_command_timeout_s': 30.0, + 'lifecycle_change_state_service_name': None, + 'lifecycle_get_state_service_name': None, + 'lifecycle_reactivate': True, + 'lifecycle_transition_timeout_s': 10.0, } base.update(overrides) return SimpleNamespace(**base) @@ -1139,6 +1295,32 @@ def test_validate_args_fault_restart_typedb_accepts_container(): validate_experiment_args(args) +def test_validate_args_fault_lifecycle_cleanup_accepts_defaults(): + """Check lifecycle-cleanup accepts default lifecycle controller args.""" + args = _base_args( + fault='lifecycle-cleanup', + invariant_profile='test-data', + duration_s=60.0, + fault_at_s=10.0, + ) + + validate_experiment_args(args) + + +def test_validate_args_lifecycle_transition_timeout_s_must_be_positive(): + """Check lifecycle transition timeout must be greater than zero.""" + args = _base_args( + fault='lifecycle-cleanup', + invariant_profile='test-data', + duration_s=60.0, + fault_at_s=10.0, + lifecycle_transition_timeout_s=0.0, + ) + + with pytest.raises(ValueError, match='--lifecycle-transition-timeout-s'): + validate_experiment_args(args) + + def test_validate_args_fault_recovery_timeout_s_must_be_positive(): """Check --fault-recovery-timeout-s must be greater than zero.""" args = _base_args(fault_recovery_timeout_s=0.0) @@ -1231,6 +1413,47 @@ def test_main_fails_when_restart_fault_controller_fails(monkeypatch): assert stress_cli.main([]) == 1 +def test_main_fails_when_lifecycle_fault_controller_fails(monkeypatch): + """Check CLI exits nonzero when lifecycle cleanup fails.""" + fault_result = FaultResult( + fault='lifecycle-cleanup', + fault_at_s=10.0, + fault_triggered_at_s=10.0, + fault_delete_success=None, + fault_delete_error=None, + fault_delete_latency_s=None, + fault_recovered_at_s=12.0, + fault_recovery_s=2.0, + fault_lifecycle_deactivate_success=True, + fault_lifecycle_cleanup_success=False, + fault_lifecycle_cleanup_error='cleanup rejected', + fault_lifecycle_reactivate=True, + ) + result = StressExperimentResult( + records=[], + invariant_records=[], + config=ResolvedStressConfig( + query_specs=[], + invariant_specs=[], + mixed_profile=None, + mixed_profile_name=None, + invariant_profile_name='test-data', + max_in_flight=1, + debug_events_output=None, + fault='lifecycle-cleanup', + fault_at_s=10.0, + fault_recovery_timeout_s=30.0, + ), + fault_result=fault_result, + ) + + monkeypatch.setattr(stress_cli.rclpy, 'init', lambda args=None: None) + monkeypatch.setattr(stress_cli.rclpy, 'shutdown', lambda: None) + monkeypatch.setattr(stress_cli, 'run_experiment', lambda args: result) + + assert stress_cli.main([]) == 1 + + def test_main_allows_periodic_invariant_failures_during_recovered_fault( monkeypatch, ): From 894b4731cbbd1b03b489987f18a038e99ce39ee2 Mon Sep 17 00:00:00 2001 From: Gustavo Date: Sun, 28 Jun 2026 19:14:50 -0300 Subject: [PATCH 31/36] update agent's files --- .serena/.gitignore | 2 + .serena/memories/project_overview.md | 1 + .serena/memories/style_and_conventions.md | 1 + .serena/memories/suggested_commands.md | 17 +++ .serena/memories/task_completion.md | 1 + .serena/project.yml | 170 ++++++++++++++++++++++ AGENTS.md | 89 +++++------ CLAUDE.md | 46 ++++++ 8 files changed, 272 insertions(+), 55 deletions(-) create mode 100644 .serena/.gitignore create mode 100644 .serena/memories/project_overview.md create mode 100644 .serena/memories/style_and_conventions.md create mode 100644 .serena/memories/suggested_commands.md create mode 100644 .serena/memories/task_completion.md create mode 100644 .serena/project.yml create mode 100644 CLAUDE.md diff --git a/.serena/.gitignore b/.serena/.gitignore new file mode 100644 index 0000000..2e510af --- /dev/null +++ b/.serena/.gitignore @@ -0,0 +1,2 @@ +/cache +/project.local.yml diff --git a/.serena/memories/project_overview.md b/.serena/memories/project_overview.md new file mode 100644 index 0000000..1248e2e --- /dev/null +++ b/.serena/memories/project_overview.md @@ -0,0 +1 @@ +ros_typedb is a ROS 2 Humble workspace repository containing three main packages: ros_typedb (core ament_python node/runtime), ros_typedb_msgs (ament_cmake message/service interfaces), and ros_typedb_tools (Python CLI tools for TypeDB schema/rule diagrams and stress tooling). The repo also contains docs under docs/source and agent planning notes under docs-agents. Target platform is Ubuntu 22.04, ROS Humble, and TypeDB 2.x. \ No newline at end of file diff --git a/.serena/memories/style_and_conventions.md b/.serena/memories/style_and_conventions.md new file mode 100644 index 0000000..16f0a14 --- /dev/null +++ b/.serena/memories/style_and_conventions.md @@ -0,0 +1 @@ +Python code follows ROS 2 ament_flake8 and ament_pep257 checks. Use 4-space indentation, snake_case for modules/functions, PascalCase for classes, descriptive ROS parameter names, and keep console entry point names aligned with setup.py. Preserve the package layout and avoid mixing generated ROS interfaces with Python runtime modules. Prefer Serena symbolic navigation for non-trivial Python code exploration and keep edits scoped to the requested package/behavior. \ No newline at end of file diff --git a/.serena/memories/suggested_commands.md b/.serena/memories/suggested_commands.md new file mode 100644 index 0000000..1fa1b42 --- /dev/null +++ b/.serena/memories/suggested_commands.md @@ -0,0 +1,17 @@ +Run from the ROS workspace root unless noted: + +```bash +source /opt/ros/humble/setup.bash +rosdep install --from-paths src --ignore-src -r -y +colcon build --symlink-install +source install/setup.bash +colcon test --event-handlers console_cohesion+ --packages-up-to ros_typedb +colcon test --event-handlers console_cohesion+ --packages-select ros_typedb_tools +``` + +Run core node: +```bash +ros2 run ros_typedb ros_typedb_interface -p schema_path:= -p data_path:= +``` + +Build docs from docs/ with `make html`. Prefer `rg`/`rg --files` for repository search. In this environment, use the running `ros_typedb` container for ROS/TypeDB-dependent commands when needed. \ No newline at end of file diff --git a/.serena/memories/task_completion.md b/.serena/memories/task_completion.md new file mode 100644 index 0000000..01b59f5 --- /dev/null +++ b/.serena/memories/task_completion.md @@ -0,0 +1 @@ +Before finishing code changes, run focused tests for the changed package where possible, plus targeted lint if full colcon test is known to include unrelated failures. For ros_typedb_tools stress tooling, useful checks include `python3 -m pytest -q ros_typedb_tools/test/test_ros_typedb_stress_experiment.py`, targeted `flake8` on changed files, `colcon build --symlink-install --packages-select ros_typedb_tools`, and `ros2 run ros_typedb_tools ros_typedb_stress_experiment --help`. Report any tests not run or failures that are unrelated to the change. \ No newline at end of file diff --git a/.serena/project.yml b/.serena/project.yml new file mode 100644 index 0000000..c46a2bd --- /dev/null +++ b/.serena/project.yml @@ -0,0 +1,170 @@ +# the name by which the project can be referenced within Serena +project_name: "ros_typedb" + + +# list of languages for which language servers are started; choose from: +# al bash clojure cpp csharp +# csharp_omnisharp dart elixir elm erlang +# fortran fsharp go groovy haskell +# haxe java julia kotlin lua +# markdown +# matlab nix pascal perl php +# php_phpactor powershell python python_jedi r +# rego ruby ruby_solargraph rust scala +# swift terraform toml typescript typescript_vts +# vue yaml zig +# (This list may be outdated. For the current list, see values of Language enum here: +# https://github.com/oraios/serena/blob/main/src/solidlsp/ls_config.py +# For some languages, there are alternative language servers, e.g. csharp_omnisharp, ruby_solargraph.) +# Note: +# - For C, use cpp +# - For JavaScript, use typescript +# - For Free Pascal/Lazarus, use pascal +# Special requirements: +# Some languages require additional setup/installations. +# See here for details: https://oraios.github.io/serena/01-about/020_programming-languages.html#language-servers +# When using multiple languages, the first language server that supports a given file will be used for that file. +# The first language is the default language and the respective language server will be used as a fallback. +# Note that when using the JetBrains backend, language servers are not used and this list is correspondingly ignored. +languages: +- python + +# the encoding used by text files in the project +# For a list of possible encodings, see https://docs.python.org/3.11/library/codecs.html#standard-encodings +encoding: "utf-8" + +# line ending convention to use when writing source files. +# Possible values: unset (use global setting), "lf", "crlf", or "native" (platform default) +# This does not affect Serena's own files (e.g. memories and configuration files), which always use native line endings. +line_ending: + +# The language backend to use for this project. +# If not set, the global setting from serena_config.yml is used. +# Valid values: LSP, JetBrains +# Note: the backend is fixed at startup. If a project with a different backend +# is activated post-init, an error will be returned. +language_backend: + +# whether to use project's .gitignore files to ignore files +ignore_all_files_in_gitignore: true + +# advanced configuration option allowing to configure language server-specific options. +# Maps the language key to the options. +# Have a look at the docstring of the constructors of the LS implementations within solidlsp (e.g., for C# or PHP) to see which options are available. +# No documentation on options means no options are available. +ls_specific_settings: {} + +# list of additional workspace folder paths for cross-package reference support (e.g. in monorepos). +# Paths can be absolute or relative to the project root. +# Each folder is registered as an LSP workspace folder, enabling language servers to discover +# symbols and references across package boundaries. +# Currently supported for: TypeScript. +# Example: +# additional_workspace_folders: +# - ../sibling-package +# - ../shared-lib +additional_workspace_folders: [] + +# list of additional paths to ignore in this project. +# Same syntax as gitignore, so you can use * and **. +# Note: global ignored_paths from serena_config.yml are also applied additively. +ignored_paths: [] + +# whether the project is in read-only mode +# If set to true, all editing tools will be disabled and attempts to use them will result in an error +# Added on 2025-04-18 +read_only: false + +# list of tool names to exclude. +# This extends the existing exclusions (e.g. from the global configuration) +# +# Below is the complete list of tools for convenience. +# To make sure you have the latest list of tools, and to view their descriptions, +# execute `uv run scripts/print_tool_overview.py`. +# +# * `activate_project`: Activates a project based on the project name or path. +# * `check_onboarding_performed`: Checks whether project onboarding was already performed. +# * `create_text_file`: Creates/overwrites a file in the project directory. +# * `delete_memory`: Delete a memory file. Should only happen if a user asks for it explicitly, +# for example by saying that the information retrieved from a memory file is no longer correct +# or no longer relevant for the project. +# * `edit_memory`: Replaces content matching a regular expression in a memory. +# * `execute_shell_command`: Executes a shell command. +# * `find_file`: Finds files in the given relative paths +# * `find_referencing_symbols`: Finds symbols that reference the given symbol using the language server backend +# * `find_symbol`: Performs a global (or local) search using the language server backend. +# * `get_current_config`: Prints the current configuration of the agent, including the active and available projects, tools, contexts, and modes. +# * `get_symbols_overview`: Gets an overview of the top-level symbols defined in a given file. +# * `initial_instructions`: Provides instructions Serena usage (i.e. the 'Serena Instructions Manual') +# for clients that do not read the initial instructions when the MCP server is connected. +# * `insert_after_symbol`: Inserts content after the end of the definition of a given symbol. +# * `insert_before_symbol`: Inserts content before the beginning of the definition of a given symbol. +# * `list_dir`: Lists files and directories in the given directory (optionally with recursion). +# * `list_memories`: List available memories. Any memory can be read using the `read_memory` tool. +# * `onboarding`: Performs onboarding (identifying the project structure and essential tasks, e.g. for testing or building). +# * `read_file`: Reads a file within the project directory. +# * `read_memory`: Read the content of a memory file. This tool should only be used if the information +# is relevant to the current task. You can infer whether the information +# is relevant from the memory file name. +# You should not read the same memory file multiple times in the same conversation. +# * `rename_memory`: Renames or moves a memory. Moving between project and global scope is supported +# (e.g., renaming "global/foo" to "bar" moves it from global to project scope). +# * `rename_symbol`: Renames a symbol throughout the codebase using language server refactoring capabilities. +# For JB, we use a separate tool. +# * `replace_content`: Replaces content in a file (optionally using regular expressions). +# * `replace_symbol_body`: Replaces the full definition of a symbol using the language server backend. +# * `safe_delete_symbol`: +# * `search_for_pattern`: Performs a search for a pattern in the project. +# * `write_memory`: Write some information (utf-8-encoded) about this project that can be useful for future tasks to a memory in md format. +# The memory name should be meaningful. +excluded_tools: [] + +# list of tools to include that would otherwise be disabled (particularly optional tools that are disabled by default). +# This extends the existing inclusions (e.g. from the global configuration). +included_optional_tools: [] + +# fixed set of tools to use as the base tool set (if non-empty), replacing Serena's default set of tools. +# This cannot be combined with non-empty excluded_tools or included_optional_tools. +fixed_tools: [] + +# list of mode names that are to be activated by default. +# The full set of modes to be activated is base_modes + default_modes. +# If the setting is undefined, the default_modes from the global configuration (serena_config.yml) apply. +# Otherwise, this overrides the setting from the global configuration (serena_config.yml). +# This setting can, in turn, be overridden by CLI parameters (--mode). +default_modes: + +# list of mode names to be activated additionally for this project, e.g. ["query-projects"] +# The full set of modes to be activated is base_modes (from global config) + default_modes + added_modes. +# See https://oraios.github.io/serena/02-usage/050_configuration.html#modes +added_modes: + +# initial prompt for the project. It will always be given to the LLM upon activating the project +# (contrary to the memories, which are loaded on demand). +initial_prompt: "" + +# time budget (seconds) per tool call for the retrieval of additional symbol information +# such as docstrings or parameter information. +# This overrides the corresponding setting in the global configuration; see the documentation there. +# If null or missing, use the setting from the global configuration. +symbol_info_budget: + +# list of regex patterns which, when matched, mark a memory entry as read‑only. +# Extends the list from the global configuration, merging the two lists. +read_only_memory_patterns: [] + +# list of regex patterns for memories to completely ignore. +# Matching memories will not appear in list_memories or activate_project output +# and cannot be accessed via read_memory or write_memory. +# To access ignored memory files, use the read_file tool on the raw file path. +# Extends the list from the global configuration, merging the two lists. +# Example: ["_archive/.*", "_episodes/.*"] +ignored_memory_patterns: [] + +# list of mode names to that are always to be included in the set of active modes +# The full set of modes to be activated is base_modes + default_modes. +# If the setting is undefined, the base_modes from the global configuration (serena_config.yml) apply. +# Otherwise, this setting overrides the global configuration. +# Set this to [] to disable base modes for this project. +# Set this to a list of mode names to always include the respective modes for this project. +base_modes: diff --git a/AGENTS.md b/AGENTS.md index 19dd3aa..e2588e8 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -92,21 +92,16 @@ Keep TypeDB interface tests split by dependency style: ## TypeDB Runtime Notes -Do not let `ensure_database_exists()` silently create an empty database. If the -configured database is missing, recreate it only by replaying stored schema/data -paths, or fail loudly when no initialization files are configured. Recovery code -that runs while `_database_query_lock` is held should use internal unlocked -helpers instead of calling the public `database_query()` path again; this avoids -recursive health checks and avoids needing `RLock`. +Do not let missing-database recovery silently create an empty database. Recreate +from configured schema/data paths, or fail loudly when no initialization files +are configured. `reload_schema=False` should skip schema redefinition only when reusing an existing database. Newly created, force-recreated, or missing-and-recreated databases must still load the configured schema before data is loaded. `delete_all_data()` must fail loudly if any delete step fails. Do not load new -data on top of potentially residual data after a failed cleanup. The current -live fixture confirms entity-first deletion works for the bundled schema/data, -but that does not prove the order is universal for every TypeDB schema. +data on top of potentially residual data after a failed cleanup. For TypeDB `fetch` queries, do not fetch a bare concept variable directly (`fetch $x;`). Fetch attributes explicitly; to fetch all attributes owned by a @@ -121,10 +116,7 @@ grouping them by concrete `(label, value_type)` before converting them to ROS `Attribute` messages. When changing ROS fetch-result conversion, cover both unit-level JSON/dict -shapes and launch integration tests against the real TypeDB service. The -existing integration fixture data already includes mixed attribute value types -on `person` (`string`, `long`, `double`, `boolean`, `datetime`) and is enough to -reproduce wildcard attribute conversion bugs without extending the fixture. +shapes and launch integration tests against the real TypeDB service. ROS service callbacks should catch and log query execution or result conversion exceptions, then return `success=False` with `error_message` populated. Do not @@ -133,57 +125,48 @@ let malformed or unexpected query results escape the callback and crash the ## ros_typedb_tools Stress Diagnostics -The stress experiment CLI is intentionally split by responsibility: - -- `ros_typedb_stress_experiment.py`: thin stress CLI entry point. -- `stress_config.py`: query specs, request records, validation, and Query - request construction. -- `stress_output.py`: summaries, result JSON, timeout JSON, and debug JSONL - writing. -- `stress_runner.py`: ROS client loop, executor handling, service readiness, - pending future processing, and timeout-barrier behavior. -- `stress_invariants.py`: invariant response evaluation and scalar extraction. -- `fake_query_service.py`: fake `Query` service used to isolate ROS - service-client behavior from TypeDB work. - When debugging Stage 2 stress timeouts, use the fake query service and the scenario runner before changing `ros_typedb_interface` behavior. If the fake service reproduces the same timeout-wave pattern, suspect ROS/rclpy/RMW/DDS or stress-harness behavior rather than TypeDB queries, schema/data, or result conversion. -Observed stress-test boundary: high-rate ROS `Query` service calls can produce -timeout waves even with fake services that do no TypeDB work. This reproduced -with Python and C++ clients/servers, Fast DDS and CycloneDDS, and both Humble -and Lyrical environments. Treat this as a ROS service transport/executor stress -limit, not as evidence of a `ros_typedb` database/query bug. For meaningful -`ros_typedb` robustness tests, cap outstanding service requests; in local -testing `--max-in-flight 10` with `--request-gap-s 0.01` was stable, while -roughly 12-15 in-flight requests started producing timeout waves. - -Move on from this transport limit unless the task is specifically to diagnose -ROS service middleware behavior. Later stress stages should run inside the -bounded envelope and focus on driver behavior: mixed read/write workloads, -lifecycle transitions under load, TypeDB restart/reconnect, malformed query -handling, schema/data reload behavior, and long-duration soak tests. +High-rate ROS `Query` service calls can produce timeout waves even with fake +services that do no TypeDB work. Treat this as a ROS service +transport/executor stress limit, not as evidence of a TypeDB/query bug. For +database robustness tests, stay inside the bounded envelope: in local testing +`--max-in-flight 10` with `--request-gap-s 0.01` was stable. Stress invariants are correctness checks that run alongside load to verify the -database still contains expected baseline facts. Keep invariant profiles as -packaged JSON files under `ros_typedb_tools/ros_typedb_tools/profiles/`, not as -hard-coded Python data. Update `setup.py` package data when adding new profile -file patterns. Current built-in profiles are: - -- `test-data`: for `ros_typedb/test/typedb_test_data/schema.tql` and - `data.tql`; checks `person`, `company`, `employment`, and the - `boss@tudelft.nl` sentinel. -- `plan-schema`: for `ros_typedb_examples/data/plan_schema.tql` and - `plan_data.tql`; checks `Plan`, `Action`, `Proposition`, plan/action - relations, and the `collect-water-sample` sentinel action. +database still contains expected baseline facts. Keep invariant and mixed +read/write profiles as packaged JSON files under +`ros_typedb_tools/ros_typedb_tools/profiles/`, not as hard-coded Python data. +Update `setup.py` package data when adding new profile file patterns. Do not use a built-in invariant profile with a different schema/data pair. A passing request-load test only proves the service answered; passing invariants prove the expected baseline data remained visible. +Stage 5 delete-database fault injection deletes the database through the real +`/delete_database` service while read load continues. The database is not +recreated by the delete service itself; recreation is lazy and happens when a +later `database_query()` calls `ensure_database_exists()`. That recovery only +works when the node was configured with valid schema/data paths. + +Stage 5 recovery should be measured from the delete fault trigger to the first +complete successful invariant pass after the fault. Periodic invariants during +active load are the primary recovery signal; the post-load recovery loop is +only a fallback when periodic checks did not observe recovery. Do not implement +recovery timing that starts only after the main load duration ends, because it +can falsely report `recovery: not recovered` even if ordinary post-delete +queries already recreated and reloaded the database. + +`ros_typedb_examples` can include launch files for reusable schema/data demos. +If a launch file needs data that normally lives under another package's `test/` +tree, install only the specific runtime fixture files through the examples +package `setup.py`; do not rely on source-tree-relative paths after +`colcon build`. + Useful stress diagnostics: ```bash @@ -201,10 +184,6 @@ If a `ros2 run` command still imports an old module after a source change, remove that package's generated `build/` and `install/` directories inside the workspace and rebuild the package. -For PR support notes under `docs-agents/`, keep changelogs brief: summary, -compact bullet list of changes, and verification commands. Avoid copying the -more verbose style from planning documents. - ## Commit & Pull Request Guidelines Recent history uses short imperative commits such as `fix lint` and diff --git a/CLAUDE.md b/CLAUDE.md new file mode 100644 index 0000000..c20f344 --- /dev/null +++ b/CLAUDE.md @@ -0,0 +1,46 @@ +# CLAUDE.md + +This file provides guidance to Claude Code (claude.ai/code) when working with code in this repository. + +## Serena + +At the start of every session, activate Serena for this project: + +```python +mcp__plugin_serena_serena__activate_project("/home/gus/ros_workspaces/suave_rebetmc_ws/src/ros_typedb") +mcp__plugin_serena_serena__initial_instructions() +``` + +Then read relevant memories (e.g. `project_overview`, `style_and_conventions`) before touching code. + +For ALL code exploration and editing, use Serena's symbolic tools instead of `Bash` grep/cat or `Read`: +- `get_symbols_overview` — survey a file's symbols +- `find_symbol` — locate a class/function by name path +- `find_referencing_symbols` — trace callers/usages +- `replace_symbol_body` — rewrite an entire symbol +- `replace_content` — targeted regex replacement within a file + +## Project guidelines + +All project conventions, TypeDB rules, testing guidelines, stress tooling notes, +architecture notes, commit guidelines, and security tips are maintained in **`AGENTS.md`** +at the repo root. Read that file now before touching any code. + +## ROSTypeDBInterface Gotchas + +**Lifecycle state check (rclpy Humble):** `get_current_state()` does not exist. The only +way to read lifecycle state programmatically is: +```python +node._state_machine.current_state[1] == 'active' +``` + +**Service naming in `__init__`:** When creating services inside `LifecycleNode.__init__`, +use `self.get_name()` — not the `node_name` constructor argument. ROS name remapping is +applied during `super().__init__()`, so `node_name` is the un-remapped name. +`self.get_name()` is correct immediately after `super().__init__()` returns. + +**`destroy_service()` race with `MultiThreadedExecutor`:** Calling `destroy_service()` from +`on_cleanup()` while the executor is spinning causes `InvalidHandle` — the executor accesses +service handles in `_take_service()` before any application-level lock. Design pattern: +create services once in `__init__`, never destroy them; gate work in callbacks by checking +lifecycle state and `typedb_interface is None` inside `_service_callback_lock`. From 5665602085647757bf73b8479ed4bca81c57f292 Mon Sep 17 00:00:00 2001 From: Gustavo Date: Sun, 28 Jun 2026 19:34:09 -0300 Subject: [PATCH 32/36] mv "stress" test to new ros_typedb_benchmark pkg --- ros_typedb_benchmark/LICENSE | 202 +++++++ ros_typedb_benchmark/README.md | 492 +++++++++++++++++ ros_typedb_benchmark/package.xml | 21 + .../resource/ros_typedb_benchmark | 0 .../ros_typedb_benchmark/__init__.py | 0 .../fake_query_service.py | 0 .../profiles/plan_schema_invariants.json | 0 .../profiles/plan_schema_mixed.json | 0 .../plan_schema_mixed_invariants.json | 0 .../profiles/test_data_invariants.json | 0 .../profiles/test_data_mixed.json | 0 .../ros_typedb_stress_experiment.py | 28 +- .../ros_typedb_benchmark}/stress_config.py | 2 +- .../stress_invariants.py | 4 +- .../ros_typedb_benchmark}/stress_mixed.py | 6 +- .../ros_typedb_benchmark}/stress_output.py | 8 +- .../stress_resolution.py | 18 +- .../ros_typedb_benchmark}/stress_runner.py | 32 +- .../scripts/run_stage2_timeout_scenarios.sh | 4 +- ros_typedb_benchmark/setup.cfg | 4 + ros_typedb_benchmark/setup.py | 42 ++ ros_typedb_benchmark/test/test_copyright.py | 25 + ros_typedb_benchmark/test/test_flake8.py | 25 + ros_typedb_benchmark/test/test_pep257.py | 23 + .../test/test_ros_typedb_stress_experiment.py | 32 +- ros_typedb_tools/README.md | 493 +----------------- ros_typedb_tools/package.xml | 2 - ros_typedb_tools/setup.py | 11 - 28 files changed, 903 insertions(+), 571 deletions(-) create mode 100644 ros_typedb_benchmark/LICENSE create mode 100644 ros_typedb_benchmark/README.md create mode 100644 ros_typedb_benchmark/package.xml create mode 100644 ros_typedb_benchmark/resource/ros_typedb_benchmark create mode 100644 ros_typedb_benchmark/ros_typedb_benchmark/__init__.py rename {ros_typedb_tools/ros_typedb_tools => ros_typedb_benchmark/ros_typedb_benchmark}/fake_query_service.py (100%) rename {ros_typedb_tools/ros_typedb_tools => ros_typedb_benchmark/ros_typedb_benchmark}/profiles/plan_schema_invariants.json (100%) rename {ros_typedb_tools/ros_typedb_tools => ros_typedb_benchmark/ros_typedb_benchmark}/profiles/plan_schema_mixed.json (100%) rename {ros_typedb_tools/ros_typedb_tools => ros_typedb_benchmark/ros_typedb_benchmark}/profiles/plan_schema_mixed_invariants.json (100%) rename {ros_typedb_tools/ros_typedb_tools => ros_typedb_benchmark/ros_typedb_benchmark}/profiles/test_data_invariants.json (100%) rename {ros_typedb_tools/ros_typedb_tools => ros_typedb_benchmark/ros_typedb_benchmark}/profiles/test_data_mixed.json (100%) rename {ros_typedb_tools/ros_typedb_tools => ros_typedb_benchmark/ros_typedb_benchmark}/ros_typedb_stress_experiment.py (93%) rename {ros_typedb_tools/ros_typedb_tools => ros_typedb_benchmark/ros_typedb_benchmark}/stress_config.py (99%) rename {ros_typedb_tools/ros_typedb_tools => ros_typedb_benchmark/ros_typedb_benchmark}/stress_invariants.py (96%) rename {ros_typedb_tools/ros_typedb_tools => ros_typedb_benchmark/ros_typedb_benchmark}/stress_mixed.py (96%) rename {ros_typedb_tools/ros_typedb_tools => ros_typedb_benchmark/ros_typedb_benchmark}/stress_output.py (98%) rename {ros_typedb_tools/ros_typedb_tools => ros_typedb_benchmark/ros_typedb_benchmark}/stress_resolution.py (88%) rename {ros_typedb_tools/ros_typedb_tools => ros_typedb_benchmark/ros_typedb_benchmark}/stress_runner.py (97%) rename {ros_typedb_tools => ros_typedb_benchmark}/scripts/run_stage2_timeout_scenarios.sh (93%) create mode 100644 ros_typedb_benchmark/setup.cfg create mode 100644 ros_typedb_benchmark/setup.py create mode 100644 ros_typedb_benchmark/test/test_copyright.py create mode 100644 ros_typedb_benchmark/test/test_flake8.py create mode 100644 ros_typedb_benchmark/test/test_pep257.py rename {ros_typedb_tools => ros_typedb_benchmark}/test/test_ros_typedb_stress_experiment.py (97%) diff --git a/ros_typedb_benchmark/LICENSE b/ros_typedb_benchmark/LICENSE new file mode 100644 index 0000000..d645695 --- /dev/null +++ b/ros_typedb_benchmark/LICENSE @@ -0,0 +1,202 @@ + + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + + END OF TERMS AND CONDITIONS + + APPENDIX: How to apply the Apache License to your work. + + To apply the Apache License to your work, attach the following + boilerplate notice, with the fields enclosed by brackets "[]" + replaced with your own identifying information. (Don't include + the brackets!) The text should be enclosed in the appropriate + comment syntax for the file format. We also recommend that a + file or class name and description of purpose be included on the + same "printed page" as the copyright notice for easier + identification within third-party archives. + + Copyright [yyyy] [name of copyright owner] + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. diff --git a/ros_typedb_benchmark/README.md b/ros_typedb_benchmark/README.md new file mode 100644 index 0000000..98feae3 --- /dev/null +++ b/ros_typedb_benchmark/README.md @@ -0,0 +1,492 @@ +# ros_typedb_benchmark + +Stress-testing, fault injection, and benchmark tools for `ros_typedb`. + +## ros_typedb_stress_experiment + +Runs fixed-count or duration-based stress experiments against the real +`/ros_typedb_interface/query` service. + +Prerequisites: + +- TypeDB is running. +- `ros_typedb_interface` is running. +- The lifecycle node has been configured so the query service exists. + +### Experiment Stages + +The stress tool is organized as staged experiments. Earlier stages are useful +on their own; later stages build on the same result format and command-line +interface. + +| Stage | Status | Purpose | +| --- | --- | --- | +| 1. Minimal real service load | Implemented | Send repeated requests to the real ROS `Query` service and measure latency, success, errors, and timeouts. | +| 2. Concurrent client stress | Implemented | Run many ROS service clients concurrently to find service/executor transport limits before blaming TypeDB or driver code. | +| 3. Correctness invariants | Implemented | Check that expected baseline facts remain visible during and after load, not only that requests returned. | +| 4. Mixed read/write load | Implemented | Read while inserting, updating, deleting, and cleaning up experiment-owned temporary data. | +| 5. Delete-database fault injection | Implemented | Delete the database during load and measure whether the driver fails loudly, reloads correctly, and recovers invariants. | +| 6. TypeDB restart fault injection | Implemented | Stop/restart TypeDB during load and measure outage, reconnect behavior, and post-recovery correctness. | +| 7. Lifecycle stress | Implemented | Trigger ROS lifecycle cleanup/configure/activate transitions during load to check teardown races. | +| 8. External schema/data profiles | Planned | Run the same harness with application-specific schema/data, query mixes, invariants, setup, and cleanup profiles. | + +Use Stage 1 when validating a single query path. Use Stage 2 when probing ROS +service throughput and timeout behavior. Use Stage 3 when checking that load +does not corrupt or hide baseline data. Use Stage 4 when validating robustness +under real read/write activity. Use Stage 5 when validating recovery after a +database deletion fault. Use Stage 6 when validating driver reconnect behavior +after the TypeDB server process or container is restarted. Use Stage 7 when +validating that ROS lifecycle cleanup waits for active query callbacks before +tearing down TypeDB resources and that the node can be configured and activated +again afterward. + +### Stage 1: Minimal Real Service Load + +Example: + +```bash +ros2 run ros_typedb_benchmark ros_typedb_stress_experiment \ + --query 'match $p isa Plan; (plan: $p, action: $a) isa has_action; fetch $a: attribute;' \ + --query-type fetch \ + --requests 2000 \ + --timeout-s 5 \ + --output ~/results/ros_typedb_stress_results.json +``` + +This is the smallest useful run. It is best for confirming that the service, +query type, timeout handling, result conversion, and JSON output work for one +known query before adding concurrency. + +### Stage 2: Concurrent Client Stress + +Concurrent read stress example: + +```bash +ros2 run ros_typedb_benchmark ros_typedb_stress_experiment \ + --clients 20 \ + --duration-s 60 \ + --timeout-s 10 \ + --mode read \ + --debug-events-output ~/results/ros_typedb_read_stress_events.jsonl \ + --output ~/results/ros_typedb_read_stress_results.json +``` + +```bash +ros2 run ros_typedb_benchmark ros_typedb_stress_experiment \ + --clients 20 \ + --duration-s 60 \ + --timeout-s 10 \ + --query 'match $x isa entity; get $x;' \ + --query-type get \ + --debug-events-output ~/results/ros_typedb_read_stress_events.jsonl \ + --output ~/results/ros_typedb_get_stress_results.json +``` + +This stage is for separating ROS service transport behavior from database +behavior. It reports throughput and latency percentiles across concurrent +clients and can write debug JSONL events for request/future/executor analysis. + +When `--output` is set, timeout-only records are also written next to the main +result file using the suffix `_timeouts.json`. Use `--timeout-output` to choose +a different path. Timeout records include request index, client id, query index, +query type, query text, latency, and whether future cancellation was requested. +The command exits successfully when the experiment runs to completion, even if +some requests fail. Use `--fail-on-failure` when a nonzero exit code is useful +for CI or scripted thresholds. + +### Stage 3: Correctness Invariants + +An invariant is a correctness check that should remain true while the stress +load is running. Request metrics show whether the service answered; invariants +show whether the database still contains expected baseline facts. + +Correctness invariant example for the bundled `ros_typedb` test schema/data: + +```bash +ros2 run ros_typedb_benchmark ros_typedb_stress_experiment \ + --clients 10 \ + --duration-s 60 \ + --timeout-s 10 \ + --mode read \ + --invariant-profile test-data \ + --output ~/results/ros_typedb_read_invariants_results.json +``` + +Use `--invariant-profile test-data` only when `ros_typedb_interface` was started +with the schema/data from `ros_typedb/test/typedb_test_data/`. The profile is +stored in `ros_typedb_benchmark/ros_typedb_benchmark/profiles/test_data_invariants.json` +and checks stable aggregate counts for `person`, `company`, and `employment`, +plus the `boss@tudelft.nl` sentinel person. This profile is not appropriate for +other schemas, such as the `ros_typedb_examples` plan schema. + +For `ros_typedb_examples/data/plan_schema.tql` and `plan_data.tql`, use: + +```bash +ros2 run ros_typedb_benchmark ros_typedb_stress_experiment \ + --clients 10 \ + --duration-s 60 \ + --timeout-s 10 \ + --mode read \ + --invariant-profile plan-schema \ + --output ~/results/ros_typedb_plan_invariants_results.json +``` + +The `plan-schema` profile is stored in +`ros_typedb_benchmark/ros_typedb_benchmark/profiles/plan_schema_invariants.json` and +checks counts for `Plan`, `Action`, `Proposition`, and the plan/action relation +types, plus a sentinel action named `collect-water-sample`. + +Checks run every 10 seconds by default and once at the end. Use +`--invariant-period-s` to adjust the periodic interval, or +`--invariant-period-s 0` to run only the final check. Invariant failures are +reported separately from request failures and always make the command exit with +status 1. + +### Stage 4: Mixed Read/Write Load + +Mixed read/write stress for the bundled `ros_typedb` test schema: + +```bash +ros2 run ros_typedb_benchmark ros_typedb_stress_experiment \ + --clients 10 \ + --duration-s 120 \ + --timeout-s 10 \ + --mode mixed \ + --invariant-profile test-data \ + --request-gap-s 0.01 \ + --max-in-flight 10 \ + --output ~/results/ros_typedb_mixed_stress_results.json +``` + +The packaged `test-data` mixed profile reads the database while inserting, +updating, and deleting experiment-owned temporary `robot` entities. It is +intended for the schema in `ros_typedb/test/typedb_test_data/schema.tql` and +cleans up remaining temporary robots before the final invariant check. + +Mixed read/write stress for the `ros_typedb_examples` plan schema: + +```bash +ros2 run ros_typedb_benchmark ros_typedb_stress_experiment \ + --clients 10 \ + --duration-s 120 \ + --timeout-s 10 \ + --mode mixed \ + --mixed-profile plan-schema \ + --invariant-profile plan-schema-mixed \ + --request-gap-s 0.01 \ + --max-in-flight 10 \ + --output ~/results/ros_typedb_plan_mixed_stress_results.json +``` + +Mixed profiles are packaged JSON files under +`ros_typedb_benchmark/ros_typedb_benchmark/profiles/`. The `plan-schema` mixed profile +writes temporary `Action` entities, so use `plan-schema-mixed` invariants when +periodic invariant checks run during that mixed workload. That invariant +profile keeps stable checks for plans, propositions, relations, and the +`collect-water-sample` sentinel action without requiring an exact total +`Action` count while temporary actions may exist. + +### Stage 5: Delete-Database Fault Injection + +This stage calls the real `delete_database` service while normal query clients +are still active. It is meant to check that a missing database does not produce +silent success against an uninitialized empty database. After the delete call, +periodic invariant passes during the active load record recovery as soon as all +checks pass once. If periodic checks do not observe recovery, the runner uses a +post-load recovery check until invariants pass or `--fault-recovery-timeout-s` +expires. + +For a reproducible run, start the bundled test-data launch file in one terminal. +It configures and activates `/ros_typedb_interface`, loads +`ros_typedb/test/typedb_test_data/schema.tql` and `data.tql`, and leaves the +schema/data paths available for recovery: + +```bash +docker exec -it ros_typedb bash -lc ' +cd /home/ubuntu-user/typedb_ws && +source /opt/ros/humble/setup.bash && +source install/setup.bash && +ros2 launch ros_typedb_examples test_data_example.launch.py' +``` + +Then run the fault-injection experiment from another terminal: + +```bash +docker exec -it ros_typedb bash -lc ' +cd /home/ubuntu-user/typedb_ws && +source /opt/ros/humble/setup.bash && +source install/setup.bash && +ros2 run ros_typedb_benchmark ros_typedb_stress_experiment \ + --clients 10 \ + --duration-s 180 \ + --timeout-s 10 \ + --mode read \ + --fault delete-database \ + --fault-at-s 60 \ + --invariant-profile test-data \ + --request-gap-s 0.01 \ + --max-in-flight 10 \ + --output /tmp/stage5_delete_database.json \ + --debug-events-output /tmp/stage5_delete_database_debug.jsonl' +``` + +The result JSON always includes a `fault` object with: + +- `fault`: configured fault name, such as `delete-database` +- `fault_at_s`: configured trigger time +- `fault_triggered_at_s`: monotonic timestamp when the fault was triggered +- `fault_delete_success`, `fault_delete_error`, and `fault_delete_latency_s` +- `fault_recovered_at_s`: first timestamp when all recovery invariants passed +- `fault_recovery_s`: elapsed time from trigger to successful recovery + +If the fault is triggered but invariants do not recover, the CLI exits with +status 1. Keep `--invariant-profile test-data` paired with the bundled +test-data launch; use a matching invariant profile for any other schema/data +pair. Keep `--max-in-flight 10` and `--request-gap-s 0.01` for robustness +testing so the experiment stays inside the known stable ROS service envelope. + +### Stage 6: TypeDB Restart Fault Injection + +This stage restarts the TypeDB server while normal query clients continue +sending requests. It is meant to check that server outages produce bounded, +explicit failures and that later requests reconnect to TypeDB without +restarting the ROS lifecycle node. Recovery is still correctness-based: the +first complete post-fault invariant pass marks `fault_recovered_at_s`. + +Run TypeDB separately from the ROS node and stress process. Do not pass the +same Docker container that is running `ros_typedb_interface` or the experiment +to `--typedb-container`, because stopping that container stops the test itself. +By default, the process fault uses `pkill -f "typedb/core/server"` to stop +TypeDB and `typedb server` to start it again: + +```bash +ros2 run ros_typedb_benchmark ros_typedb_stress_experiment \ + --clients 10 \ + --duration-s 240 \ + --timeout-s 10 \ + --mode read \ + --fault restart-typedb \ + --typedb-restart-delay-s 2 \ + --fault-at-s 60 \ + --invariant-profile test-data \ + --request-gap-s 0.01 \ + --max-in-flight 10 \ + --output /tmp/stage6_restart_typedb.json \ + --debug-events-output /tmp/stage6_restart_typedb_debug.jsonl +``` + +Override the process commands with `--typedb-stop-command` and +`--typedb-start-command` when needed. For a TypeDB-only Docker container, pass +`--typedb-container`; it overrides the process commands: + +```bash +--typedb-container typedb_server +``` + +The restart controller runs in a background thread, so the main experiment loop +keeps spinning ROS futures and sending requests during the outage. The result +JSON adds restart fields to the `fault` object: + +- `fault_restart_stop_success`, `fault_restart_stop_error`, and + `fault_restart_stop_latency_s` +- `fault_restart_start_success`, `fault_restart_start_error`, and + `fault_restart_start_latency_s` +- `fault_restart_delay_s`: configured delay between stop and start +- `fault_restart_outage_s`: elapsed controller time from stop start to start + completion +- `fault_observed_outage_s`: elapsed time from the first failed post-fault + request result to the first later successful request result + +If the fault is triggered but invariants do not recover, the CLI exits with +status 1. Request failures during the outage remain recorded in the JSON output; +use `--fail-on-failure` only when you want any request failure to fail the +command even if invariants later recover. + +### Stage 7: Lifecycle Stress + +This stage calls the ROS lifecycle `change_state` service while normal query +clients continue sending requests. The default sequence is deactivate, cleanup, +configure, then activate. Cleanup destroys the query and delete services and +waits for active service callbacks before closing TypeDB resources; the +experiment verifies that this does not race with in-flight requests and that +the node recovers after reactivation. The controller checks `get_state` after +each transition and only sends the next transition after the expected lifecycle +state is observed. + +When using a launch file that automatically reactivates the node after every +deactivate, disable that behavior for this experiment. The bundled test-data +launch file defaults to normal auto-reactivation, so start it for Stage 7 with: + +```bash +ros2 launch ros_typedb_examples test_data_example.launch.py \ + reactivate_on_deactivate:=False +``` + +Then run the stress experiment: + +```bash +ros2 run ros_typedb_benchmark ros_typedb_stress_experiment \ + --clients 10 \ + --duration-s 180 \ + --timeout-s 10 \ + --mode read \ + --fault lifecycle-cleanup \ + --fault-at-s 60 \ + --invariant-profile test-data \ + --request-gap-s 0.01 \ + --max-in-flight 10 \ + --output /tmp/stage7_lifecycle_cleanup.json \ + --debug-events-output /tmp/stage7_lifecycle_cleanup_debug.jsonl +``` + +The lifecycle `change_state` and `get_state` services default to the +`--service-name` prefix with `/change_state` and `/get_state` appended. For +example, the default query service `/ros_typedb_interface/query` maps to +`/ros_typedb_interface/change_state` and `/ros_typedb_interface/get_state`. +Override them with `--lifecycle-change-state-service-name` and +`--lifecycle-get-state-service-name` when the launched lifecycle node uses a +different name, such as `/ros_typedb/change_state` and +`/ros_typedb/get_state`. + +By default the node is configured and activated again after cleanup. Use +`--no-lifecycle-reactivate` only when you intentionally want to leave the node +cleaned up; in that mode recovery invariants are not expected to pass unless +something else reactivates the node. + +The result JSON adds lifecycle fields to the `fault` object: + +- `fault_lifecycle_deactivate_success`, `fault_lifecycle_deactivate_error`, + and `fault_lifecycle_deactivate_latency_s` +- `fault_lifecycle_cleanup_success`, `fault_lifecycle_cleanup_error`, and + `fault_lifecycle_cleanup_latency_s` +- `fault_lifecycle_configure_success`, `fault_lifecycle_configure_error`, and + `fault_lifecycle_configure_latency_s` +- `fault_lifecycle_activate_success`, `fault_lifecycle_activate_error`, and + `fault_lifecycle_activate_latency_s` +- `fault_lifecycle_reactivate`: whether configure/activate was requested +- `fault_observed_outage_s`: elapsed time from the first failed post-fault + request result to the first later successful request result + +If a lifecycle transition is rejected or times out, the CLI exits with status +1. If transitions succeed but invariants do not recover, the CLI also exits +with status 1. Request failures while the lifecycle node is inactive or cleaned +up remain recorded in the JSON output. + +### Planned Fault-Injection Stages + +Stage 8 is not implemented yet. It is a placeholder for the next robustness +check: + +- Stage 8, external profiles: load schema-specific query mixes, invariants, + setup, and cleanup from user-provided JSON/YAML profile files instead of only + the packaged examples. + +### Diagnostics and Boundaries + +For ROS client/executor debugging, `--debug-events-output` writes JSONL events +for request sends, `spin_once()` calls, completed futures, timeouts, and periodic +pending-request snapshots. Use `--executor single` or `--executor multi` to +compare explicit executors against the default `rclpy.spin_once(node)` behavior. +Use `--request-gap-s` to add a small per-client delay between requests, or +`--max-in-flight` to cap total outstanding service requests independently of the +number of clients. + +For `ros_typedb` robustness experiments, prefer bounded service concurrency. +High-rate unbounded Query service calls produced timeout waves even when TypeDB +was removed from the path by using fake Query services. The same behavior was +observed with Python and C++ clients/servers, Fast DDS and CycloneDDS, and ROS +Humble and Lyrical. Treat that as a ROS service transport/executor stress limit, +not as a database or result-conversion failure. + +A practical bounded baseline is: + +```bash +ros2 run ros_typedb_benchmark ros_typedb_stress_experiment \ + --clients 20 \ + --duration-s 60 \ + --timeout-s 10 \ + --mode read \ + --request-gap-s 0.01 \ + --max-in-flight 10 \ + --output ~/results/ros_typedb_read_bounded_results.json +``` + +Use larger `--max-in-flight` values only when intentionally probing ROS service +transport saturation. In local tests, about 12-15 in-flight requests started to +produce timeout waves, while 10 or fewer stayed stable. + +To isolate ROS service/client behavior from TypeDB work, run a fake Query +service in one terminal: + +```bash +ros2 run ros_typedb_benchmark ros_typedb_fake_query_service +``` + +Then run the same stress command against it from another terminal. + +To run the common timeout-debug scenarios in sequence, use: + +```bash +bash src/ros_typedb/ros_typedb_benchmark/scripts/run_stage2_timeout_scenarios.sh +``` + +The script runs real read/global, real read/multi, fake read/global, fake +read/multi, and real explicit entity-get/global scenarios. Override defaults +with environment variables such as `CLIENTS`, `DURATION_S`, `TIMEOUT_S`, and +`OUTPUT_DIR`. + +Useful options: + +- `--service-name`: query service name. Defaults to `/ros_typedb_interface/query` +- `--query`: optional TypeDB query to send on each request +- `--query-type`: one of `define`, `delete`, `fetch`, `get`, + `get_aggregate`, `insert`, or `update`; required when `--query` is used +- `--requests`: number of requests to send when `--duration-s` is omitted +- `--clients`: number of concurrent service clients +- `--duration-s`: run for this many seconds instead of a fixed request count +- `--mode`: built-in query mix to use when `--query` is omitted; supports + `read` and `mixed` +- `--mixed-profile`: schema-specific mixed-mode profile, such as `test-data` + or `plan-schema`; `auto` derives it from the invariant profile when possible +- `--request-gap-s`: minimum delay between requests from the same client +- `--max-in-flight`: maximum number of outstanding requests across all clients +- `--executor`: client executor mode, one of `global`, `single`, or `multi` +- `--debug-events-output`: optional JSONL debug event path +- `--invariant-profile`: optional correctness profile, such as `test-data`, + `plan-schema`, or `plan-schema-mixed` +- `--invariant-period-s`: seconds between periodic invariant checks +- `--timeout-s`: per-request client and server timeout +- `--output`: optional JSON results path +- `--timeout-output`: optional timeout-only JSON results path +- `--fail-on-failure`: return exit code 1 when any request fails + +## Tests + +```Bash +colcon test --event-handlers console_cohesion+ --packages-select ros_typedb_benchmark +``` + +## Docker + +The commands below assume you are running them from the workspace root: + +```Bash +cd +``` + +Start dev container **without** display and the `ros_typedb` directory mounted: + +```Bash +docker run -it --rm --name ros_typedb -v /etc/localtime:/etc/localtime:ro -v $PWD/src/ros_typedb:/home/ubuntu-user/typedb_ws/src/ros_typedb -v $PWD/ros_typedb_results/:/home/ubuntu-user/results/ ros_typedb +``` + +Start new terminal in the container: + +```Bash +docker exec -it ros_typedb bash +``` + + + export RMW_IMPLEMENTATION=rmw_fastrtps_cpp + +export RMW_IMPLEMENTATION=rmw_cyclonedds_cpp diff --git a/ros_typedb_benchmark/package.xml b/ros_typedb_benchmark/package.xml new file mode 100644 index 0000000..ee54723 --- /dev/null +++ b/ros_typedb_benchmark/package.xml @@ -0,0 +1,21 @@ + + + + ros_typedb_benchmark + 0.0.0 + Stress-testing and fault-injection benchmark tools for ros_typedb. + Gustavo Rezende + Apache-2.0 + + rclpy + ros_typedb_msgs + + ament_copyright + ament_flake8 + ament_pep257 + python3-pytest + + + ament_python + + diff --git a/ros_typedb_benchmark/resource/ros_typedb_benchmark b/ros_typedb_benchmark/resource/ros_typedb_benchmark new file mode 100644 index 0000000..e69de29 diff --git a/ros_typedb_benchmark/ros_typedb_benchmark/__init__.py b/ros_typedb_benchmark/ros_typedb_benchmark/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/ros_typedb_tools/ros_typedb_tools/fake_query_service.py b/ros_typedb_benchmark/ros_typedb_benchmark/fake_query_service.py similarity index 100% rename from ros_typedb_tools/ros_typedb_tools/fake_query_service.py rename to ros_typedb_benchmark/ros_typedb_benchmark/fake_query_service.py diff --git a/ros_typedb_tools/ros_typedb_tools/profiles/plan_schema_invariants.json b/ros_typedb_benchmark/ros_typedb_benchmark/profiles/plan_schema_invariants.json similarity index 100% rename from ros_typedb_tools/ros_typedb_tools/profiles/plan_schema_invariants.json rename to ros_typedb_benchmark/ros_typedb_benchmark/profiles/plan_schema_invariants.json diff --git a/ros_typedb_tools/ros_typedb_tools/profiles/plan_schema_mixed.json b/ros_typedb_benchmark/ros_typedb_benchmark/profiles/plan_schema_mixed.json similarity index 100% rename from ros_typedb_tools/ros_typedb_tools/profiles/plan_schema_mixed.json rename to ros_typedb_benchmark/ros_typedb_benchmark/profiles/plan_schema_mixed.json diff --git a/ros_typedb_tools/ros_typedb_tools/profiles/plan_schema_mixed_invariants.json b/ros_typedb_benchmark/ros_typedb_benchmark/profiles/plan_schema_mixed_invariants.json similarity index 100% rename from ros_typedb_tools/ros_typedb_tools/profiles/plan_schema_mixed_invariants.json rename to ros_typedb_benchmark/ros_typedb_benchmark/profiles/plan_schema_mixed_invariants.json diff --git a/ros_typedb_tools/ros_typedb_tools/profiles/test_data_invariants.json b/ros_typedb_benchmark/ros_typedb_benchmark/profiles/test_data_invariants.json similarity index 100% rename from ros_typedb_tools/ros_typedb_tools/profiles/test_data_invariants.json rename to ros_typedb_benchmark/ros_typedb_benchmark/profiles/test_data_invariants.json diff --git a/ros_typedb_tools/ros_typedb_tools/profiles/test_data_mixed.json b/ros_typedb_benchmark/ros_typedb_benchmark/profiles/test_data_mixed.json similarity index 100% rename from ros_typedb_tools/ros_typedb_tools/profiles/test_data_mixed.json rename to ros_typedb_benchmark/ros_typedb_benchmark/profiles/test_data_mixed.json diff --git a/ros_typedb_tools/ros_typedb_tools/ros_typedb_stress_experiment.py b/ros_typedb_benchmark/ros_typedb_benchmark/ros_typedb_stress_experiment.py similarity index 93% rename from ros_typedb_tools/ros_typedb_tools/ros_typedb_stress_experiment.py rename to ros_typedb_benchmark/ros_typedb_benchmark/ros_typedb_stress_experiment.py index 720e8aa..bc4cea8 100644 --- a/ros_typedb_tools/ros_typedb_tools/ros_typedb_stress_experiment.py +++ b/ros_typedb_benchmark/ros_typedb_benchmark/ros_typedb_stress_experiment.py @@ -22,20 +22,20 @@ import rclpy -from ros_typedb_tools.stress_config import INVARIANT_PROFILE_NAMES -from ros_typedb_tools.stress_config import MIXED_PROFILE_NAMES -from ros_typedb_tools.stress_config import QUERY_TYPE_BY_NAME -from ros_typedb_tools.stress_config import DEFAULT_TYPEDB_START_COMMAND -from ros_typedb_tools.stress_config import DEFAULT_TYPEDB_STOP_COMMAND -from ros_typedb_tools.stress_config import InvariantRecord -from ros_typedb_tools.stress_config import StressExperimentResult -from ros_typedb_tools.stress_output import default_timeout_output_path -from ros_typedb_tools.stress_output import print_summary -from ros_typedb_tools.stress_output import summarize_invariant_records -from ros_typedb_tools.stress_output import summarize_records -from ros_typedb_tools.stress_output import write_results -from ros_typedb_tools.stress_output import write_timeout_results -from ros_typedb_tools.stress_runner import run_experiment +from ros_typedb_benchmark.stress_config import INVARIANT_PROFILE_NAMES +from ros_typedb_benchmark.stress_config import MIXED_PROFILE_NAMES +from ros_typedb_benchmark.stress_config import QUERY_TYPE_BY_NAME +from ros_typedb_benchmark.stress_config import DEFAULT_TYPEDB_START_COMMAND +from ros_typedb_benchmark.stress_config import DEFAULT_TYPEDB_STOP_COMMAND +from ros_typedb_benchmark.stress_config import InvariantRecord +from ros_typedb_benchmark.stress_config import StressExperimentResult +from ros_typedb_benchmark.stress_output import default_timeout_output_path +from ros_typedb_benchmark.stress_output import print_summary +from ros_typedb_benchmark.stress_output import summarize_invariant_records +from ros_typedb_benchmark.stress_output import summarize_records +from ros_typedb_benchmark.stress_output import write_results +from ros_typedb_benchmark.stress_output import write_timeout_results +from ros_typedb_benchmark.stress_runner import run_experiment def build_argument_parser() -> argparse.ArgumentParser: diff --git a/ros_typedb_tools/ros_typedb_tools/stress_config.py b/ros_typedb_benchmark/ros_typedb_benchmark/stress_config.py similarity index 99% rename from ros_typedb_tools/ros_typedb_tools/stress_config.py rename to ros_typedb_benchmark/ros_typedb_benchmark/stress_config.py index 8bc3f0e..3d65bd7 100644 --- a/ros_typedb_tools/ros_typedb_tools/stress_config.py +++ b/ros_typedb_benchmark/ros_typedb_benchmark/stress_config.py @@ -25,7 +25,7 @@ from ros_typedb_msgs.srv import Query if TYPE_CHECKING: - from ros_typedb_tools.stress_mixed import MixedQueryProfile + from ros_typedb_benchmark.stress_mixed import MixedQueryProfile QUERY_TYPE_BY_NAME = { diff --git a/ros_typedb_tools/ros_typedb_tools/stress_invariants.py b/ros_typedb_benchmark/ros_typedb_benchmark/stress_invariants.py similarity index 96% rename from ros_typedb_tools/ros_typedb_tools/stress_invariants.py rename to ros_typedb_benchmark/ros_typedb_benchmark/stress_invariants.py index 98ba879..d4e6a1e 100644 --- a/ros_typedb_tools/ros_typedb_tools/stress_invariants.py +++ b/ros_typedb_benchmark/ros_typedb_benchmark/stress_invariants.py @@ -20,8 +20,8 @@ from rcl_interfaces.msg import ParameterType -from ros_typedb_tools.stress_config import InvariantRecord -from ros_typedb_tools.stress_config import InvariantSpec +from ros_typedb_benchmark.stress_config import InvariantRecord +from ros_typedb_benchmark.stress_config import InvariantSpec def _parameter_value_to_python(parameter_value: Any) -> Any: diff --git a/ros_typedb_tools/ros_typedb_tools/stress_mixed.py b/ros_typedb_benchmark/ros_typedb_benchmark/stress_mixed.py similarity index 96% rename from ros_typedb_tools/ros_typedb_tools/stress_mixed.py rename to ros_typedb_benchmark/ros_typedb_benchmark/stress_mixed.py index a0ec6de..8bd468f 100644 --- a/ros_typedb_tools/ros_typedb_tools/stress_mixed.py +++ b/ros_typedb_benchmark/ros_typedb_benchmark/stress_mixed.py @@ -20,9 +20,9 @@ from pathlib import Path from typing import Any -from ros_typedb_tools.stress_config import MIXED_PROFILE_FILES -from ros_typedb_tools.stress_config import QUERY_TYPE_BY_NAME -from ros_typedb_tools.stress_config import QuerySpec +from ros_typedb_benchmark.stress_config import MIXED_PROFILE_FILES +from ros_typedb_benchmark.stress_config import QUERY_TYPE_BY_NAME +from ros_typedb_benchmark.stress_config import QuerySpec MIXED_PROFILE_DIR = Path(__file__).with_name('profiles') diff --git a/ros_typedb_tools/ros_typedb_tools/stress_output.py b/ros_typedb_benchmark/ros_typedb_benchmark/stress_output.py similarity index 98% rename from ros_typedb_tools/ros_typedb_tools/stress_output.py rename to ros_typedb_benchmark/ros_typedb_benchmark/stress_output.py index a466a28..a669e70 100644 --- a/ros_typedb_tools/ros_typedb_tools/stress_output.py +++ b/ros_typedb_benchmark/ros_typedb_benchmark/stress_output.py @@ -24,10 +24,10 @@ import time from typing import Any -from ros_typedb_tools.stress_config import FaultResult -from ros_typedb_tools.stress_config import InvariantRecord -from ros_typedb_tools.stress_config import QuerySpec -from ros_typedb_tools.stress_config import RequestRecord +from ros_typedb_benchmark.stress_config import FaultResult +from ros_typedb_benchmark.stress_config import InvariantRecord +from ros_typedb_benchmark.stress_config import QuerySpec +from ros_typedb_benchmark.stress_config import RequestRecord def _latency_percentile( diff --git a/ros_typedb_tools/ros_typedb_tools/stress_resolution.py b/ros_typedb_benchmark/ros_typedb_benchmark/stress_resolution.py similarity index 88% rename from ros_typedb_tools/ros_typedb_tools/stress_resolution.py rename to ros_typedb_benchmark/ros_typedb_benchmark/stress_resolution.py index e2bb7af..4962678 100644 --- a/ros_typedb_tools/ros_typedb_tools/stress_resolution.py +++ b/ros_typedb_benchmark/ros_typedb_benchmark/stress_resolution.py @@ -18,15 +18,15 @@ import argparse from pathlib import Path -from ros_typedb_tools.stress_config import build_invariant_specs -from ros_typedb_tools.stress_config import build_query_specs -from ros_typedb_tools.stress_config import DEFAULT_TYPEDB_START_COMMAND -from ros_typedb_tools.stress_config import DEFAULT_TYPEDB_STOP_COMMAND -from ros_typedb_tools.stress_config import QuerySpec -from ros_typedb_tools.stress_config import ResolvedStressConfig -from ros_typedb_tools.stress_config import validate_experiment_args -from ros_typedb_tools.stress_mixed import load_mixed_query_profile -from ros_typedb_tools.stress_mixed import MixedQueryProfile +from ros_typedb_benchmark.stress_config import build_invariant_specs +from ros_typedb_benchmark.stress_config import build_query_specs +from ros_typedb_benchmark.stress_config import DEFAULT_TYPEDB_START_COMMAND +from ros_typedb_benchmark.stress_config import DEFAULT_TYPEDB_STOP_COMMAND +from ros_typedb_benchmark.stress_config import QuerySpec +from ros_typedb_benchmark.stress_config import ResolvedStressConfig +from ros_typedb_benchmark.stress_config import validate_experiment_args +from ros_typedb_benchmark.stress_mixed import load_mixed_query_profile +from ros_typedb_benchmark.stress_mixed import MixedQueryProfile def resolve_stress_config( diff --git a/ros_typedb_tools/ros_typedb_tools/stress_runner.py b/ros_typedb_benchmark/ros_typedb_benchmark/stress_runner.py similarity index 97% rename from ros_typedb_tools/ros_typedb_tools/stress_runner.py rename to ros_typedb_benchmark/ros_typedb_benchmark/stress_runner.py index 4fb02c3..272dbda 100644 --- a/ros_typedb_tools/ros_typedb_tools/stress_runner.py +++ b/ros_typedb_benchmark/ros_typedb_benchmark/stress_runner.py @@ -34,22 +34,22 @@ from lifecycle_msgs.srv import GetState from ros_typedb_msgs.srv import Query -from ros_typedb_tools.stress_config import build_query_request -from ros_typedb_tools.stress_config import DEFAULT_TYPEDB_START_COMMAND -from ros_typedb_tools.stress_config import DEFAULT_TYPEDB_STOP_COMMAND -from ros_typedb_tools.stress_config import FaultResult -from ros_typedb_tools.stress_config import InvariantRecord -from ros_typedb_tools.stress_config import InvariantSpec -from ros_typedb_tools.stress_config import QuerySpec -from ros_typedb_tools.stress_config import RequestRecord -from ros_typedb_tools.stress_config import StressExperimentResult -from ros_typedb_tools.stress_invariants import build_timeout_invariant_record -from ros_typedb_tools.stress_invariants import evaluate_invariant_response -from ros_typedb_tools.stress_mixed import build_mixed_cleanup_query_spec -from ros_typedb_tools.stress_mixed import build_mixed_query_spec -from ros_typedb_tools.stress_mixed import MixedQueryProfile -from ros_typedb_tools.stress_output import DebugEventWriter -from ros_typedb_tools.stress_resolution import resolve_stress_config +from ros_typedb_benchmark.stress_config import build_query_request +from ros_typedb_benchmark.stress_config import DEFAULT_TYPEDB_START_COMMAND +from ros_typedb_benchmark.stress_config import DEFAULT_TYPEDB_STOP_COMMAND +from ros_typedb_benchmark.stress_config import FaultResult +from ros_typedb_benchmark.stress_config import InvariantRecord +from ros_typedb_benchmark.stress_config import InvariantSpec +from ros_typedb_benchmark.stress_config import QuerySpec +from ros_typedb_benchmark.stress_config import RequestRecord +from ros_typedb_benchmark.stress_config import StressExperimentResult +from ros_typedb_benchmark.stress_invariants import build_timeout_invariant_record +from ros_typedb_benchmark.stress_invariants import evaluate_invariant_response +from ros_typedb_benchmark.stress_mixed import build_mixed_cleanup_query_spec +from ros_typedb_benchmark.stress_mixed import build_mixed_query_spec +from ros_typedb_benchmark.stress_mixed import MixedQueryProfile +from ros_typedb_benchmark.stress_output import DebugEventWriter +from ros_typedb_benchmark.stress_resolution import resolve_stress_config from std_srvs.srv import Empty diff --git a/ros_typedb_tools/scripts/run_stage2_timeout_scenarios.sh b/ros_typedb_benchmark/scripts/run_stage2_timeout_scenarios.sh similarity index 93% rename from ros_typedb_tools/scripts/run_stage2_timeout_scenarios.sh rename to ros_typedb_benchmark/scripts/run_stage2_timeout_scenarios.sh index be8b8d9..e699c9d 100755 --- a/ros_typedb_tools/scripts/run_stage2_timeout_scenarios.sh +++ b/ros_typedb_benchmark/scripts/run_stage2_timeout_scenarios.sh @@ -24,7 +24,7 @@ run_stress() { echo echo "==> Running $name" - ros2 run ros_typedb_tools ros_typedb_stress_experiment \ + ros2 run ros_typedb_benchmark ros_typedb_stress_experiment \ --service-name "$service_name" \ --clients "$CLIENTS" \ --duration-s "$DURATION_S" \ @@ -37,7 +37,7 @@ run_stress() { start_fake_service() { echo echo "==> Starting fake Query service on $FAKE_SERVICE_NAME" - ros2 run ros_typedb_tools ros_typedb_fake_query_service \ + ros2 run ros_typedb_benchmark ros_typedb_fake_query_service \ --service-name "$FAKE_SERVICE_NAME" \ >"$OUTPUT_DIR/fake_query_service.log" 2>&1 & FAKE_SERVICE_PID="$!" diff --git a/ros_typedb_benchmark/setup.cfg b/ros_typedb_benchmark/setup.cfg new file mode 100644 index 0000000..ef08e70 --- /dev/null +++ b/ros_typedb_benchmark/setup.cfg @@ -0,0 +1,4 @@ +[develop] +script_dir=$base/lib/ros_typedb_benchmark +[install] +install_scripts=$base/lib/ros_typedb_benchmark diff --git a/ros_typedb_benchmark/setup.py b/ros_typedb_benchmark/setup.py new file mode 100644 index 0000000..0452e5f --- /dev/null +++ b/ros_typedb_benchmark/setup.py @@ -0,0 +1,42 @@ +"""Package setup for ros_typedb_benchmark.""" + +from setuptools import find_packages, setup + +package_name = 'ros_typedb_benchmark' + +setup( + name=package_name, + version='0.0.0', + packages=find_packages(exclude=['test']), + package_data={ + package_name: ['profiles/*.json'], + }, + data_files=[ + ('share/ament_index/resource_index/packages', + ['resource/' + package_name]), + ('share/' + package_name, ['package.xml']), + ], + install_requires=['setuptools'], + zip_safe=True, + maintainer='Gustavo Rezende', + maintainer_email='g.rezendesilva@tudelft.nl', + description='Stress-testing and fault-injection benchmark tools for ros_typedb.', + license='Apache-2.0', + extras_require={ + 'test': [ + 'pytest', + ], + }, + entry_points={ + 'console_scripts': [ + ( + 'ros_typedb_stress_experiment = ' + 'ros_typedb_benchmark.ros_typedb_stress_experiment:main' + ), + ( + 'ros_typedb_fake_query_service = ' + 'ros_typedb_benchmark.fake_query_service:fake_query_service_main' + ), + ], + }, +) diff --git a/ros_typedb_benchmark/test/test_copyright.py b/ros_typedb_benchmark/test/test_copyright.py new file mode 100644 index 0000000..97a3919 --- /dev/null +++ b/ros_typedb_benchmark/test/test_copyright.py @@ -0,0 +1,25 @@ +# Copyright 2015 Open Source Robotics Foundation, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from ament_copyright.main import main +import pytest + + +# Remove the `skip` decorator once the source file(s) have a copyright header +@pytest.mark.skip(reason='No copyright header has been placed in the generated source file.') +@pytest.mark.copyright +@pytest.mark.linter +def test_copyright(): + rc = main(argv=['.', 'test']) + assert rc == 0, 'Found errors' diff --git a/ros_typedb_benchmark/test/test_flake8.py b/ros_typedb_benchmark/test/test_flake8.py new file mode 100644 index 0000000..27ee107 --- /dev/null +++ b/ros_typedb_benchmark/test/test_flake8.py @@ -0,0 +1,25 @@ +# Copyright 2017 Open Source Robotics Foundation, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from ament_flake8.main import main_with_errors +import pytest + + +@pytest.mark.flake8 +@pytest.mark.linter +def test_flake8(): + rc, errors = main_with_errors(argv=[]) + assert rc == 0, \ + 'Found %d code style errors / warnings:\n' % len(errors) + \ + '\n'.join(errors) diff --git a/ros_typedb_benchmark/test/test_pep257.py b/ros_typedb_benchmark/test/test_pep257.py new file mode 100644 index 0000000..b234a38 --- /dev/null +++ b/ros_typedb_benchmark/test/test_pep257.py @@ -0,0 +1,23 @@ +# Copyright 2015 Open Source Robotics Foundation, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from ament_pep257.main import main +import pytest + + +@pytest.mark.linter +@pytest.mark.pep257 +def test_pep257(): + rc = main(argv=['.', 'test']) + assert rc == 0, 'Found code style errors / warnings' diff --git a/ros_typedb_tools/test/test_ros_typedb_stress_experiment.py b/ros_typedb_benchmark/test/test_ros_typedb_stress_experiment.py similarity index 97% rename from ros_typedb_tools/test/test_ros_typedb_stress_experiment.py rename to ros_typedb_benchmark/test/test_ros_typedb_stress_experiment.py index d562070..0a39c32 100644 --- a/ros_typedb_tools/test/test_ros_typedb_stress_experiment.py +++ b/ros_typedb_benchmark/test/test_ros_typedb_stress_experiment.py @@ -12,14 +12,14 @@ from ros_typedb_msgs.msg import Attribute from ros_typedb_msgs.msg import QueryResult from ros_typedb_msgs.msg import ResultTree -from ros_typedb_tools.fake_query_service import ( +from ros_typedb_benchmark.fake_query_service import ( build_fake_service_argument_parser, ) -import ros_typedb_tools.ros_typedb_stress_experiment as stress_cli -from ros_typedb_tools.ros_typedb_stress_experiment import ( +import ros_typedb_benchmark.ros_typedb_stress_experiment as stress_cli +from ros_typedb_benchmark.ros_typedb_stress_experiment import ( build_argument_parser, ) -from ros_typedb_tools.stress_config import ( +from ros_typedb_benchmark.stress_config import ( build_invariant_specs, build_query_specs, DEFAULT_TYPEDB_START_COMMAND, @@ -33,12 +33,12 @@ StressExperimentResult, validate_experiment_args, ) -from ros_typedb_tools.stress_invariants import evaluate_invariant_response -from ros_typedb_tools.stress_mixed import build_mixed_cleanup_query_spec -from ros_typedb_tools.stress_mixed import build_mixed_query_spec -from ros_typedb_tools.stress_mixed import load_mixed_query_profile -from ros_typedb_tools.stress_mixed import make_mixed_key -from ros_typedb_tools.stress_output import ( +from ros_typedb_benchmark.stress_invariants import evaluate_invariant_response +from ros_typedb_benchmark.stress_mixed import build_mixed_cleanup_query_spec +from ros_typedb_benchmark.stress_mixed import build_mixed_query_spec +from ros_typedb_benchmark.stress_mixed import load_mixed_query_profile +from ros_typedb_benchmark.stress_mixed import make_mixed_key +from ros_typedb_benchmark.stress_output import ( DebugEventWriter, default_timeout_output_path, summarize_invariant_records, @@ -46,8 +46,8 @@ write_results, write_timeout_results, ) -from ros_typedb_tools.stress_resolution import resolve_stress_config -from ros_typedb_tools.stress_runner import ( +from ros_typedb_benchmark.stress_resolution import resolve_stress_config +from ros_typedb_benchmark.stress_runner import ( _build_typedb_restart_commands, _call_lifecycle_transition, _compute_observed_fault_outage_s, @@ -57,8 +57,8 @@ _run_fault_command, _start_fault_command, ) -from ros_typedb_tools.stress_runner import _RunState -from ros_typedb_tools.stress_runner import run_experiment +from ros_typedb_benchmark.stress_runner import _RunState +from ros_typedb_benchmark.stress_runner import run_experiment def _record( @@ -739,7 +739,7 @@ def fake_run(command, **kwargs): return Completed() monkeypatch.setattr( - 'ros_typedb_tools.stress_runner.subprocess.run', + 'ros_typedb_benchmark.stress_runner.subprocess.run', fake_run, ) @@ -768,7 +768,7 @@ def fake_popen(command, **kwargs): return RunningProcess() monkeypatch.setattr( - 'ros_typedb_tools.stress_runner.subprocess.Popen', + 'ros_typedb_benchmark.stress_runner.subprocess.Popen', fake_popen, ) diff --git a/ros_typedb_tools/README.md b/ros_typedb_tools/README.md index 7304a6b..52c815b 100644 --- a/ros_typedb_tools/README.md +++ b/ros_typedb_tools/README.md @@ -4,13 +4,12 @@ Reusable TypeDB 2 helper scripts for generating schema and rule diagrams. ## Overview -This package currently provides three console scripts: +This package provides two console scripts for TypeDB schema and rule visualization: - `typedb_schema_diagram`: generate a diagram for one schema file or a merged diagram for multiple schema files. - `typedb_rule_diagram`: generate a rule dependency/read-write diagram for one or more schema files that contain rules. -- `ros_typedb_stress_experiment`: send repeated requests to the real `ros_typedb` query service and report request metrics. -The diagram tools can write `.svg` or `.dot` output. +Both tools can write `.svg` or `.dot` output. ## Prerequisites @@ -97,491 +96,3 @@ Useful options: - `--orientation`: `vertical` or `horizontal` - `--no-rule-dependencies`: suppress rule-to-rule dependency edges -## ros_typedb_stress_experiment - -Runs fixed-count or duration-based stress experiments against the real -`/ros_typedb_interface/query` service. - -Prerequisites: - -- TypeDB is running. -- `ros_typedb_interface` is running. -- The lifecycle node has been configured so the query service exists. - -### Experiment Stages - -The stress tool is organized as staged experiments. Earlier stages are useful -on their own; later stages build on the same result format and command-line -interface. - -| Stage | Status | Purpose | -| --- | --- | --- | -| 1. Minimal real service load | Implemented | Send repeated requests to the real ROS `Query` service and measure latency, success, errors, and timeouts. | -| 2. Concurrent client stress | Implemented | Run many ROS service clients concurrently to find service/executor transport limits before blaming TypeDB or driver code. | -| 3. Correctness invariants | Implemented | Check that expected baseline facts remain visible during and after load, not only that requests returned. | -| 4. Mixed read/write load | Implemented | Read while inserting, updating, deleting, and cleaning up experiment-owned temporary data. | -| 5. Delete-database fault injection | Implemented | Delete the database during load and measure whether the driver fails loudly, reloads correctly, and recovers invariants. | -| 6. TypeDB restart fault injection | Implemented | Stop/restart TypeDB during load and measure outage, reconnect behavior, and post-recovery correctness. | -| 7. Lifecycle stress | Implemented | Trigger ROS lifecycle cleanup/configure/activate transitions during load to check teardown races. | -| 8. External schema/data profiles | Planned | Run the same harness with application-specific schema/data, query mixes, invariants, setup, and cleanup profiles. | - -Use Stage 1 when validating a single query path. Use Stage 2 when probing ROS -service throughput and timeout behavior. Use Stage 3 when checking that load -does not corrupt or hide baseline data. Use Stage 4 when validating robustness -under real read/write activity. Use Stage 5 when validating recovery after a -database deletion fault. Use Stage 6 when validating driver reconnect behavior -after the TypeDB server process or container is restarted. Use Stage 7 when -validating that ROS lifecycle cleanup waits for active query callbacks before -tearing down TypeDB resources and that the node can be configured and activated -again afterward. - -### Stage 1: Minimal Real Service Load - -Example: - -```bash -ros2 run ros_typedb_tools ros_typedb_stress_experiment \ - --query 'match $p isa Plan; (plan: $p, action: $a) isa has_action; fetch $a: attribute;' \ - --query-type fetch \ - --requests 2000 \ - --timeout-s 5 \ - --output ~/results/ros_typedb_stress_results.json -``` - -This is the smallest useful run. It is best for confirming that the service, -query type, timeout handling, result conversion, and JSON output work for one -known query before adding concurrency. - -### Stage 2: Concurrent Client Stress - -Concurrent read stress example: - -```bash -ros2 run ros_typedb_tools ros_typedb_stress_experiment \ - --clients 20 \ - --duration-s 60 \ - --timeout-s 10 \ - --mode read \ - --debug-events-output ~/results/ros_typedb_read_stress_events.jsonl \ - --output ~/results/ros_typedb_read_stress_results.json -``` - -```bash -ros2 run ros_typedb_tools ros_typedb_stress_experiment \ - --clients 20 \ - --duration-s 60 \ - --timeout-s 10 \ - --query 'match $x isa entity; get $x;' \ - --query-type get \ - --debug-events-output ~/results/ros_typedb_read_stress_events.jsonl \ - --output ~/results/ros_typedb_get_stress_results.json -``` - -This stage is for separating ROS service transport behavior from database -behavior. It reports throughput and latency percentiles across concurrent -clients and can write debug JSONL events for request/future/executor analysis. - -When `--output` is set, timeout-only records are also written next to the main -result file using the suffix `_timeouts.json`. Use `--timeout-output` to choose -a different path. Timeout records include request index, client id, query index, -query type, query text, latency, and whether future cancellation was requested. -The command exits successfully when the experiment runs to completion, even if -some requests fail. Use `--fail-on-failure` when a nonzero exit code is useful -for CI or scripted thresholds. - -### Stage 3: Correctness Invariants - -An invariant is a correctness check that should remain true while the stress -load is running. Request metrics show whether the service answered; invariants -show whether the database still contains expected baseline facts. - -Correctness invariant example for the bundled `ros_typedb` test schema/data: - -```bash -ros2 run ros_typedb_tools ros_typedb_stress_experiment \ - --clients 10 \ - --duration-s 60 \ - --timeout-s 10 \ - --mode read \ - --invariant-profile test-data \ - --output ~/results/ros_typedb_read_invariants_results.json -``` - -Use `--invariant-profile test-data` only when `ros_typedb_interface` was started -with the schema/data from `ros_typedb/test/typedb_test_data/`. The profile is -stored in `ros_typedb_tools/ros_typedb_tools/profiles/test_data_invariants.json` -and checks stable aggregate counts for `person`, `company`, and `employment`, -plus the `boss@tudelft.nl` sentinel person. This profile is not appropriate for -other schemas, such as the `ros_typedb_examples` plan schema. - -For `ros_typedb_examples/data/plan_schema.tql` and `plan_data.tql`, use: - -```bash -ros2 run ros_typedb_tools ros_typedb_stress_experiment \ - --clients 10 \ - --duration-s 60 \ - --timeout-s 10 \ - --mode read \ - --invariant-profile plan-schema \ - --output ~/results/ros_typedb_plan_invariants_results.json -``` - -The `plan-schema` profile is stored in -`ros_typedb_tools/ros_typedb_tools/profiles/plan_schema_invariants.json` and -checks counts for `Plan`, `Action`, `Proposition`, and the plan/action relation -types, plus a sentinel action named `collect-water-sample`. - -Checks run every 10 seconds by default and once at the end. Use -`--invariant-period-s` to adjust the periodic interval, or -`--invariant-period-s 0` to run only the final check. Invariant failures are -reported separately from request failures and always make the command exit with -status 1. - -### Stage 4: Mixed Read/Write Load - -Mixed read/write stress for the bundled `ros_typedb` test schema: - -```bash -ros2 run ros_typedb_tools ros_typedb_stress_experiment \ - --clients 10 \ - --duration-s 120 \ - --timeout-s 10 \ - --mode mixed \ - --invariant-profile test-data \ - --request-gap-s 0.01 \ - --max-in-flight 10 \ - --output ~/results/ros_typedb_mixed_stress_results.json -``` - -The packaged `test-data` mixed profile reads the database while inserting, -updating, and deleting experiment-owned temporary `robot` entities. It is -intended for the schema in `ros_typedb/test/typedb_test_data/schema.tql` and -cleans up remaining temporary robots before the final invariant check. - -Mixed read/write stress for the `ros_typedb_examples` plan schema: - -```bash -ros2 run ros_typedb_tools ros_typedb_stress_experiment \ - --clients 10 \ - --duration-s 120 \ - --timeout-s 10 \ - --mode mixed \ - --mixed-profile plan-schema \ - --invariant-profile plan-schema-mixed \ - --request-gap-s 0.01 \ - --max-in-flight 10 \ - --output ~/results/ros_typedb_plan_mixed_stress_results.json -``` - -Mixed profiles are packaged JSON files under -`ros_typedb_tools/ros_typedb_tools/profiles/`. The `plan-schema` mixed profile -writes temporary `Action` entities, so use `plan-schema-mixed` invariants when -periodic invariant checks run during that mixed workload. That invariant -profile keeps stable checks for plans, propositions, relations, and the -`collect-water-sample` sentinel action without requiring an exact total -`Action` count while temporary actions may exist. - -### Stage 5: Delete-Database Fault Injection - -This stage calls the real `delete_database` service while normal query clients -are still active. It is meant to check that a missing database does not produce -silent success against an uninitialized empty database. After the delete call, -periodic invariant passes during the active load record recovery as soon as all -checks pass once. If periodic checks do not observe recovery, the runner uses a -post-load recovery check until invariants pass or `--fault-recovery-timeout-s` -expires. - -For a reproducible run, start the bundled test-data launch file in one terminal. -It configures and activates `/ros_typedb_interface`, loads -`ros_typedb/test/typedb_test_data/schema.tql` and `data.tql`, and leaves the -schema/data paths available for recovery: - -```bash -docker exec -it ros_typedb bash -lc ' -cd /home/ubuntu-user/typedb_ws && -source /opt/ros/humble/setup.bash && -source install/setup.bash && -ros2 launch ros_typedb_examples test_data_example.launch.py' -``` - -Then run the fault-injection experiment from another terminal: - -```bash -docker exec -it ros_typedb bash -lc ' -cd /home/ubuntu-user/typedb_ws && -source /opt/ros/humble/setup.bash && -source install/setup.bash && -ros2 run ros_typedb_tools ros_typedb_stress_experiment \ - --clients 10 \ - --duration-s 180 \ - --timeout-s 10 \ - --mode read \ - --fault delete-database \ - --fault-at-s 60 \ - --invariant-profile test-data \ - --request-gap-s 0.01 \ - --max-in-flight 10 \ - --output /tmp/stage5_delete_database.json \ - --debug-events-output /tmp/stage5_delete_database_debug.jsonl' -``` - -The result JSON always includes a `fault` object with: - -- `fault`: configured fault name, such as `delete-database` -- `fault_at_s`: configured trigger time -- `fault_triggered_at_s`: monotonic timestamp when the fault was triggered -- `fault_delete_success`, `fault_delete_error`, and `fault_delete_latency_s` -- `fault_recovered_at_s`: first timestamp when all recovery invariants passed -- `fault_recovery_s`: elapsed time from trigger to successful recovery - -If the fault is triggered but invariants do not recover, the CLI exits with -status 1. Keep `--invariant-profile test-data` paired with the bundled -test-data launch; use a matching invariant profile for any other schema/data -pair. Keep `--max-in-flight 10` and `--request-gap-s 0.01` for robustness -testing so the experiment stays inside the known stable ROS service envelope. - -### Stage 6: TypeDB Restart Fault Injection - -This stage restarts the TypeDB server while normal query clients continue -sending requests. It is meant to check that server outages produce bounded, -explicit failures and that later requests reconnect to TypeDB without -restarting the ROS lifecycle node. Recovery is still correctness-based: the -first complete post-fault invariant pass marks `fault_recovered_at_s`. - -Run TypeDB separately from the ROS node and stress process. Do not pass the -same Docker container that is running `ros_typedb_interface` or the experiment -to `--typedb-container`, because stopping that container stops the test itself. -By default, the process fault uses `pkill -f "typedb/core/server"` to stop -TypeDB and `typedb server` to start it again: - -```bash -ros2 run ros_typedb_tools ros_typedb_stress_experiment \ - --clients 10 \ - --duration-s 240 \ - --timeout-s 10 \ - --mode read \ - --fault restart-typedb \ - --typedb-restart-delay-s 2 \ - --fault-at-s 60 \ - --invariant-profile test-data \ - --request-gap-s 0.01 \ - --max-in-flight 10 \ - --output /tmp/stage6_restart_typedb.json \ - --debug-events-output /tmp/stage6_restart_typedb_debug.jsonl -``` - -Override the process commands with `--typedb-stop-command` and -`--typedb-start-command` when needed. For a TypeDB-only Docker container, pass -`--typedb-container`; it overrides the process commands: - -```bash ---typedb-container typedb_server -``` - -The restart controller runs in a background thread, so the main experiment loop -keeps spinning ROS futures and sending requests during the outage. The result -JSON adds restart fields to the `fault` object: - -- `fault_restart_stop_success`, `fault_restart_stop_error`, and - `fault_restart_stop_latency_s` -- `fault_restart_start_success`, `fault_restart_start_error`, and - `fault_restart_start_latency_s` -- `fault_restart_delay_s`: configured delay between stop and start -- `fault_restart_outage_s`: elapsed controller time from stop start to start - completion -- `fault_observed_outage_s`: elapsed time from the first failed post-fault - request result to the first later successful request result - -If the fault is triggered but invariants do not recover, the CLI exits with -status 1. Request failures during the outage remain recorded in the JSON output; -use `--fail-on-failure` only when you want any request failure to fail the -command even if invariants later recover. - -### Stage 7: Lifecycle Stress - -This stage calls the ROS lifecycle `change_state` service while normal query -clients continue sending requests. The default sequence is deactivate, cleanup, -configure, then activate. Cleanup destroys the query and delete services and -waits for active service callbacks before closing TypeDB resources; the -experiment verifies that this does not race with in-flight requests and that -the node recovers after reactivation. The controller checks `get_state` after -each transition and only sends the next transition after the expected lifecycle -state is observed. - -When using a launch file that automatically reactivates the node after every -deactivate, disable that behavior for this experiment. The bundled test-data -launch file defaults to normal auto-reactivation, so start it for Stage 7 with: - -```bash -ros2 launch ros_typedb_examples test_data_example.launch.py \ - reactivate_on_deactivate:=False -``` - -Then run the stress experiment: - -```bash -ros2 run ros_typedb_tools ros_typedb_stress_experiment \ - --clients 10 \ - --duration-s 180 \ - --timeout-s 10 \ - --mode read \ - --fault lifecycle-cleanup \ - --fault-at-s 60 \ - --invariant-profile test-data \ - --request-gap-s 0.01 \ - --max-in-flight 10 \ - --output /tmp/stage7_lifecycle_cleanup.json \ - --debug-events-output /tmp/stage7_lifecycle_cleanup_debug.jsonl -``` - -The lifecycle `change_state` and `get_state` services default to the -`--service-name` prefix with `/change_state` and `/get_state` appended. For -example, the default query service `/ros_typedb_interface/query` maps to -`/ros_typedb_interface/change_state` and `/ros_typedb_interface/get_state`. -Override them with `--lifecycle-change-state-service-name` and -`--lifecycle-get-state-service-name` when the launched lifecycle node uses a -different name, such as `/ros_typedb/change_state` and -`/ros_typedb/get_state`. - -By default the node is configured and activated again after cleanup. Use -`--no-lifecycle-reactivate` only when you intentionally want to leave the node -cleaned up; in that mode recovery invariants are not expected to pass unless -something else reactivates the node. - -The result JSON adds lifecycle fields to the `fault` object: - -- `fault_lifecycle_deactivate_success`, `fault_lifecycle_deactivate_error`, - and `fault_lifecycle_deactivate_latency_s` -- `fault_lifecycle_cleanup_success`, `fault_lifecycle_cleanup_error`, and - `fault_lifecycle_cleanup_latency_s` -- `fault_lifecycle_configure_success`, `fault_lifecycle_configure_error`, and - `fault_lifecycle_configure_latency_s` -- `fault_lifecycle_activate_success`, `fault_lifecycle_activate_error`, and - `fault_lifecycle_activate_latency_s` -- `fault_lifecycle_reactivate`: whether configure/activate was requested -- `fault_observed_outage_s`: elapsed time from the first failed post-fault - request result to the first later successful request result - -If a lifecycle transition is rejected or times out, the CLI exits with status -1. If transitions succeed but invariants do not recover, the CLI also exits -with status 1. Request failures while the lifecycle node is inactive or cleaned -up remain recorded in the JSON output. - -### Planned Fault-Injection Stages - -Stage 8 is not implemented yet. It is a placeholder for the next robustness -check: - -- Stage 8, external profiles: load schema-specific query mixes, invariants, - setup, and cleanup from user-provided JSON/YAML profile files instead of only - the packaged examples. - -### Diagnostics and Boundaries - -For ROS client/executor debugging, `--debug-events-output` writes JSONL events -for request sends, `spin_once()` calls, completed futures, timeouts, and periodic -pending-request snapshots. Use `--executor single` or `--executor multi` to -compare explicit executors against the default `rclpy.spin_once(node)` behavior. -Use `--request-gap-s` to add a small per-client delay between requests, or -`--max-in-flight` to cap total outstanding service requests independently of the -number of clients. - -For `ros_typedb` robustness experiments, prefer bounded service concurrency. -High-rate unbounded Query service calls produced timeout waves even when TypeDB -was removed from the path by using fake Query services. The same behavior was -observed with Python and C++ clients/servers, Fast DDS and CycloneDDS, and ROS -Humble and Lyrical. Treat that as a ROS service transport/executor stress limit, -not as a database or result-conversion failure. - -A practical bounded baseline is: - -```bash -ros2 run ros_typedb_tools ros_typedb_stress_experiment \ - --clients 20 \ - --duration-s 60 \ - --timeout-s 10 \ - --mode read \ - --request-gap-s 0.01 \ - --max-in-flight 10 \ - --output ~/results/ros_typedb_read_bounded_results.json -``` - -Use larger `--max-in-flight` values only when intentionally probing ROS service -transport saturation. In local tests, about 12-15 in-flight requests started to -produce timeout waves, while 10 or fewer stayed stable. - -To isolate ROS service/client behavior from TypeDB work, run a fake Query -service in one terminal: - -```bash -ros2 run ros_typedb_tools ros_typedb_fake_query_service -``` - -Then run the same stress command against it from another terminal. - -To run the common timeout-debug scenarios in sequence, use: - -```bash -bash src/ros_typedb/ros_typedb_tools/scripts/run_stage2_timeout_scenarios.sh -``` - -The script runs real read/global, real read/multi, fake read/global, fake -read/multi, and real explicit entity-get/global scenarios. Override defaults -with environment variables such as `CLIENTS`, `DURATION_S`, `TIMEOUT_S`, and -`OUTPUT_DIR`. - -Useful options: - -- `--service-name`: query service name. Defaults to `/ros_typedb_interface/query` -- `--query`: optional TypeDB query to send on each request -- `--query-type`: one of `define`, `delete`, `fetch`, `get`, - `get_aggregate`, `insert`, or `update`; required when `--query` is used -- `--requests`: number of requests to send when `--duration-s` is omitted -- `--clients`: number of concurrent service clients -- `--duration-s`: run for this many seconds instead of a fixed request count -- `--mode`: built-in query mix to use when `--query` is omitted; supports - `read` and `mixed` -- `--mixed-profile`: schema-specific mixed-mode profile, such as `test-data` - or `plan-schema`; `auto` derives it from the invariant profile when possible -- `--request-gap-s`: minimum delay between requests from the same client -- `--max-in-flight`: maximum number of outstanding requests across all clients -- `--executor`: client executor mode, one of `global`, `single`, or `multi` -- `--debug-events-output`: optional JSONL debug event path -- `--invariant-profile`: optional correctness profile, such as `test-data`, - `plan-schema`, or `plan-schema-mixed` -- `--invariant-period-s`: seconds between periodic invariant checks -- `--timeout-s`: per-request client and server timeout -- `--output`: optional JSON results path -- `--timeout-output`: optional timeout-only JSON results path -- `--fail-on-failure`: return exit code 1 when any request fails - -## Tests - -```Bash -colcon test --event-handlers console_cohesion+ --packages-select ros_typedb_tools -``` - -## Docker - -The commands below assume you are running them from the workspace root: - -```Bash -cd -``` - -Start dev container **without** display and the `ros_typedb` directory mounted: - -```Bash -docker run -it --rm --name ros_typedb -v /etc/localtime:/etc/localtime:ro -v $PWD/src/ros_typedb:/home/ubuntu-user/typedb_ws/src/ros_typedb -v $PWD/ros_typedb_results/:/home/ubuntu-user/results/ ros_typedb -``` - -Start new terminal in the container: - -```Bash -docker exec -it ros_typedb bash -``` - - - export RMW_IMPLEMENTATION=rmw_fastrtps_cpp - -export RMW_IMPLEMENTATION=rmw_cyclonedds_cpp diff --git a/ros_typedb_tools/package.xml b/ros_typedb_tools/package.xml index 392819f..6aae9c1 100644 --- a/ros_typedb_tools/package.xml +++ b/ros_typedb_tools/package.xml @@ -9,8 +9,6 @@ python3-pydot graphviz - rclpy - ros_typedb_msgs ament_copyright ament_flake8 diff --git a/ros_typedb_tools/setup.py b/ros_typedb_tools/setup.py index ef20ffa..1f52fbe 100644 --- a/ros_typedb_tools/setup.py +++ b/ros_typedb_tools/setup.py @@ -8,9 +8,6 @@ name=package_name, version='0.0.0', packages=find_packages(exclude=['test']), - package_data={ - package_name: ['profiles/*.json'], - }, data_files=[ ('share/ament_index/resource_index/packages', ['resource/' + package_name]), @@ -40,14 +37,6 @@ 'typedb_rule_diagram = ' 'ros_typedb_tools.typedb_rule_diagram:main' ), - ( - 'ros_typedb_stress_experiment = ' - 'ros_typedb_tools.ros_typedb_stress_experiment:main' - ), - ( - 'ros_typedb_fake_query_service = ' - 'ros_typedb_tools.fake_query_service:fake_query_service_main' - ), ], }, ) From 264f44b2eba05ca101bdf114f8fbda98ce33f714 Mon Sep 17 00:00:00 2001 From: Gustavo Date: Sun, 28 Jun 2026 19:34:39 -0300 Subject: [PATCH 33/36] update AGENTS.md --- AGENTS.md | 14 ++++++++------ 1 file changed, 8 insertions(+), 6 deletions(-) diff --git a/AGENTS.md b/AGENTS.md index e2588e8..e3331e8 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -2,14 +2,15 @@ ## Project Structure & Module Organization -This repository contains three ROS 2 packages. `ros_typedb/` is the core +This repository contains four ROS 2 packages. `ros_typedb/` is the core `ament_python` package, with runtime code in `ros_typedb/ros_typedb/`, launch files in `ros_typedb/launch/`, and tests plus TypeDB fixtures in `ros_typedb/test/`. `ros_typedb_msgs/` is the `ament_cmake` interface package; message definitions live in `msg/` and services in `srv/`. `ros_typedb_tools/` contains Python command-line tools for TypeDB schema and rule diagrams, with -tests and fixtures under `ros_typedb_tools/test/`. Sphinx documentation is in -`docs/source/`. +tests and fixtures under `ros_typedb_tools/test/`. `ros_typedb_benchmark/` +contains stress-testing, fault-injection, and benchmark tools. Sphinx +documentation is in `docs/source/`. ## Build, Test, and Development Commands @@ -22,6 +23,7 @@ colcon build --symlink-install source install/setup.bash colcon test --event-handlers console_cohesion+ --packages-up-to ros_typedb colcon test --event-handlers console_cohesion+ --packages-select ros_typedb_tools +colcon test --event-handlers console_cohesion+ --packages-select ros_typedb_benchmark ``` Use `ros2 run ros_typedb ros_typedb_interface -p schema_path:= -p data_path:=` @@ -123,7 +125,7 @@ exceptions, then return `success=False` with `error_message` populated. Do not let malformed or unexpected query results escape the callback and crash the `ros_typedb_interface` node. -## ros_typedb_tools Stress Diagnostics +## ros_typedb_benchmark Stress Diagnostics When debugging Stage 2 stress timeouts, use the fake query service and the scenario runner before changing `ros_typedb_interface` behavior. If the fake @@ -140,7 +142,7 @@ database robustness tests, stay inside the bounded envelope: in local testing Stress invariants are correctness checks that run alongside load to verify the database still contains expected baseline facts. Keep invariant and mixed read/write profiles as packaged JSON files under -`ros_typedb_tools/ros_typedb_tools/profiles/`, not as hard-coded Python data. +`ros_typedb_benchmark/ros_typedb_benchmark/profiles/`, not as hard-coded Python data. Update `setup.py` package data when adding new profile file patterns. Do not use a built-in invariant profile with a different schema/data pair. A @@ -170,7 +172,7 @@ package `setup.py`; do not rely on source-tree-relative paths after Useful stress diagnostics: ```bash -bash src/ros_typedb/ros_typedb_tools/scripts/run_stage2_timeout_scenarios.sh +bash src/ros_typedb/ros_typedb_benchmark/scripts/run_stage2_timeout_scenarios.sh ``` The script runs real read/global, real read/multi, fake read/global, fake From f667424e4db6e4adc8a183c169f53cf3797915d2 Mon Sep 17 00:00:00 2001 From: Gustavo Date: Sun, 28 Jun 2026 19:34:49 -0300 Subject: [PATCH 34/36] update README --- README.md | 100 ++---------------------------------------------------- 1 file changed, 3 insertions(+), 97 deletions(-) diff --git a/README.md b/README.md index 3085339..fad7b1d 100644 --- a/README.md +++ b/README.md @@ -5,7 +5,7 @@ This package provides a basic generic integration between ROS and [typeDB](https://typedb.com/). The package was designed in a way to enable users to easily extend it to fulfill their needs, the package design is explained in the [Package Design](#package-design) section. -This package was tested in Ubuntu 22.04 with ROS Humble and typedb v2.28.3. +This package was tested in Ubuntu 22.04 with ROS Humble and typedb v2.29.1. ## Install @@ -17,7 +17,7 @@ Follow the [official instructions](https://docs.ros.org/en/humble/Installation/U #### Install TypeDB -**Note:** This package has been tested with TypeDB version `2.28.3` and the typedb python driver version `2.28.0`. +**Note:** This package has been tested with TypeDB version `2.29.1` and the typedb python driver version `2.29.7`. Install typedb: follow the[official instructions](https://typedb.com/docs/typedb/2.x/installation.html). @@ -134,101 +134,7 @@ ros2 run ros_typedb ros_typedb_interface -p schema_path:= -p data_p **Note:** Make sure to replace and with the real path for your schema and data file **Note 2:** Remember that ros_typedb_interface is a [LifeCycle](https://design.ros2.org/articles/node_lifecycle.html) node, so you need to change its state to active before using it. Check the [lifecycle tutorial](https://github.com/ros2/demos/tree/rolling/lifecycle). -### Run the stress experiment - -`ros_typedb_tools` provides a stress experiment CLI that sends repeated or -duration-based concurrent requests to the real `ros_typedb_interface/query` -service and reports latency, successes, failures, timeouts, and optional JSON -results. - -First, start TypeDB, run `ros_typedb_interface`, and configure the lifecycle -node so the query service is available. Then run: - -```bash -ros2 run ros_typedb_tools ros_typedb_stress_experiment \ - --query 'match $x isa entity; fetch $x: attribute;' \ - --query-type fetch \ - --requests 20 \ - --timeout-s 5 \ - --output /tmp/ros_typedb_stress_results.json -``` - -For concurrent read stress, omit `--query` and use the built-in read query mix: - -```bash -ros2 run ros_typedb_tools ros_typedb_stress_experiment \ - --clients 20 \ - --duration-s 60 \ - --timeout-s 10 \ - --mode read \ - --output /tmp/ros_typedb_read_stress_results.json -``` - -Use `--service-name` if the node name is different from the default -`/ros_typedb_interface/query`. - -An invariant is a correctness check that should remain true while the stress -load is running. Request metrics show whether the service answered; invariants -show whether the database still contains expected baseline facts. - -Use `--invariant-profile test-data` only when `ros_typedb_interface` was started -with the bundled test schema/data from `ros_typedb/test/typedb_test_data/`: - -```bash -ros2 run ros_typedb_tools ros_typedb_stress_experiment \ - --clients 10 \ - --duration-s 60 \ - --timeout-s 10 \ - --mode read \ - --invariant-profile test-data \ - --output /tmp/ros_typedb_read_invariants_results.json -``` - -The `test-data` profile is stored in -`ros_typedb_tools/ros_typedb_tools/profiles/test_data_invariants.json`. It -periodically checks stable counts for `person`, `company`, and `employment`, -plus the `boss@tudelft.nl` sentinel person. This profile is not appropriate for -other schemas, such as the `ros_typedb_examples` plan schema. The command exits -with status 1 if an invariant fails. - -For the `ros_typedb_examples` plan schema/data, use: - -```bash -ros2 run ros_typedb_tools ros_typedb_stress_experiment \ - --clients 10 \ - --duration-s 60 \ - --timeout-s 10 \ - --mode read \ - --invariant-profile plan-schema \ - --output /tmp/ros_typedb_plan_invariants_results.json -``` - -The `plan-schema` profile is stored in -`ros_typedb_tools/ros_typedb_tools/profiles/plan_schema_invariants.json`. It -checks counts for `Plan`, `Action`, `Proposition`, and the plan/action relation -types, plus a sentinel action named `collect-water-sample`. - -For robustness experiments, keep the ROS service load bounded. High-rate -unbounded service calls produced timeout waves even with fake Query services -implemented in both Python and C++, across Fast DDS, CycloneDDS, ROS Humble, -and ROS Lyrical. That points to a ROS service transport/executor stress limit -rather than TypeDB query execution or `ros_typedb` result conversion. - -A practical bounded read-stress baseline is: - -```bash -ros2 run ros_typedb_tools ros_typedb_stress_experiment \ - --clients 20 \ - --duration-s 60 \ - --timeout-s 10 \ - --mode read \ - --request-gap-s 0.01 \ - --max-in-flight 10 \ - --output /tmp/ros_typedb_read_bounded_results.json -``` - -Use higher in-flight counts only when intentionally testing ROS service -transport saturation, not when validating `ros_typedb` database robustness. +For stress testing, fault injection, and correctness invariant experiments, see [`ros_typedb_benchmark/README.md`](ros_typedb_benchmark/README.md). ## Extend the package From 9c4cdbd17a2136bb8c854b55ef868a442cc6ab94 Mon Sep 17 00:00:00 2001 From: Gustavo Date: Mon, 29 Jun 2026 14:07:07 -0300 Subject: [PATCH 35/36] ros_typedb: make lifecycle auto-activation controllable --- ros_typedb/launch/ros_typedb.launch.py | 38 ++++++++++++++++++- ros_typedb_benchmark/README.md | 22 ++++++----- .../launch/plan_example.launch.py | 31 +++++++++++++++ .../launch/test_data_example.launch.py | 31 +++++++++++++++ 4 files changed, 111 insertions(+), 11 deletions(-) diff --git a/ros_typedb/launch/ros_typedb.launch.py b/ros_typedb/launch/ros_typedb.launch.py index 57f8269..b6a2e09 100644 --- a/ros_typedb/launch/ros_typedb.launch.py +++ b/ros_typedb/launch/ros_typedb.launch.py @@ -1,3 +1,5 @@ +"""Launch the ros_typedb lifecycle node.""" + # Copyright 2024 Gustavo Rezende Silva # # Licensed under the Apache License, Version 2.0 (the "License"); @@ -16,15 +18,19 @@ from launch.actions import EmitEvent from launch.actions import RegisterEventHandler from launch.conditions import IfCondition +from launch.conditions import UnlessCondition from launch.events import matches_action from launch.substitutions import LaunchConfiguration + from launch_ros.actions import LifecycleNode from launch_ros.event_handlers import OnStateTransition from launch_ros.events.lifecycle import ChangeState + import lifecycle_msgs def generate_launch_description(): + """Launch the ros_typedb lifecycle node.""" schema_path = LaunchConfiguration('schema_path') data_path = LaunchConfiguration('data_path') database_name = LaunchConfiguration('database_name') @@ -32,6 +38,9 @@ def generate_launch_description(): force_data = LaunchConfiguration('force_data') force_database = LaunchConfiguration('force_database') infer = LaunchConfiguration('infer') + auto_activate_on_configure = LaunchConfiguration( + 'auto_activate_on_configure' + ) reactivate_on_deactivate = LaunchConfiguration('reactivate_on_deactivate') schema_path_arg = DeclareLaunchArgument( @@ -82,6 +91,12 @@ def generate_launch_description(): description='reactivate the lifecycle node after deactivate' ) + auto_activate_on_configure_arg = DeclareLaunchArgument( + 'auto_activate_on_configure', + default_value='True', + description='activate the lifecycle node after configure' + ) + ros_typedb_node = LifecycleNode( package='ros_typedb', executable='ros_typedb', @@ -106,6 +121,24 @@ def generate_launch_description(): ) ) + ros_typedb_node_initial_activate_event = RegisterEventHandler( + OnStateTransition( + target_lifecycle_node=ros_typedb_node, + start_state='configuring', + goal_state='inactive', + entities=[ + EmitEvent(event=ChangeState( + lifecycle_node_matcher=matches_action(ros_typedb_node), + transition_id=( + lifecycle_msgs.msg.Transition.TRANSITION_ACTIVATE + ), + )), + ], + handle_once=True, + ), + condition=UnlessCondition(auto_activate_on_configure), + ) + ros_typedb_node_activate_event = RegisterEventHandler( OnStateTransition( target_lifecycle_node=ros_typedb_node, @@ -119,7 +152,8 @@ def generate_launch_description(): ), )), ], - ) + ), + condition=IfCondition(auto_activate_on_configure), ) ros_typedb_node_reactivate_event = RegisterEventHandler( @@ -147,9 +181,11 @@ def generate_launch_description(): force_data_arg, force_database_arg, infer_arg, + auto_activate_on_configure_arg, reactivate_on_deactivate_arg, ros_typedb_node, ros_typedb_node_config_event, + ros_typedb_node_initial_activate_event, ros_typedb_node_activate_event, ros_typedb_node_reactivate_event, ]) diff --git a/ros_typedb_benchmark/README.md b/ros_typedb_benchmark/README.md index 98feae3..e7b346c 100644 --- a/ros_typedb_benchmark/README.md +++ b/ros_typedb_benchmark/README.md @@ -306,19 +306,21 @@ command even if invariants later recover. This stage calls the ROS lifecycle `change_state` service while normal query clients continue sending requests. The default sequence is deactivate, cleanup, -configure, then activate. Cleanup destroys the query and delete services and -waits for active service callbacks before closing TypeDB resources; the -experiment verifies that this does not race with in-flight requests and that -the node recovers after reactivation. The controller checks `get_state` after -each transition and only sends the next transition after the expected lifecycle -state is observed. - -When using a launch file that automatically reactivates the node after every -deactivate, disable that behavior for this experiment. The bundled test-data -launch file defaults to normal auto-reactivation, so start it for Stage 7 with: +configure, then activate. Cleanup closes TypeDB resources while service +callbacks are gated by lifecycle state; the experiment verifies that this does +not race with in-flight requests and that the node recovers after reactivation. +The controller checks `get_state` after each transition and only sends the next +transition after the expected lifecycle state is observed. + +When using a launch file that automatically activates the node, disable repeated +launch-managed activation for this experiment. The bundled test-data launch +file defaults to normal auto-activation; with `auto_activate_on_configure` set +to `False`, launch still activates the node once at startup but leaves later +cleanup/configure/activate transitions under the benchmark controller. ```bash ros2 launch ros_typedb_examples test_data_example.launch.py \ + auto_activate_on_configure:=False \ reactivate_on_deactivate:=False ``` diff --git a/ros_typedb_examples/launch/plan_example.launch.py b/ros_typedb_examples/launch/plan_example.launch.py index 3aa3348..7ed1494 100644 --- a/ros_typedb_examples/launch/plan_example.launch.py +++ b/ros_typedb_examples/launch/plan_example.launch.py @@ -22,6 +22,7 @@ from launch.actions import EmitEvent from launch.actions import RegisterEventHandler from launch.conditions import IfCondition +from launch.conditions import UnlessCondition from launch.events import matches_action from launch.substitutions import LaunchConfiguration @@ -53,6 +54,9 @@ def generate_launch_description(): reload_schema = LaunchConfiguration('reload_schema') infer = LaunchConfiguration('infer') query_timeout_s = LaunchConfiguration('query_timeout_s') + auto_activate_on_configure = LaunchConfiguration( + 'auto_activate_on_configure' + ) reactivate_on_deactivate = LaunchConfiguration('reactivate_on_deactivate') database_name_arg = DeclareLaunchArgument( @@ -103,6 +107,12 @@ def generate_launch_description(): description='reactivate the lifecycle node after deactivate', ) + auto_activate_on_configure_arg = DeclareLaunchArgument( + 'auto_activate_on_configure', + default_value='True', + description='activate the lifecycle node after configure', + ) + ros_typedb_node = LifecycleNode( package='ros_typedb', executable='ros_typedb', @@ -129,6 +139,24 @@ def generate_launch_description(): ), ) + initial_activate_event = RegisterEventHandler( + OnStateTransition( + target_lifecycle_node=ros_typedb_node, + start_state='configuring', + goal_state='inactive', + entities=[ + EmitEvent(event=ChangeState( + lifecycle_node_matcher=matches_action(ros_typedb_node), + transition_id=( + lifecycle_msgs.msg.Transition.TRANSITION_ACTIVATE + ), + )), + ], + handle_once=True, + ), + condition=UnlessCondition(auto_activate_on_configure), + ) + activate_event = RegisterEventHandler( OnStateTransition( target_lifecycle_node=ros_typedb_node, @@ -143,6 +171,7 @@ def generate_launch_description(): )), ], ), + condition=IfCondition(auto_activate_on_configure), ) reactivate_event = RegisterEventHandler( @@ -170,9 +199,11 @@ def generate_launch_description(): reload_schema_arg, infer_arg, query_timeout_s_arg, + auto_activate_on_configure_arg, reactivate_on_deactivate_arg, ros_typedb_node, configure_event, + initial_activate_event, activate_event, reactivate_event, ]) diff --git a/ros_typedb_examples/launch/test_data_example.launch.py b/ros_typedb_examples/launch/test_data_example.launch.py index e555614..f118647 100644 --- a/ros_typedb_examples/launch/test_data_example.launch.py +++ b/ros_typedb_examples/launch/test_data_example.launch.py @@ -22,6 +22,7 @@ from launch.actions import EmitEvent from launch.actions import RegisterEventHandler from launch.conditions import IfCondition +from launch.conditions import UnlessCondition from launch.events import matches_action from launch.substitutions import LaunchConfiguration @@ -55,6 +56,9 @@ def generate_launch_description(): reload_schema = LaunchConfiguration('reload_schema') infer = LaunchConfiguration('infer') query_timeout_s = LaunchConfiguration('query_timeout_s') + auto_activate_on_configure = LaunchConfiguration( + 'auto_activate_on_configure' + ) reactivate_on_deactivate = LaunchConfiguration('reactivate_on_deactivate') database_name_arg = DeclareLaunchArgument( @@ -105,6 +109,12 @@ def generate_launch_description(): description='reactivate the lifecycle node after deactivate', ) + auto_activate_on_configure_arg = DeclareLaunchArgument( + 'auto_activate_on_configure', + default_value='True', + description='activate the lifecycle node after configure', + ) + ros_typedb_node = LifecycleNode( package='ros_typedb', executable='ros_typedb', @@ -131,6 +141,24 @@ def generate_launch_description(): ), ) + initial_activate_event = RegisterEventHandler( + OnStateTransition( + target_lifecycle_node=ros_typedb_node, + start_state='configuring', + goal_state='inactive', + entities=[ + EmitEvent(event=ChangeState( + lifecycle_node_matcher=matches_action(ros_typedb_node), + transition_id=( + lifecycle_msgs.msg.Transition.TRANSITION_ACTIVATE + ), + )), + ], + handle_once=True, + ), + condition=UnlessCondition(auto_activate_on_configure), + ) + activate_event = RegisterEventHandler( OnStateTransition( target_lifecycle_node=ros_typedb_node, @@ -145,6 +173,7 @@ def generate_launch_description(): )), ], ), + condition=IfCondition(auto_activate_on_configure), ) reactivate_event = RegisterEventHandler( @@ -172,9 +201,11 @@ def generate_launch_description(): reload_schema_arg, infer_arg, query_timeout_s_arg, + auto_activate_on_configure_arg, reactivate_on_deactivate_arg, ros_typedb_node, configure_event, + initial_activate_event, activate_event, reactivate_event, ]) From 5172a6fda823a886e82dd2fbd563947378e8a7c2 Mon Sep 17 00:00:00 2001 From: Gustavo Date: Sat, 25 Jul 2026 17:14:06 -0300 Subject: [PATCH 36/36] update benchmarks --- .serena/project.yml | 140 +++-- ros_typedb_benchmark/package.xml | 1 + .../profiles/my_application_profile.yaml | 74 +++ .../ros_typedb_stress_experiment.py | 22 + .../ros_typedb_benchmark/stress_config.py | 3 + .../ros_typedb_benchmark/stress_profile.py | 205 +++++++ .../ros_typedb_benchmark/stress_resolution.py | 123 ++++- .../ros_typedb_benchmark/stress_runner.py | 132 +++++ ros_typedb_benchmark/setup.py | 2 +- .../test/test_ros_typedb_stress_experiment.py | 513 ++++++++++++++++++ 10 files changed, 1128 insertions(+), 87 deletions(-) create mode 100644 ros_typedb_benchmark/ros_typedb_benchmark/profiles/my_application_profile.yaml create mode 100644 ros_typedb_benchmark/ros_typedb_benchmark/stress_profile.py diff --git a/.serena/project.yml b/.serena/project.yml index c46a2bd..76fec0f 100644 --- a/.serena/project.yml +++ b/.serena/project.yml @@ -1,24 +1,31 @@ -# the name by which the project can be referenced within Serena +# the name by which the project can be referenced within Serena/when chatting with the LLM. project_name: "ros_typedb" - -# list of languages for which language servers are started; choose from: -# al bash clojure cpp csharp -# csharp_omnisharp dart elixir elm erlang -# fortran fsharp go groovy haskell -# haxe java julia kotlin lua -# markdown -# matlab nix pascal perl php -# php_phpactor powershell python python_jedi r +# list of languages for which language servers are started (LSP backend only); choose from: +# ada al angular ansible bash +# bsl clojure cpp cpp_ccls crystal +# csharp csharp_omnisharp cue dart elixir +# elm erlang fortran fsharp gdscript +# go groovy haskell haxe hlsl +# html java json julia kotlin +# latex lean4 lua luau markdown +# matlab msl nix ocaml pascal +# perl php php_phpactor php_phpantom powershell +# python python_jedi python_pyrefly python_ty r # rego ruby ruby_solargraph rust scala -# swift terraform toml typescript typescript_vts -# vue yaml zig -# (This list may be outdated. For the current list, see values of Language enum here: -# https://github.com/oraios/serena/blob/main/src/solidlsp/ls_config.py -# For some languages, there are alternative language servers, e.g. csharp_omnisharp, ruby_solargraph.) +# scss solidity svelte swift systemverilog +# terraform toml typescript typescript_vts vue +# yaml zig +# (This list may be outdated; generated with scripts/print_language_list.py; +# For the current list, see values of Language enum here: +# https://github.com/oraios/serena/blob/main/src/solidlsp/ls_config.py) +# For some languages, there are alternative language servers, e.g. csharp_omnisharp, ruby_solargraph.) # Note: # - For C, use cpp # - For JavaScript, use typescript +# - For Angular projects, use angular (subsumes typescript+html; requires `npm install` in the project root) +# - For Svelte projects, use svelte (subsumes typescript/javascript for .svelte projects; requires npm) +# - For SCSS / Sass / plain CSS, use scss (some-sass-language-server handles all three) # - For Free Pascal/Lazarus, use pascal # Special requirements: # Some languages require additional setup/installations. @@ -50,21 +57,10 @@ ignore_all_files_in_gitignore: true # advanced configuration option allowing to configure language server-specific options. # Maps the language key to the options. -# Have a look at the docstring of the constructors of the LS implementations within solidlsp (e.g., for C# or PHP) to see which options are available. -# No documentation on options means no options are available. +# The settings are considered only if the project is trusted (see global configuration to define trusted projects). +# See https://oraios.github.io/serena/02-usage/050_configuration.html#language-server-specific-settings ls_specific_settings: {} -# list of additional workspace folder paths for cross-package reference support (e.g. in monorepos). -# Paths can be absolute or relative to the project root. -# Each folder is registered as an LSP workspace folder, enabling language servers to discover -# symbols and references across package boundaries. -# Currently supported for: TypeScript. -# Example: -# additional_workspace_folders: -# - ../sibling-package -# - ../shared-lib -additional_workspace_folders: [] - # list of additional paths to ignore in this project. # Same syntax as gitignore, so you can use * and **. # Note: global ignored_paths from serena_config.yml are also applied additively. @@ -77,61 +73,27 @@ read_only: false # list of tool names to exclude. # This extends the existing exclusions (e.g. from the global configuration) -# -# Below is the complete list of tools for convenience. -# To make sure you have the latest list of tools, and to view their descriptions, -# execute `uv run scripts/print_tool_overview.py`. -# -# * `activate_project`: Activates a project based on the project name or path. -# * `check_onboarding_performed`: Checks whether project onboarding was already performed. -# * `create_text_file`: Creates/overwrites a file in the project directory. -# * `delete_memory`: Delete a memory file. Should only happen if a user asks for it explicitly, -# for example by saying that the information retrieved from a memory file is no longer correct -# or no longer relevant for the project. -# * `edit_memory`: Replaces content matching a regular expression in a memory. -# * `execute_shell_command`: Executes a shell command. -# * `find_file`: Finds files in the given relative paths -# * `find_referencing_symbols`: Finds symbols that reference the given symbol using the language server backend -# * `find_symbol`: Performs a global (or local) search using the language server backend. -# * `get_current_config`: Prints the current configuration of the agent, including the active and available projects, tools, contexts, and modes. -# * `get_symbols_overview`: Gets an overview of the top-level symbols defined in a given file. -# * `initial_instructions`: Provides instructions Serena usage (i.e. the 'Serena Instructions Manual') -# for clients that do not read the initial instructions when the MCP server is connected. -# * `insert_after_symbol`: Inserts content after the end of the definition of a given symbol. -# * `insert_before_symbol`: Inserts content before the beginning of the definition of a given symbol. -# * `list_dir`: Lists files and directories in the given directory (optionally with recursion). -# * `list_memories`: List available memories. Any memory can be read using the `read_memory` tool. -# * `onboarding`: Performs onboarding (identifying the project structure and essential tasks, e.g. for testing or building). -# * `read_file`: Reads a file within the project directory. -# * `read_memory`: Read the content of a memory file. This tool should only be used if the information -# is relevant to the current task. You can infer whether the information -# is relevant from the memory file name. -# You should not read the same memory file multiple times in the same conversation. -# * `rename_memory`: Renames or moves a memory. Moving between project and global scope is supported -# (e.g., renaming "global/foo" to "bar" moves it from global to project scope). -# * `rename_symbol`: Renames a symbol throughout the codebase using language server refactoring capabilities. -# For JB, we use a separate tool. -# * `replace_content`: Replaces content in a file (optionally using regular expressions). -# * `replace_symbol_body`: Replaces the full definition of a symbol using the language server backend. -# * `safe_delete_symbol`: -# * `search_for_pattern`: Performs a search for a pattern in the project. -# * `write_memory`: Write some information (utf-8-encoded) about this project that can be useful for future tasks to a memory in md format. -# The memory name should be meaningful. +# Find the list of tools here: https://oraios.github.io/serena/01-about/035_tools.html excluded_tools: [] # list of tools to include that would otherwise be disabled (particularly optional tools that are disabled by default). # This extends the existing inclusions (e.g. from the global configuration). +# Find the list of tools here: https://oraios.github.io/serena/01-about/035_tools.html included_optional_tools: [] # fixed set of tools to use as the base tool set (if non-empty), replacing Serena's default set of tools. # This cannot be combined with non-empty excluded_tools or included_optional_tools. +# Find the list of tools here: https://oraios.github.io/serena/01-about/035_tools.html fixed_tools: [] -# list of mode names that are to be activated by default. -# The full set of modes to be activated is base_modes + default_modes. -# If the setting is undefined, the default_modes from the global configuration (serena_config.yml) apply. +# list of mode names that are to be activated by default, overriding the setting in the global configuration. +# The full set of modes to be activated is base_modes (from global config) + default_modes + added_modes. +# If the setting is undefined/empty, the default_modes from the global configuration (serena_config.yml) apply. # Otherwise, this overrides the setting from the global configuration (serena_config.yml). +# Therefore, you can set this to [] if you do not want the default modes defined in the global config to apply +# for this project. # This setting can, in turn, be overridden by CLI parameters (--mode). +# See https://oraios.github.io/serena/02-usage/050_configuration.html#modes default_modes: # list of mode names to be activated additionally for this project, e.g. ["query-projects"] @@ -168,3 +130,39 @@ ignored_memory_patterns: [] # Set this to [] to disable base modes for this project. # Set this to a list of mode names to always include the respective modes for this project. base_modes: + +# list of additional workspace folder paths for cross-package reference support. +# Paths can be absolute or relative to the project root. +# Each folder is registered as an LSP workspace folder, enabling language servers to discover +# symbols and references across package boundaries, but these folders are not indexed by Serena, +# i.e. the respective symbols will not be found using Serena's symbol search tools. +# Example: +# additional_workspace_folders: +# - ../sibling-package +# - ../shared-lib +ls_additional_workspace_folders: [] + +# list of workspace folder paths (LSP backend only). +# These folders will be used to build up Serena's symbol index. +# Paths must be within the project root and should thus be relative to the project root. +# Furthermore, the paths should not be filtered by ignore settings. +# Default setting: The entire project root folder (".") is considered. +# In (large) monorepos, this can be used to index only subfolders of the project root, e.g. +# ls_workspace_folders: +# - "./subproject1" +# - "./subproject2" +ls_workspace_folders: +- . + +# optional shell command to run before the language backend (LSP or JetBrains) is initialised. +# the command runs in the project root directory and is only executed if the project is trusted +# (see trusted_project_path_patterns in the global configuration). +# serena waits for the command to exit: a non-zero exit code is logged as an error but does not +# abort activation. a per-project timeout (activation_command_timeout, default 180s) is the safety +# backstop for non-terminating commands; on expiry the process is killed and activation continues. +# example: activation_command: "npx nx run-many -t build" +activation_command: + +# maximum time in seconds to wait for activation_command to complete before killing it (default 180s). +# must be a positive number. +activation_command_timeout: 180.0 diff --git a/ros_typedb_benchmark/package.xml b/ros_typedb_benchmark/package.xml index ee54723..cc683e8 100644 --- a/ros_typedb_benchmark/package.xml +++ b/ros_typedb_benchmark/package.xml @@ -9,6 +9,7 @@ rclpy ros_typedb_msgs + python3-yaml ament_copyright ament_flake8 diff --git a/ros_typedb_benchmark/ros_typedb_benchmark/profiles/my_application_profile.yaml b/ros_typedb_benchmark/ros_typedb_benchmark/profiles/my_application_profile.yaml new file mode 100644 index 0000000..6777805 --- /dev/null +++ b/ros_typedb_benchmark/ros_typedb_benchmark/profiles/my_application_profile.yaml @@ -0,0 +1,74 @@ +# Sample external stress profile for ros_typedb_benchmark. +# +# Copy this file, fill in the sections relevant to your application, and +# remove the rest. Pass it with: +# +# ros_typedb_stress_experiment --profile-file /path/to/my_profile.yaml +# +# All paths (schema_path, data_path) are resolved relative to THIS file. + +description: "My application stress profile" + +# --------------------------------------------------------------------------- +# Setup — runs once BEFORE the load experiment starts +# --------------------------------------------------------------------------- + +# Optional: path to a TypeDB schema TQL file sent as a 'define' query. +# schema_path: schema/my_schema.tql + +# Optional: path to a TypeDB data TQL file sent as an 'insert' query +# after the schema (if given). +# data_path: data/my_data.tql + +# Optional: arbitrary setup queries executed after schema_path and data_path. +# Useful when setup logic is more complex than a single file. +# setup_queries: +# - query: "insert $x isa my-entity, has name 'seed';" +# query_type: insert + +# --------------------------------------------------------------------------- +# Teardown — runs once AFTER the final invariant checks +# --------------------------------------------------------------------------- + +# teardown_queries: +# - query: "match $x isa my-entity; delete $x isa my-entity;" +# query_type: delete + +# --------------------------------------------------------------------------- +# Read queries — used in --mode read (the default) +# --------------------------------------------------------------------------- + +# read_queries: +# - query: "match $x isa my-entity; get $x;" +# query_type: get +# - query: "match $x isa my-relation; get $x;" +# query_type: get + +# --------------------------------------------------------------------------- +# Mixed mode — used in --mode mixed +# Conflicts with --mixed-profile (omit that flag or set it to 'auto') +# --------------------------------------------------------------------------- + +# mixed: +# # Key template used to generate and clean up unique entities per run. +# # {key} is replaced with a UUID-derived string each iteration. +# cleanup_query: > +# match $x isa my-entity, has name "{key}"; +# delete $x isa my-entity; +# queries: +# - query: "insert $x isa my-entity, has name \"{key}\";" +# query_type: insert +# tracks_cleanup: true # this query creates the {key} entity +# - query: "match $x isa my-entity; get $x;" +# query_type: get + +# --------------------------------------------------------------------------- +# Invariants — run periodically and at experiment end +# Conflicts with --invariant-profile (omit that flag or set it to 'none') +# --------------------------------------------------------------------------- + +# invariants: +# - name: entity_count +# query: "match $x isa my-entity; get $x; count;" +# query_type: get_aggregate +# expected_value: 1 diff --git a/ros_typedb_benchmark/ros_typedb_benchmark/ros_typedb_stress_experiment.py b/ros_typedb_benchmark/ros_typedb_benchmark/ros_typedb_stress_experiment.py index bc4cea8..58188a2 100644 --- a/ros_typedb_benchmark/ros_typedb_benchmark/ros_typedb_stress_experiment.py +++ b/ros_typedb_benchmark/ros_typedb_benchmark/ros_typedb_stress_experiment.py @@ -299,6 +299,28 @@ def build_argument_parser() -> argparse.ArgumentParser: default=10.0, help='Seconds to wait for each lifecycle transition. Defaults to 10.', ) + parser.add_argument( + '--profile-file', + help=( + 'Path to a YAML or JSON stress profile. Overrides ' + '--invariant-profile and --mixed-profile for sections ' + 'the file defines.' + ), + ) + parser.add_argument( + '--schema-path', + help=( + 'Path to a TypeDB schema TQL file. Sent as a define ' + 'setup query before the experiment starts.' + ), + ) + parser.add_argument( + '--data-path', + help=( + 'Path to a TypeDB data TQL file. Sent as an insert ' + 'setup query after --schema-path (if given).' + ), + ) return parser diff --git a/ros_typedb_benchmark/ros_typedb_benchmark/stress_config.py b/ros_typedb_benchmark/ros_typedb_benchmark/stress_config.py index 3d65bd7..81d7cb6 100644 --- a/ros_typedb_benchmark/ros_typedb_benchmark/stress_config.py +++ b/ros_typedb_benchmark/ros_typedb_benchmark/stress_config.py @@ -17,6 +17,7 @@ import argparse from dataclasses import dataclass +from dataclasses import field import json from pathlib import Path from typing import Any @@ -158,6 +159,8 @@ class ResolvedStressConfig: lifecycle_get_state_service_name: str | None = None lifecycle_reactivate: bool = True lifecycle_transition_timeout_s: float = 10.0 + setup_specs: list[QuerySpec] = field(default_factory=list) + teardown_specs: list[QuerySpec] = field(default_factory=list) @dataclass(frozen=True) diff --git a/ros_typedb_benchmark/ros_typedb_benchmark/stress_profile.py b/ros_typedb_benchmark/ros_typedb_benchmark/stress_profile.py new file mode 100644 index 0000000..24af17b --- /dev/null +++ b/ros_typedb_benchmark/ros_typedb_benchmark/stress_profile.py @@ -0,0 +1,205 @@ +# Copyright 2026 Gustavo Rezende Silva +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +"""Loader for external YAML/JSON stress experiment profiles.""" + +from __future__ import annotations + +import json +from dataclasses import dataclass +from pathlib import Path + +from ros_typedb_benchmark.stress_config import InvariantSpec +from ros_typedb_benchmark.stress_config import QUERY_TYPE_BY_NAME +from ros_typedb_benchmark.stress_config import QuerySpec +from ros_typedb_benchmark.stress_mixed import MixedQueryProfile +from ros_typedb_benchmark.stress_mixed import MixedQueryTemplate + + +@dataclass(frozen=True) +class StressProfile: + """Loaded external stress profile.""" + + description: str + setup_specs: list[QuerySpec] + teardown_specs: list[QuerySpec] + read_query_specs: list[QuerySpec] | None + mixed_profile: MixedQueryProfile | None + invariant_specs: list[InvariantSpec] | None + + +def load_stress_profile(profile_path: Path) -> StressProfile: + """Load a YAML or JSON stress profile from disk. + + YAML is used for ``.yaml``/``.yml`` extensions; JSON for everything else. + Raises ``FileNotFoundError`` if the file does not exist, ``ValueError`` + for invalid content. + """ + profile_path = Path(profile_path) + if not profile_path.exists(): + raise FileNotFoundError( + f'profile file not found: {profile_path}' + ) + raw = profile_path.read_text(encoding='utf-8') + if profile_path.suffix.lower() in ('.yaml', '.yml'): + import yaml # noqa: PLC0415 + payload = yaml.safe_load(raw) or {} + else: + payload = json.loads(raw) + + description = payload.get('description', '') + setup_specs = _parse_setup_specs(payload, profile_path) + teardown_specs = _parse_query_spec_list( + payload.get('teardown_queries', []), 'teardown_queries' + ) + read_query_specs = _parse_read_query_specs(payload) + mixed_profile = _parse_mixed_profile(payload) + invariant_specs = _parse_invariant_specs(payload) + + return StressProfile( + description=description, + setup_specs=setup_specs, + teardown_specs=teardown_specs, + read_query_specs=read_query_specs, + mixed_profile=mixed_profile, + invariant_specs=invariant_specs, + ) + + +def _parse_setup_specs( + payload: dict, + profile_path: Path, +) -> list[QuerySpec]: + """Build setup specs from schema_path, data_path, and setup_queries.""" + specs: list[QuerySpec] = [] + + schema_path = payload.get('schema_path') + if schema_path: + schema_file = _resolve_path(schema_path, profile_path) + specs.append( + QuerySpec( + query=schema_file.read_text(encoding='utf-8'), + query_type='define', + ) + ) + + data_path = payload.get('data_path') + if data_path: + data_file = _resolve_path(data_path, profile_path) + specs.append( + QuerySpec( + query=data_file.read_text(encoding='utf-8'), + query_type='insert', + ) + ) + + specs.extend( + _parse_query_spec_list( + payload.get('setup_queries', []), 'setup_queries' + ) + ) + return specs + + +def _resolve_path(path_str: str, profile_path: Path) -> Path: + """Resolve a path from the profile, relative to the profile file.""" + p = Path(path_str) + if not p.is_absolute(): + p = profile_path.parent / p + if not p.exists(): + raise FileNotFoundError(f'referenced file not found: {p}') + return p + + +def _parse_query_spec_list( + items: list, + section: str, +) -> list[QuerySpec]: + """Parse a list of query dicts into QuerySpec objects.""" + specs = [] + for item in items: + query_type = item.get('query_type', '') + if query_type not in QUERY_TYPE_BY_NAME: + raise ValueError( + f'unsupported query type {query_type!r} in {section}' + ) + specs.append( + QuerySpec( + query=item['query'], + query_type=query_type, + ) + ) + return specs + + +def _parse_read_query_specs(payload: dict) -> list[QuerySpec] | None: + """Parse read_queries section; None if absent.""" + raw = payload.get('read_queries') + if raw is None: + return None + return _parse_query_spec_list(raw, 'read_queries') + + +def _parse_mixed_profile(payload: dict) -> MixedQueryProfile | None: + """Parse mixed section; None if absent.""" + raw = payload.get('mixed') + if raw is None: + return None + templates = [] + for item in raw.get('queries', []): + query_type = item.get('query_type', '') + if query_type not in QUERY_TYPE_BY_NAME: + raise ValueError( + f'unsupported query type {query_type!r} in mixed.queries' + ) + templates.append( + MixedQueryTemplate( + query=item['query'], + query_type=query_type, + tracks_cleanup=item.get('tracks_cleanup', False), + clears_cleanup=item.get('clears_cleanup', False), + ) + ) + if not templates: + raise ValueError('mixed profile has no queries') + cleanup_query = raw.get('cleanup_query', '') + if not cleanup_query: + raise ValueError('mixed profile missing cleanup_query') + return MixedQueryProfile( + name='external', + cleanup_query=cleanup_query, + templates=templates, + ) + + +def _parse_invariant_specs(payload: dict) -> list[InvariantSpec] | None: + """Parse invariants section; None if absent.""" + raw = payload.get('invariants') + if raw is None: + return None + specs = [] + for item in raw: + query_type = item.get('query_type', '') + if query_type not in QUERY_TYPE_BY_NAME: + raise ValueError( + f'unsupported query type {query_type!r} in invariants' + ) + specs.append( + InvariantSpec( + name=item['name'], + query=item['query'], + query_type=query_type, + expected_value=item['expected_value'], + ) + ) + return specs diff --git a/ros_typedb_benchmark/ros_typedb_benchmark/stress_resolution.py b/ros_typedb_benchmark/ros_typedb_benchmark/stress_resolution.py index 4962678..05493ef 100644 --- a/ros_typedb_benchmark/ros_typedb_benchmark/stress_resolution.py +++ b/ros_typedb_benchmark/ros_typedb_benchmark/stress_resolution.py @@ -27,6 +27,8 @@ from ros_typedb_benchmark.stress_config import validate_experiment_args from ros_typedb_benchmark.stress_mixed import load_mixed_query_profile from ros_typedb_benchmark.stress_mixed import MixedQueryProfile +from ros_typedb_benchmark.stress_profile import load_stress_profile +from ros_typedb_benchmark.stress_profile import StressProfile def resolve_stress_config( @@ -42,9 +44,17 @@ def resolve_stress_config( """ validate_experiment_args(args) - invariant_specs = build_invariant_specs(args) - mixed_profile = _resolve_dynamic_mixed_profile(args) - query_specs = _build_runtime_query_specs(args, mixed_profile) + external_profile = _load_external_profile(args) + _check_profile_conflicts(args, external_profile) + + setup_specs = _build_setup_specs(args, external_profile) + teardown_specs = ( + external_profile.teardown_specs if external_profile else [] + ) + invariant_specs = _resolve_invariant_specs(args, external_profile) + mixed_profile = _resolve_mixed_profile(args, external_profile) + query_specs = _build_runtime_query_specs(args, mixed_profile, + external_profile) debug_events_output = ( Path(args.debug_events_output).expanduser() if args.debug_events_output @@ -97,13 +107,95 @@ def resolve_stress_config( lifecycle_get_state_service_name=lifecycle_get_state_service_name, lifecycle_reactivate=lifecycle_reactivate, lifecycle_transition_timeout_s=lifecycle_transition_timeout_s, + setup_specs=setup_specs, + teardown_specs=teardown_specs, ) -def _resolve_dynamic_mixed_profile( +def _load_external_profile( + args: argparse.Namespace, +) -> StressProfile | None: + """Load the external profile file if --profile-file is given.""" + profile_file = getattr(args, 'profile_file', None) + if not profile_file: + return None + return load_stress_profile(Path(profile_file)) + + +def _check_profile_conflicts( args: argparse.Namespace, + profile: StressProfile | None, +) -> None: + """Raise ValueError when profile file and CLI selectors define the same section.""" + if profile is None: + return + if ( + profile.invariant_specs is not None + and args.invariant_profile != 'none' + ): + raise ValueError( + '--profile-file defines invariants; remove --invariant-profile ' + 'or set it to none' + ) + if ( + profile.mixed_profile is not None + and args.mode == 'mixed' + and getattr(args, 'mixed_profile', 'auto') != 'auto' + ): + raise ValueError( + '--profile-file defines a mixed section; remove --mixed-profile ' + 'or leave it at auto' + ) + + +def _build_setup_specs( + args: argparse.Namespace, + profile: StressProfile | None, +) -> list[QuerySpec]: + """Build setup specs from CLI flags and profile file.""" + specs: list[QuerySpec] = [] + + schema_path = getattr(args, 'schema_path', None) + if schema_path: + specs.append( + QuerySpec( + query=Path(schema_path).read_text(encoding='utf-8'), + query_type='define', + ) + ) + + data_path = getattr(args, 'data_path', None) + if data_path: + specs.append( + QuerySpec( + query=Path(data_path).read_text(encoding='utf-8'), + query_type='insert', + ) + ) + + if profile is not None: + specs.extend(profile.setup_specs) + + return specs + + +def _resolve_invariant_specs( + args: argparse.Namespace, + profile: StressProfile | None, +) -> list: + """Profile invariants win; fall back to CLI --invariant-profile.""" + if profile is not None and profile.invariant_specs is not None: + return profile.invariant_specs + return build_invariant_specs(args) + + +def _resolve_mixed_profile( + args: argparse.Namespace, + profile: StressProfile | None, ) -> MixedQueryProfile | None: - """Load a mixed profile only when the workload is profile-driven.""" + """Profile mixed section wins; fall back to CLI --mode mixed.""" + if profile is not None and profile.mixed_profile is not None: + return profile.mixed_profile if args.mode == 'mixed' and args.query is None: return load_mixed_query_profile(args) return None @@ -112,15 +204,16 @@ def _resolve_dynamic_mixed_profile( def _build_runtime_query_specs( args: argparse.Namespace, mixed_profile: MixedQueryProfile | None, + profile: StressProfile | None, ) -> list[QuerySpec]: """Build fixed query specs or metadata specs for dynamic mixed mode.""" - if mixed_profile is None: - return build_query_specs(args) - - # Dynamic mixed requests are rendered per client and request index at - # runtime. These QuerySpecs preserve the selected profile's query mix for - # JSON output and debug readability. - return [ - QuerySpec(query=template.query, query_type=template.query_type) - for template in mixed_profile.templates - ] + if mixed_profile is not None: + return [ + QuerySpec(query=template.query, query_type=template.query_type) + for template in mixed_profile.templates + ] + + if profile is not None and profile.read_query_specs is not None: + return profile.read_query_specs + + return build_query_specs(args) diff --git a/ros_typedb_benchmark/ros_typedb_benchmark/stress_runner.py b/ros_typedb_benchmark/ros_typedb_benchmark/stress_runner.py index 272dbda..729e64d 100644 --- a/ros_typedb_benchmark/ros_typedb_benchmark/stress_runner.py +++ b/ros_typedb_benchmark/ros_typedb_benchmark/stress_runner.py @@ -656,6 +656,104 @@ def _run_cleanup_request( return +def _run_oneshot_query( + *, + node: Any, + executor: Any | None, + client: Any, + state: _RunState, + query_spec: QuerySpec, + timeout_s: float, + phase: str, + debug_events: DebugEventWriter, +) -> None: + """Send one synchronous query and record the result. + + Used for setup and teardown queries that run outside the main load loop. + """ + started_at_s = time.monotonic() + request = build_query_request( + query_spec.query, + query_spec.query_type, + timeout_s, + ) + future = client.call_async(request) + pending = _PendingRequest( + client_id=-1, + index=state.next_index, + query_index=-1, + query_spec=query_spec, + future=future, + started_at_s=started_at_s, + ) + state.next_index += 1 + debug_events.log( + 'oneshot_query_sent', + phase=phase, + index=pending.index, + query_type=query_spec.query_type, + ) + + timed_out, now_s = _spin_until_future_done( + future, node, executor, timeout_s, started_at_s + ) + if timed_out: + _record_timeout( + pending=pending, + now_s=now_s, + pending_count=1, + state=state, + debug_events=debug_events, + ) + debug_events.log( + 'oneshot_query_timeout', + phase=phase, + index=pending.index, + ) + return + _record_done_future( + pending=pending, + now_s=now_s, + pending_count=1, + state=state, + debug_events=debug_events, + ) + debug_events.log( + 'oneshot_query_complete', + phase=phase, + index=pending.index, + ) + + +def _run_phase_queries( + *, + node: Any, + executor: Any | None, + client: Any, + specs: list[QuerySpec], + state: _RunState, + phase: str, + timeout_s: float, + debug_events: DebugEventWriter, +) -> None: + """Run a list of oneshot queries sequentially for setup or teardown.""" + if not specs: + return + debug_events.log('phase_queries_started', phase=phase, count=len(specs)) + for spec in specs: + _run_oneshot_query( + node=node, + executor=executor, + client=client, + state=state, + query_spec=spec, + timeout_s=timeout_s, + phase=phase, + debug_events=debug_events, + ) + debug_events.log('phase_queries_finished', phase=phase) + + def _cleanup_mixed_data( *, args: argparse.Namespace, @@ -1376,6 +1474,11 @@ def run_experiment(args: argparse.Namespace) -> StressExperimentResult: if config.fault == 'lifecycle-cleanup' else None ) + setup_teardown_client = ( + node.create_client(Query, args.service_name) + if config.setup_specs or config.teardown_specs + else None + ) deadline_s = ( time.monotonic() + args.duration_s if args.duration_s is not None @@ -1425,11 +1528,27 @@ def run_experiment(args: argparse.Namespace) -> StressExperimentResult: [lifecycle_get_state_client] if lifecycle_get_state_client is not None else [] + ) + + ( + [setup_teardown_client] + if setup_teardown_client is not None + else [] ), service_name=args.service_name, wait_service_timeout_s=args.wait_service_timeout_s, debug_events=debug_events, ) + if setup_teardown_client is not None and config.setup_specs: + _run_phase_queries( + node=node, + executor=executor, + client=setup_teardown_client, + specs=config.setup_specs, + state=state, + phase='setup', + timeout_s=args.timeout_s, + debug_events=debug_events, + ) experiment_started_at_s = time.monotonic() while ( @@ -1571,6 +1690,19 @@ def run_experiment(args: argparse.Namespace) -> StressExperimentResult: debug_events=debug_events, ) + # 4. Teardown (complement of setup — runs last) + if setup_teardown_client is not None and config.teardown_specs: + _run_phase_queries( + node=node, + executor=executor, + client=setup_teardown_client, + specs=config.teardown_specs, + state=state, + phase='teardown', + timeout_s=args.timeout_s, + debug_events=debug_events, + ) + debug_events.log( 'experiment_finished', records=len(state.records), diff --git a/ros_typedb_benchmark/setup.py b/ros_typedb_benchmark/setup.py index 0452e5f..c9dc7e4 100644 --- a/ros_typedb_benchmark/setup.py +++ b/ros_typedb_benchmark/setup.py @@ -9,7 +9,7 @@ version='0.0.0', packages=find_packages(exclude=['test']), package_data={ - package_name: ['profiles/*.json'], + package_name: ['profiles/*.json', 'profiles/*.yaml'], }, data_files=[ ('share/ament_index/resource_index/packages', diff --git a/ros_typedb_benchmark/test/test_ros_typedb_stress_experiment.py b/ros_typedb_benchmark/test/test_ros_typedb_stress_experiment.py index 0a39c32..4ef9b52 100644 --- a/ros_typedb_benchmark/test/test_ros_typedb_stress_experiment.py +++ b/ros_typedb_benchmark/test/test_ros_typedb_stress_experiment.py @@ -6,6 +6,7 @@ from types import SimpleNamespace import pytest +import yaml from rcl_interfaces.msg import ParameterType from rcl_interfaces.msg import ParameterValue @@ -28,6 +29,7 @@ InvariantRecord, InvariantSpec, QUERY_TYPE_BY_NAME, + QuerySpec, RequestRecord, ResolvedStressConfig, StressExperimentResult, @@ -149,6 +151,9 @@ def exception(self): def result(self): return self._result + def cancel(self): + pass + class _CapturingClient: """Minimal service client that captures the last async request.""" @@ -1558,3 +1563,511 @@ def test_main_fails_final_invariant_failure_after_recovered_fault(monkeypatch): monkeypatch.setattr(stress_cli, 'run_experiment', lambda args: result) assert stress_cli.main([]) == 1 + + +# --------------------------------------------------------------------------- +# Task 1: stress_profile.py tests +# --------------------------------------------------------------------------- + +def _write_yaml(tmp_path, data, filename='profile.yaml'): + """Write data as YAML to a temp file and return the Path.""" + p = Path(tmp_path) / filename + p.write_text(yaml.dump(data), encoding='utf-8') + return p + + +def _write_json_file(tmp_path, data, filename='profile.json'): + """Write data as JSON to a temp file and return the Path.""" + p = Path(tmp_path) / filename + p.write_text(json.dumps(data), encoding='utf-8') + return p + + +def test_load_stress_profile_empty_yaml(tmp_path): + """Minimal profile with no sections loads without error.""" + from ros_typedb_benchmark.stress_profile import load_stress_profile + p = _write_yaml(tmp_path, {'description': 'test'}) + profile = load_stress_profile(p) + assert profile.description == 'test' + assert profile.setup_specs == [] + assert profile.teardown_specs == [] + assert profile.read_query_specs is None + assert profile.mixed_profile is None + assert profile.invariant_specs is None + + +def test_load_stress_profile_json_extension(tmp_path): + """JSON files are loaded when extension is .json.""" + from ros_typedb_benchmark.stress_profile import load_stress_profile + p = _write_json_file(tmp_path, {'description': 'json-test'}) + profile = load_stress_profile(p) + assert profile.description == 'json-test' + + +def test_load_stress_profile_setup_queries(tmp_path): + """setup_queries become QuerySpec list.""" + from ros_typedb_benchmark.stress_profile import load_stress_profile + data = { + 'setup_queries': [ + {'query': 'define person sub entity;', 'query_type': 'define'}, + ] + } + p = _write_yaml(tmp_path, data) + profile = load_stress_profile(p) + assert len(profile.setup_specs) == 1 + assert profile.setup_specs[0].query == 'define person sub entity;' + assert profile.setup_specs[0].query_type == 'define' + assert profile.setup_specs[0].cleanup_key is None + + +def test_load_stress_profile_teardown_queries(tmp_path): + """teardown_queries become QuerySpec list.""" + from ros_typedb_benchmark.stress_profile import load_stress_profile + data = { + 'teardown_queries': [ + { + 'query': 'match $x isa person; delete $x isa person;', + 'query_type': 'delete', + }, + ] + } + p = _write_yaml(tmp_path, data) + profile = load_stress_profile(p) + assert len(profile.teardown_specs) == 1 + assert profile.teardown_specs[0].query_type == 'delete' + + +def test_load_stress_profile_schema_path(tmp_path): + """schema_path in profile is read and placed as first setup_spec.""" + from ros_typedb_benchmark.stress_profile import load_stress_profile + schema_file = Path(tmp_path) / 'schema.tql' + schema_file.write_text('define robot sub entity;', encoding='utf-8') + data = {'schema_path': str(schema_file)} + p = _write_yaml(tmp_path, data) + profile = load_stress_profile(p) + assert len(profile.setup_specs) == 1 + assert 'define robot sub entity;' in profile.setup_specs[0].query + assert profile.setup_specs[0].query_type == 'define' + + +def test_load_stress_profile_data_path(tmp_path): + """data_path in profile is read and placed as a setup_spec.""" + from ros_typedb_benchmark.stress_profile import load_stress_profile + data_file = Path(tmp_path) / 'data.tql' + data_file.write_text('insert $r isa robot;', encoding='utf-8') + data = {'data_path': str(data_file)} + p = _write_yaml(tmp_path, data) + profile = load_stress_profile(p) + assert len(profile.setup_specs) == 1 + assert profile.setup_specs[0].query_type == 'insert' + + +def test_load_stress_profile_schema_before_data(tmp_path): + """schema_path setup spec appears before data_path setup spec.""" + from ros_typedb_benchmark.stress_profile import load_stress_profile + schema_file = Path(tmp_path) / 'schema.tql' + schema_file.write_text('define robot sub entity;', encoding='utf-8') + data_file = Path(tmp_path) / 'data.tql' + data_file.write_text('insert $r isa robot;', encoding='utf-8') + data = {'schema_path': str(schema_file), 'data_path': str(data_file)} + p = _write_yaml(tmp_path, data) + profile = load_stress_profile(p) + assert len(profile.setup_specs) == 2 + assert profile.setup_specs[0].query_type == 'define' + assert profile.setup_specs[1].query_type == 'insert' + + +def test_load_stress_profile_read_queries(tmp_path): + """read_queries populate read_query_specs.""" + from ros_typedb_benchmark.stress_profile import load_stress_profile + data = { + 'read_queries': [ + {'query': 'match $x isa entity; get $x;', 'query_type': 'get'}, + ] + } + p = _write_yaml(tmp_path, data) + profile = load_stress_profile(p) + assert profile.read_query_specs is not None + assert len(profile.read_query_specs) == 1 + assert profile.read_query_specs[0].query_type == 'get' + + +def test_load_stress_profile_invariants(tmp_path): + """invariants section populates invariant_specs.""" + from ros_typedb_benchmark.stress_profile import load_stress_profile + data = { + 'invariants': [ + { + 'name': 'entity-count', + 'query': 'match $x isa entity; get $x; count;', + 'query_type': 'get_aggregate', + 'expected_value': 5, + } + ] + } + p = _write_yaml(tmp_path, data) + profile = load_stress_profile(p) + assert profile.invariant_specs is not None + assert len(profile.invariant_specs) == 1 + assert profile.invariant_specs[0].name == 'entity-count' + assert profile.invariant_specs[0].expected_value == 5 + + +def test_load_stress_profile_mixed(tmp_path): + """mixed section populates mixed_profile.""" + from ros_typedb_benchmark.stress_profile import load_stress_profile + data = { + 'mixed': { + 'cleanup_query': ( + 'match $r isa robot, has name "{key}";' + ' delete $r isa robot;' + ), + 'queries': [ + { + 'query': 'insert $r isa robot, has name "{key}";', + 'query_type': 'insert', + 'tracks_cleanup': True, + }, + { + 'query': 'match $r isa robot; get $r;', + 'query_type': 'get', + }, + ], + } + } + p = _write_yaml(tmp_path, data) + profile = load_stress_profile(p) + assert profile.mixed_profile is not None + assert len(profile.mixed_profile.templates) == 2 + assert profile.mixed_profile.templates[0].tracks_cleanup is True + assert profile.mixed_profile.cleanup_query.startswith('match') + + +def test_load_stress_profile_invalid_query_type(tmp_path): + """Unknown query_type in setup_queries raises ValueError.""" + from ros_typedb_benchmark.stress_profile import load_stress_profile + data = { + 'setup_queries': [ + {'query': 'define x sub entity;', 'query_type': 'bogus'} + ] + } + p = _write_yaml(tmp_path, data) + with pytest.raises(ValueError, match='unsupported'): + load_stress_profile(p) + + +def test_load_stress_profile_missing_file(): + """Missing profile file raises FileNotFoundError.""" + from ros_typedb_benchmark.stress_profile import load_stress_profile + with pytest.raises(FileNotFoundError): + load_stress_profile(Path('/nonexistent/profile.yaml')) + + +# --------------------------------------------------------------------------- +# Task 2: ResolvedStressConfig fields + CLI args tests +# --------------------------------------------------------------------------- + +def test_resolved_stress_config_has_setup_and_teardown_fields(): + """ResolvedStressConfig accepts setup_specs and teardown_specs.""" + spec = QuerySpec(query='define x sub entity;', query_type='define') + config = ResolvedStressConfig( + query_specs=[], + invariant_specs=[], + mixed_profile=None, + mixed_profile_name=None, + invariant_profile_name='none', + max_in_flight=1, + debug_events_output=None, + fault='none', + fault_at_s=None, + fault_recovery_timeout_s=30.0, + setup_specs=[spec], + teardown_specs=[spec], + ) + assert len(config.setup_specs) == 1 + assert len(config.teardown_specs) == 1 + + +def test_resolved_stress_config_defaults_empty_setup_teardown(): + """setup_specs and teardown_specs default to empty lists.""" + config = ResolvedStressConfig( + query_specs=[], + invariant_specs=[], + mixed_profile=None, + mixed_profile_name=None, + invariant_profile_name='none', + max_in_flight=1, + debug_events_output=None, + fault='none', + fault_at_s=None, + fault_recovery_timeout_s=30.0, + ) + assert config.setup_specs == [] + assert config.teardown_specs == [] + + +def test_cli_profile_file_arg(): + """--profile-file argument is accepted by the argument parser.""" + parser = build_argument_parser() + args = parser.parse_args(['--profile-file', '/tmp/profile.yaml']) + assert args.profile_file == '/tmp/profile.yaml' + + +def test_cli_schema_path_arg(): + """--schema-path argument is accepted by the argument parser.""" + parser = build_argument_parser() + args = parser.parse_args(['--schema-path', '/tmp/schema.tql']) + assert args.schema_path == '/tmp/schema.tql' + + +def test_cli_data_path_arg(): + """--data-path argument is accepted by the argument parser.""" + parser = build_argument_parser() + args = parser.parse_args(['--data-path', '/tmp/data.tql']) + assert args.data_path == '/tmp/data.tql' + + +# --------------------------------------------------------------------------- +# Task 3: stress_resolution.py tests +# --------------------------------------------------------------------------- + +def _base_args(**overrides): + """Return a minimal valid Namespace for resolve_stress_config.""" + defaults = dict( + service_name='/ros_typedb_interface/query', + query=None, + query_type=None, + requests=1, + clients=1, + duration_s=None, + timeout_s=10.0, + wait_service_timeout_s=10.0, + request_gap_s=0.0, + max_in_flight=None, + executor='global', + executor_threads=2, + mode='read', + mixed_profile='auto', + invariant_profile='none', + invariant_period_s=10.0, + output=None, + timeout_output=None, + debug_events_output=None, + fail_on_failure=False, + fault='none', + fault_at_s=None, + fault_recovery_timeout_s=30.0, + delete_database_service_name=None, + typedb_container=None, + typedb_stop_command='pkill -f "typedb/core/server"', + typedb_start_command='typedb server', + typedb_restart_delay_s=2.0, + fault_command_timeout_s=30.0, + lifecycle_change_state_service_name=None, + lifecycle_get_state_service_name=None, + lifecycle_reactivate=True, + lifecycle_transition_timeout_s=10.0, + profile_file=None, + schema_path=None, + data_path=None, + ) + defaults.update(overrides) + return SimpleNamespace(**defaults) + + +def test_resolve_no_profile_file_unchanged(tmp_path): + """resolve_stress_config without --profile-file behaves as before.""" + args = _base_args() + config = resolve_stress_config(args) + assert config.setup_specs == [] + assert config.teardown_specs == [] + + +def test_resolve_profile_file_invariants(tmp_path): + """--profile-file with invariants populates invariant_specs.""" + data = { + 'invariants': [ + { + 'name': 'count', + 'query': 'match $x isa entity; get $x; count;', + 'query_type': 'get_aggregate', + 'expected_value': 0, + } + ] + } + p = _write_yaml(tmp_path, data) + args = _base_args(profile_file=str(p)) + config = resolve_stress_config(args) + assert len(config.invariant_specs) == 1 + assert config.invariant_specs[0].name == 'count' + + +def test_resolve_profile_file_setup_and_teardown(tmp_path): + """--profile-file setup/teardown specs propagate to config.""" + data = { + 'setup_queries': [ + {'query': 'define x sub entity;', 'query_type': 'define'} + ], + 'teardown_queries': [ + { + 'query': 'match $x isa x; delete $x isa x;', + 'query_type': 'delete', + } + ], + } + p = _write_yaml(tmp_path, data) + args = _base_args(profile_file=str(p)) + config = resolve_stress_config(args) + assert len(config.setup_specs) == 1 + assert len(config.teardown_specs) == 1 + + +def test_resolve_schema_path_becomes_setup_spec(tmp_path): + """--schema-path creates a define setup_spec.""" + schema_file = Path(tmp_path) / 'schema.tql' + schema_file.write_text('define robot sub entity;', encoding='utf-8') + args = _base_args(schema_path=str(schema_file)) + config = resolve_stress_config(args) + assert len(config.setup_specs) == 1 + assert config.setup_specs[0].query_type == 'define' + + +def test_resolve_data_path_becomes_setup_spec(tmp_path): + """--data-path creates an insert setup_spec.""" + data_file = Path(tmp_path) / 'data.tql' + data_file.write_text('insert $r isa robot;', encoding='utf-8') + args = _base_args(data_path=str(data_file)) + config = resolve_stress_config(args) + assert len(config.setup_specs) == 1 + assert config.setup_specs[0].query_type == 'insert' + + +def test_resolve_schema_and_data_path_order(tmp_path): + """schema_path spec precedes data_path spec in setup_specs.""" + schema_file = Path(tmp_path) / 'schema.tql' + schema_file.write_text('define robot sub entity;', encoding='utf-8') + data_file = Path(tmp_path) / 'data.tql' + data_file.write_text('insert $r isa robot;', encoding='utf-8') + args = _base_args( + schema_path=str(schema_file), data_path=str(data_file) + ) + config = resolve_stress_config(args) + assert len(config.setup_specs) == 2 + assert config.setup_specs[0].query_type == 'define' + assert config.setup_specs[1].query_type == 'insert' + + +def test_resolve_profile_file_conflicts_with_invariant_profile(tmp_path): + """profile-file with invariants + --invariant-profile raises ValueError.""" + data = { + 'invariants': [ + { + 'name': 'count', + 'query': 'match $x isa entity; get $x; count;', + 'query_type': 'get_aggregate', + 'expected_value': 0, + } + ] + } + p = _write_yaml(tmp_path, data) + args = _base_args(profile_file=str(p), invariant_profile='test-data') + with pytest.raises(ValueError, match='--invariant-profile'): + resolve_stress_config(args) + + +def test_resolve_profile_file_conflicts_with_mixed_profile(tmp_path): + """profile-file with mixed section + explicit --mixed-profile raises ValueError.""" + data = { + 'mixed': { + 'cleanup_query': ( + 'match $r isa robot, has name "{key}";' + ' delete $r isa robot;' + ), + 'queries': [ + { + 'query': 'insert $r isa robot, has name "{key}";', + 'query_type': 'insert', + 'tracks_cleanup': True, + }, + ], + } + } + p = _write_yaml(tmp_path, data) + args = _base_args( + profile_file=str(p), mode='mixed', mixed_profile='test-data' + ) + with pytest.raises(ValueError, match='--mixed-profile'): + resolve_stress_config(args) + + +# --------------------------------------------------------------------------- +# Task 4: _run_oneshot_query tests +# --------------------------------------------------------------------------- + +def test_run_oneshot_query_calls_service(monkeypatch): + """_run_oneshot_query sends the query to the client and records success.""" + import ros_typedb_benchmark.stress_runner as runner_mod + from ros_typedb_benchmark.stress_runner import _run_oneshot_query + + monkeypatch.setattr(runner_mod, '_spin_once', lambda node, executor: None) + + success_response = SimpleNamespace( + success=True, error_message='', results=[] + ) + future = _DoneFuture(result=success_response) + client = _CapturingClient(future) + state = _run_state() + + with DebugEventWriter(None) as debug_events: + _run_oneshot_query( + node=None, + executor=None, + client=client, + state=state, + query_spec=QuerySpec( + query='define x sub entity;', query_type='define' + ), + timeout_s=5.0, + phase='setup', + debug_events=debug_events, + ) + + assert client.request is not None + assert state.next_index == 1 + assert len(state.records) == 1 + assert state.records[0].success is True + + +def test_run_oneshot_query_timeout_records_failure(monkeypatch): + """_run_oneshot_query records a timeout when future does not complete.""" + import ros_typedb_benchmark.stress_runner as runner_mod + from ros_typedb_benchmark.stress_runner import _run_oneshot_query + + def fake_spin_until(future, node, executor, timeout_s, started_at_s): + return True, started_at_s + timeout_s + + monkeypatch.setattr( + runner_mod, '_spin_until_future_done', fake_spin_until + ) + monkeypatch.setattr(runner_mod, '_spin_once', lambda node, executor: None) + + future = _DoneFuture() + client = _CapturingClient(future) + state = _run_state() + + with DebugEventWriter(None) as debug_events: + _run_oneshot_query( + node=None, + executor=None, + client=client, + state=state, + query_spec=QuerySpec( + query='define x sub entity;', query_type='define' + ), + timeout_s=1.0, + phase='setup', + debug_events=debug_events, + ) + + assert len(state.records) == 1 + assert state.records[0].timed_out is True