Add option to exclude open gaps from gap-events output

pyproject.toml

@@ -12,7 +12,7 @@ namespaces = false
 
 [project]
 name = "pipe-gaps"
-version = "0.14.3"
+version = "0.15.0"
 description = "Tools for detecting interruptions in vessel position reporting systems (e.g., AIS, VMS)."
 readme = "README.md"
 license = "Apache-2.0"

src/pipe_gaps/assets/queries/events.sql.j2

@@ -68,6 +68,10 @@ WITH
         {% if end_date %}
           AND (end_timestamp IS NULL OR DATE(end_timestamp) < '{{ end_date }}')
         {% endif %}
+
+        {% if exclude_open_gaps %}
+          AND end_timestamp IS NOT NULL
+        {% endif %}
     ),
 
     ############

src/pipe_gaps/cli/commands/gap_events.py

@@ -36,6 +36,7 @@
 HELP_DATE_RANGE = "Create gap events for this date range, e.g., «2024-01-01,2024-01-02»."
 HELP_BQ_PROJECT = "Project to use when executing the events query."
 HELP_DRY_RUN = "If True, executes queries in dry run mode."
+HELP_EXCLUDE_OPEN_GAPS = "If passed, excludes open gaps (no end_timestamp yet) from the output."
 
 
 class GapEvents(Command):
@@ -67,6 +68,7 @@ def options(self):
                    help=HELP_BQ_IN_VESSELS_BYYEAR_FLAG_FIELD, default=None),
             Option("--bq-out-gap-events", type=str, help=HELP_BQ_OUT_GAP_EVENTS),
             Option("--mock-bq-clients", type=bool, help=HELP_MOCK_BQ_CLIENTS),
+            Option("--exclude-open-gaps", type=bool, help=HELP_EXCLUDE_OPEN_GAPS),
             Option("--labels", type=str, nargs="*", action=NestedKeyValueAction, help=HELP_LABELS),
         ]
 

src/pipe_gaps/pipelines/events/config.py

@@ -23,6 +23,7 @@ class GapEventsConfig(PipelineConfig):
     bq_in_vessels_byyear_flag_field: str | None = None
     labels: dict = field(default_factory=dict)
     dry_run: bool = False
+    exclude_open_gaps: bool = False
 
     @property
     def vessels_byyear_flag_field(self) -> str:

src/pipe_gaps/pipelines/events/main.py

@@ -60,6 +60,7 @@ def template_vars(self) -> dict:
             "vessel_info_flag_field": self.config.vessels_byyear_flag_field,
             "start_date": self.config.start_date,
             "end_date": self.config.end_date,
+            "exclude_open_gaps": self.config.exclude_open_gaps,
             "regions": [region["name"] for region in self.regions],
         }
 

tests/pipelines/events/test_main.py

@@ -88,6 +88,29 @@ def test_end_date_filter_lets_open_gaps_through(basic_config_kwargs):
     assert "end_timestamp IS NULL OR DATE(end_timestamp) <" in sql
 
 
+def test_exclude_open_gaps_defaults_to_false(basic_config_kwargs):
+    config = GapEventsConfig.from_namespace(
+        SimpleNamespace(**basic_config_kwargs, unknown_parsed_args={}),
+        version="test",
+        name="test",
+    )
+    sql = GapEventQuery(config).render()
+    assert "end_timestamp IS NOT NULL" not in sql
+
+
+def test_exclude_open_gaps_drops_them_from_the_output(basic_config_kwargs):
+    """VMS wants open gaps excluded until the open-gap peaks at the end of the last
+    processed day (expected, not a bug) are addressed.
+    """
+    config = GapEventsConfig.from_namespace(
+        SimpleNamespace(**basic_config_kwargs, exclude_open_gaps=True, unknown_parsed_args={}),
+        version="test",
+        name="test",
+    )
+    sql = GapEventQuery(config).render()
+    assert "AND end_timestamp IS NOT NULL" in sql
+
+
 def test_fetch_regions_registry_queries_the_given_table():
     bq_helper = BigQueryHelper.mocked()