WIP - Java Wrapper for Elasticsearch's ES|QL Parser

.github/workflows/esql-validation.yml

@@ -44,56 +44,28 @@ jobs:
 
           echo "run_esql=true" >> $GITHUB_ENV
 
-      - name: Check out repository
-        env:
-          DR_CLOUD_ID: ${{ secrets.dr_cloud_id }}
-          DR_API_KEY: ${{ secrets.dr_api_key }}
-        if: ${{ !env.DR_CLOUD_ID && !env.DR_API_KEY && env.run_esql == 'true' }}
-        uses: actions/checkout@93cb6efe18208431cddfb8368fd83d5badbf9bfd # v5
-        with:
-          path: elastic-container
-          repository: peasead/elastic-container
-
-      - name: Build and run containers
-        env:
-          DR_CLOUD_ID: ${{ secrets.dr_cloud_id }}
-          DR_API_KEY: ${{ secrets.dr_api_key }}
-        if: ${{ !env.DR_CLOUD_ID && !env.DR_API_KEY && env.run_esql == 'true' }}
-        run: |
-          cd elastic-container
-          GENERATED_PASSWORD=$(openssl rand -base64 16)
-          sed -i "s|changeme|$GENERATED_PASSWORD|" .env
-          echo "::add-mask::$GENERATED_PASSWORD"
-          echo "GENERATED_PASSWORD=$GENERATED_PASSWORD" >> $GITHUB_ENV
-          set -x
-          bash elastic-container.sh update-version
-          bash elastic-container.sh start
-
-      - name: Get API Key and setup auth
-        env:
-          DR_CLOUD_ID: ${{ secrets.dr_cloud_id }}
-          DR_API_KEY: ${{ secrets.dr_api_key }}
-          DR_ELASTICSEARCH_URL: "https://localhost:9200"
-          ES_USER: "elastic"
-          ES_PASSWORD: ${{ env.GENERATED_PASSWORD }}
-        if: ${{ !env.DR_CLOUD_ID && !env.DR_API_KEY && env.run_esql == 'true' }}
-        run: |
-          cd detection-rules
-          response=$(curl -k -X POST -u "$ES_USER:$ES_PASSWORD" -H "Content-Type: application/json" -d '{
-              "name": "tmp-api-key",
-              "expiration": "1d"
-          }' "$DR_ELASTICSEARCH_URL/_security/api_key")
-
-          DR_API_KEY=$(echo "$response" | jq -r '.encoded')
-          echo "::add-mask::$DR_API_KEY"
-          echo "DR_API_KEY=$DR_API_KEY" >> $GITHUB_ENV
-
       - name: Set up Python 3.13
         if: ${{ env.run_esql == 'true' }}
         uses: actions/setup-python@e797f83bcb11b83ae66e0230d6156d7c80228e7c # v6
         with:
           python-version: '3.13'
 
+      - name: Set up JDK 21
+        if: ${{ env.run_esql == 'true' }}
+        uses: actions/setup-java@v4
+        with:
+          distribution: 'temurin'
+          java-version: '21'
+
+      - name: Check out elastic/elasticsearch for ES|QL validator build
+        if: ${{ env.run_esql == 'true' }}
+        uses: actions/checkout@93cb6efe18208431cddfb8368fd83d5badbf9bfd # v5
+        with:
+          repository: elastic/elasticsearch
+          path: elasticsearch
+          # Match the stack version the rules currently target. Update alongside
+          # detection_rules.schemas.get_latest_stack_version().
+
       - name: Install dependencies
         if: ${{ env.run_esql == 'true' }}
         run: |
@@ -102,14 +74,12 @@ jobs:
           pip cache purge
           pip install .[dev]
 
-      - name: Remote Test ESQL Rules
+      - name: Test ES|QL Rules
         if: ${{ env.run_esql == 'true' }}
         env:
-          DR_CLOUD_ID: ${{ secrets.dr_cloud_id || '' }}
-          DR_KIBANA_URL: ${{ secrets.dr_cloud_id == '' && 'https://localhost:5601' || '' }}
-          DR_ELASTICSEARCH_URL: ${{ secrets.dr_cloud_id == '' && 'https://localhost:9200' || '' }}
-          DR_API_KEY: ${{ secrets.dr_api_key || env.DR_API_KEY }}
-          DR_IGNORE_SSL_ERRORS: ${{ secrets.dr_cloud_id == '' && 'true' || '' }}
+          # Point the validator's build script at the elasticsearch checkout above.
+          # The first compile is slow; gradle build cache speeds up later runs.
+          ES_HOME: ${{ github.workspace }}/elasticsearch
         run: |
           cd detection-rules
-          python -m detection_rules dev test esql-remote-validation
+          python -m detection_rules dev test esql-validation

.github/workflows/lock-versions.yml

@@ -87,7 +87,7 @@ jobs:
 
       - name: Build release package with navigator files
         env:
-          DR_REMOTE_ESQL_VALIDATION: "true"
+          DR_ESQL_VALIDATION: "true"
           DR_CLOUD_ID: ${{ secrets.dr_cloud_id || '' }}
           DR_KIBANA_URL: ${{ secrets.dr_cloud_id == '' && 'https://localhost:5601' || '' }}
           DR_ELASTICSEARCH_URL: ${{ secrets.dr_cloud_id == '' && 'https://localhost:9200' || '' }}
@@ -111,7 +111,7 @@ jobs:
       - name: Lock the versions
         env:
           BRANCHES: "${{github.event.inputs.branches}}"
-          DR_REMOTE_ESQL_VALIDATION: "true"
+          DR_ESQL_VALIDATION: "true"
           DR_CLOUD_ID: ${{ secrets.dr_cloud_id || '' }}
           DR_KIBANA_URL: ${{ secrets.dr_cloud_id == '' && 'https://localhost:5601' || '' }}
           DR_ELASTICSEARCH_URL: ${{ secrets.dr_cloud_id == '' && 'https://localhost:9200' || '' }}

.github/workflows/pythonpackage.yml

@@ -40,8 +40,8 @@ jobs:
       env:
         # only run the test test_rule_change_has_updated_date on pull request events to main
         GITHUB_EVENT_NAME: "${{ github.event_name}}"
-        # only run remote validation if repo is set to do so otherwise defer to .github/workflows/esql-validation.yml
-        DR_REMOTE_ESQL_VALIDATION: "${{ vars.remote_esql_validation }}"
+        # only run ES|QL validation if the repo opts in; otherwise defer to .github/workflows/esql-validation.yml
+        DR_ESQL_VALIDATION: "${{ vars.esql_validation }}"
         DR_CLOUD_ID: ${{ secrets.dr_cloud_id }}
         DR_KIBANA_URL: ${{ secrets.dr_cloud_id }}
         DR_ELASTICSEARCH_URL: ${{ secrets.dr_cloud_id }}

.github/workflows/release-fleet.yml

@@ -161,7 +161,7 @@ jobs:
 
         - name: Build release package
           env:
-            DR_REMOTE_ESQL_VALIDATION: "true"
+            DR_ESQL_VALIDATION: "true"
             DR_CLOUD_ID: ${{ secrets.dr_cloud_id || '' }}
             DR_KIBANA_URL: ${{ secrets.dr_cloud_id == '' && 'https://localhost:5601' || '' }}
             DR_ELASTICSEARCH_URL: ${{ secrets.dr_cloud_id == '' && 'https://localhost:9200' || '' }}

CLI.md

@@ -55,9 +55,9 @@ In `_config.yaml`, `bypass_optional_elastic_validation: true` enables all of the
 Using the environment variable `DR_CLI_MAX_WIDTH` will set a custom max width for the click CLI. 
 For instance, some users may want to increase the default value in cases where help messages are cut off. 
 
-Using the environment variable `DR_REMOTE_ESQL_VALIDATION` will enable remote ESQL validation for rules that use ESQL queries. This validation will be performed whenever the rule is loaded including for example the view-rule command. This requires the appropriate kibana_url or cloud_id, api_key, and es_url to be set in the config file or as environment variables.
+Using the environment variable `DR_ESQL_VALIDATION` will enable ES|QL validation for rules that use ES|QL queries. This validation runs locally via the embedded Java validator (`lib/esql-validator`) and is performed whenever the rule is loaded — including, for example, the `view-rule` command. No Elasticsearch or Kibana credentials are required.
 
-Using the environment variable `DR_SKIP_EMPTY_INDEX_CLEANUP` will disable the cleanup of remote testing indexes that are created as part of the remote ESQL validation. By default, these indexes are deleted after the validation is complete, or upon validation error.
+Using the environment variable `DR_SKIP_EMPTY_INDEX_CLEANUP` will disable the cleanup of any stale `rule-test-*` / `test-*` indexes left over from older remote-validation runs. Current validation does not create any such indexes — the variable only affects opportunistic cleanup triggered when validation errors fire with a live Elasticsearch client configured.
 
 ## Importing rules into the repo
 

detection_rules/devtools.py

@@ -26,7 +26,6 @@
 import requests.exceptions
 import yaml
 from elasticsearch import BadRequestError, Elasticsearch
-from elasticsearch import ConnectionError as ESConnectionError
 from eql.table import Table  # type: ignore[reportMissingTypeStubs]
 from eql.utils import load_dump  # type: ignore[reportMissingTypeStubs, reportUnknownVariableType]
 from kibana.connector import Kibana  # type: ignore[reportMissingTypeStubs]
@@ -43,6 +42,7 @@
 from .esql_errors import (
     ESQL_EXCEPTION_TYPES,
 )
+from .esql_parser import shared_validator
 from .eswrap import CollectEvents, add_range_to_dsl
 from .ghwrap import GithubClient, update_gist
 from .integrations import (
@@ -57,8 +57,6 @@
 from .misc import (
     PYTHON_LICENSE,
     add_client,
-    get_default_elasticsearch_client,
-    get_default_kibana_client,
     raise_client_error,
 )
 from .packaging import CURRENT_RELEASE_PATH, PACKAGE_FILE, RELEASE_DIR, Package
@@ -1414,70 +1412,56 @@ def rule_event_search(  # noqa: PLR0913
         raise_client_error("Rule is not a query rule!")
 
 
-@test_group.command("esql-remote-validation")
+@test_group.command("esql-validation")
 @click.option(
     "--verbosity",
     type=click.IntRange(0, 1),
     default=0,
     help="Set verbosity level: 0 for minimal output, 1 for detailed output.",
 )
-def esql_remote_validation(
+def esql_validation(
     verbosity: int,
 ) -> None:
-    """Search using a rule file against an Elasticsearch instance."""
-
+    """Validate all production ES|QL rules locally via the embedded Java validator."""
+    # Validation is fully local: no Kibana or Elasticsearch client is required.
+    # shared_validator() spawns one JVM daemon up front and reuses it across every
+    # rule, instead of paying ~2-3s startup per rule.
     rule_collection: RuleCollection = RuleCollection.default().filter(production_filter)
     esql_rules = [r for r in rule_collection if r.contents.data.type == "esql"]
 
     click.echo(f"ESQL rules loaded: {len(esql_rules)}")
 
     if not esql_rules:
         return
-    # TODO(eric-forte-elastic): @add_client https://github.com/elastic/detection-rules/issues/5156  # noqa: FIX002
-    with get_default_kibana_client() as kibana_client, get_default_elasticsearch_client() as elastic_client:
-        if not kibana_client or not elastic_client:
-            raise_client_error("Skipping remote validation due to missing client")
-
-        failed_count = 0
-        fail_list: list[str] = []
-        max_retries = 3
+
+    failed_count = 0
+    fail_list: list[str] = []
+    with shared_validator():
         for r in esql_rules:
-            retry_count = 0
-            while retry_count < max_retries:
-                try:
-                    validator = ESQLValidator(r.contents.data.query)  # type: ignore[reportIncompatibleMethodOverride]
-                    _ = validator.remote_validate_rule_contents(kibana_client, elastic_client, r.contents, verbosity)
-                    break
-                except (ValueError, BadRequestError, *ESQL_EXCEPTION_TYPES) as e:  # type: ignore[reportUnknownMemberType]
-                    e_type = type(e)  # type: ignore[reportUnknownMemberType]
-                    if isinstance(e, ESQL_EXCEPTION_TYPES):
-                        click.echo(click.style(f"{r.contents.data.rule_id} ", fg="red", bold=True), nl=False)
-                        _ = e.show()  # type: ignore[reportUnknownMemberType]
-                    else:
-                        click.echo(f"FAILURE: {e_type}: {e}")  # type: ignore[reportUnknownMemberType]
-                    fail_list.append(f"{r.contents.data.rule_id}  FAILURE: {e_type}: {e}")  # type: ignore[reportUnknownMemberType]
-                    failed_count += 1
-                    break
-                except ESConnectionError as e:
-                    retry_count += 1
-                    click.echo(f"Connection error: {e}. Retrying {retry_count}/{max_retries}...")
-                    time.sleep(30)
-                    if retry_count == max_retries:
-                        click.echo(f"FAILURE: {e} after {max_retries} retries")
-                        fail_list.append(f"FAILURE: {e} after {max_retries} retries")
-                        failed_count += 1
-
-        click.echo(f"Total rules: {len(esql_rules)}")
-        click.echo(f"Failed rules: {failed_count}")
-
-        _ = Path("failed_rules.log").write_text("\n".join(fail_list), encoding="utf-8")
-        click.echo("Failed rules written to failed_rules.log")
-        if failed_count > 0:
-            click.echo("Failed rule IDs:")
-            uuids = {line.split()[0] for line in fail_list}
-            click.echo("\n".join(uuids))
-            ctx = click.get_current_context()
-            ctx.exit(1)
+            try:
+                validator = ESQLValidator(r.contents.data.query)  # type: ignore[reportIncompatibleMethodOverride]
+                _ = validator.remote_validate_rule_contents(None, None, r.contents, verbosity)
+            except (ValueError, BadRequestError, *ESQL_EXCEPTION_TYPES) as e:  # type: ignore[reportUnknownMemberType]
+                e_type = type(e)  # type: ignore[reportUnknownMemberType]
+                if isinstance(e, ESQL_EXCEPTION_TYPES):
+                    click.echo(click.style(f"{r.contents.data.rule_id} ", fg="red", bold=True), nl=False)
+                    _ = e.show()  # type: ignore[reportUnknownMemberType]
+                else:
+                    click.echo(f"FAILURE: {e_type}: {e}")  # type: ignore[reportUnknownMemberType]
+                fail_list.append(f"{r.contents.data.rule_id}  FAILURE: {e_type}: {e}")  # type: ignore[reportUnknownMemberType]
+                failed_count += 1
+
+    click.echo(f"Total rules: {len(esql_rules)}")
+    click.echo(f"Failed rules: {failed_count}")
+
+    _ = Path("failed_rules.log").write_text("\n".join(fail_list), encoding="utf-8")
+    click.echo("Failed rules written to failed_rules.log")
+    if failed_count > 0:
+        click.echo("Failed rule IDs:")
+        uuids = {line.split()[0] for line in fail_list}
+        click.echo("\n".join(uuids))
+        ctx = click.get_current_context()
+        ctx.exit(1)
 
 
 @test_group.command("rule-survey")

detection_rules/esql_errors.py

@@ -38,12 +38,16 @@ def cleanup_empty_indices(
 class EsqlKibanaBaseError(ClientError):
     """Base class for ESQL exceptions with cleanup logic."""
 
+    # elastic_client is optional: it's only used to clean up stale rule-test-* /
+    # test-* indices from previous remote runs. Local-only callers pass None and
+    # the cleanup is skipped.
     def __init__(
         self,
         message: str,
-        elastic_client: Elasticsearch,
+        elastic_client: Elasticsearch | None = None,
     ) -> None:
-        cleanup_empty_indices(elastic_client)
+        if elastic_client is not None:
+            cleanup_empty_indices(elastic_client)
         super().__init__(message, original_error=self)
 
 

detection_rules/esql_parser/__init__.py

@@ -0,0 +1,26 @@
+# Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+# or more contributor license agreements. Licensed under the Elastic License
+# 2.0; you may not use this file except in compliance with the Elastic License
+# 2.0.
+
+"""Python wrapper around the Elasticsearch Java ES|QL parser & verifier.
+
+Spawns the JVM-based daemon in ``lib/esql-validator`` and exchanges
+line-delimited JSON over stdin/stdout to validate arbitrary ES|QL queries.
+"""
+
+from .validator import (
+    EsqlValidator,
+    ValidationError,
+    ValidationResult,
+    get_shared_validator,
+    shared_validator,
+)
+
+__all__ = (
+    "EsqlValidator",
+    "ValidationError",
+    "ValidationResult",
+    "get_shared_validator",
+    "shared_validator",
+)