feat(metrics): durable JDBC metrics extension (Scope 2)

CHANGELOG.md

@@ -109,13 +109,14 @@ request adding CHANGELOG notes for breaking (!) changes and possibly other secti
 
 - The (Before/After)CommitViewEvent has been removed.
 - The (Before/After)CommitTableEvent has been removed.
-- The `PolarisMetricsReporter.reportMetric()` method signature has been extended to include a `receivedTimestamp` parameter of type `java.time.Instant`.
+- The `PolarisMetricsReporter` SPI (previously in `runtime/service`) has been replaced by `IcebergMetricsReporter` in `polaris-core`. Custom reporters must implement `org.apache.polaris.core.metrics.IcebergMetricsReporter` and update the `@Identifier` annotation for CDI selection via `polaris.iceberg-metrics.reporting.type`. The method signature now includes a `receivedTimestamp` parameter of type `java.time.Instant`.
 - The `ExternalCatalogFactory.createCatalog()` and `createGenericCatalog()` method signatures have been extended to include a `catalogProperties` parameter of type `Map<String, String>` for passing through proxy and timeout settings to federated catalog HTTP clients.
 - Metrics reporting now requires the `TABLE_READ_DATA` privilege on the target table for read (scan) metrics and `TABLE_WRITE_DATA` for write (commit) metrics.
 - The `REVOKE_CATALOG_ROLE_FROM_PRINCIPAL_ROLE` operation no longer requires the `PRINCIPAL_ROLE_MANAGE_GRANTS_FOR_GRANTEE` privilege. Only `CATALOG_ROLE_MANAGE_GRANTS_ON_SECURABLE` on the catalog role is now required, making revoke symmetric with assign. This allows catalog administrators to fully manage catalog role assignments without requiring elevated privileges on principal roles.
 
 ### New Features
 
+- Added a **beta** Table Metrics REST API (`/api/metrics-reports/v1/`) for querying Iceberg scan and commit metrics. In this release the read path returns an empty result set; durable storage backing is provided by the `polaris-extensions-metrics-reports-jdbc` extension in a follow-up release. Querying requires the new `TABLE_READ_METRICS` privilege on the target table.
 - Added `deploymentAnnotations` support in Helm chart.
 - Added KMS properties (optional) to catalog storage config to enable S3 data encryption.
 - Added `topologySpreadConstraints` support in Helm chart.

api/metrics-reports-service/build.gradle.kts

@@ -0,0 +1,93 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+import org.openapitools.generator.gradle.plugin.tasks.GenerateTask
+
+plugins {
+  alias(libs.plugins.openapi.generator)
+  id("polaris-client")
+  id("org.kordamp.gradle.jandex")
+}
+
+dependencies {
+  implementation(project(":polaris-core"))
+
+  compileOnly(platform(libs.jackson.bom))
+  compileOnly("com.fasterxml.jackson.core:jackson-annotations")
+
+  compileOnly(libs.jakarta.annotation.api)
+  compileOnly(libs.jakarta.inject.api)
+  compileOnly(libs.jakarta.validation.api)
+  compileOnly(libs.microprofile.fault.tolerance.api)
+  compileOnly(libs.swagger.annotations)
+
+  implementation(libs.jakarta.servlet.api)
+  implementation(libs.jakarta.ws.rs.api)
+
+  compileOnly(platform(libs.micrometer.bom))
+  compileOnly("io.micrometer:micrometer-core")
+
+  implementation(libs.slf4j.api)
+}
+
+val rootDir = rootProject.layout.projectDirectory
+val specsDir = rootDir.dir("spec")
+val templatesDir = rootDir.dir("server-templates")
+val generatedDir = project.layout.buildDirectory.dir("generated-openapi")
+val generatedOpenApiSrcDir = project.layout.buildDirectory.dir("generated-openapi/src/main/java")
+
+openApiGenerate {
+  inputSpec = provider { specsDir.file("metrics-reports-service.yml").asFile.absolutePath }
+  generatorName = "jaxrs-resteasy"
+  outputDir = provider { generatedDir.get().asFile.absolutePath }
+  apiPackage = "org.apache.polaris.service.metrics.api"
+  modelPackage = "org.apache.polaris.core.metrics.api.model"
+  ignoreFileOverride.set(provider { rootDir.file(".openapi-generator-ignore").asFile.absolutePath })
+  removeOperationIdPrefix.set(true)
+  templateDir.set(provider { templatesDir.asFile.absolutePath })
+  globalProperties.put("apis", "")
+  globalProperties.put("models", "")
+  globalProperties.put("apiDocs", "false")
+  globalProperties.put("modelTests", "false")
+  configOptions.put("openApiNullable", "false")
+  configOptions.put("useBeanValidation", "true")
+  configOptions.put("sourceFolder", "src/main/java")
+  configOptions.put("useJakartaEe", "true")
+  configOptions.put("generateBuilders", "true")
+  configOptions.put("generateConstructorWithAllArgs", "true")
+  configOptions.put("hideGenerationTimestamp", "true")
+  additionalProperties.put("apiNamePrefix", "Polaris")
+  additionalProperties.put("apiNameSuffix", "Api")
+  additionalProperties.put("metricsPrefix", "polaris")
+  serverVariables.put("basePath", "api/metrics-reports/v1")
+}
+
+listOf("sourcesJar", "compileJava", "processResources").forEach { task ->
+  tasks.named(task) { dependsOn("openApiGenerate") }
+}
+
+sourceSets { main { java { srcDir(generatedOpenApiSrcDir) } } }
+
+tasks.named<GenerateTask>("openApiGenerate") {
+  inputs.dir(templatesDir)
+  inputs.dir(specsDir)
+  actions.addFirst { delete { delete(generatedDir) } }
+}
+
+tasks.named("javadoc") { dependsOn("jandex") }

bom/build.gradle.kts

@@ -30,6 +30,7 @@ dependencies {
     api(project(":polaris-api-iceberg-service"))
     api(project(":polaris-api-management-model"))
     api(project(":polaris-api-management-service"))
+    api(project(":polaris-api-metrics-reports-service"))
 
     api(project(":polaris-container-spec-helper"))
     api(project(":polaris-azurite-testcontainer"))
@@ -101,6 +102,8 @@ dependencies {
     api(project(":polaris-extensions-federation-bigquery"))
     api(project(":polaris-extensions-federation-hadoop"))
     api(project(":polaris-extensions-federation-hive"))
+    api(project(":polaris-extensions-metrics-reports-jdbc"))
+    api(project(":polaris-extensions-metrics-reports-spi"))
     api(project(":polaris-hms-testcontainer"))
 
     api(project(":polaris-spark-3.5_2.12"))
@@ -112,6 +115,7 @@ dependencies {
       api(project(":polaris-spark-4.0_2.13"))
       api(project(":polaris-spark-integration-4.0_2.13"))
     }
+    api(project(":polaris-extensions-metrics-reports"))
 
     api(project(":polaris-admin"))
     api(project(":polaris-runtime-common"))

extensions/metrics-reports/impl/build.gradle.kts

@@ -0,0 +1,51 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+plugins {
+  id("polaris-server")
+  id("org.kordamp.gradle.jandex")
+}
+
+dependencies {
+  implementation(project(":polaris-core"))
+  implementation(project(":polaris-api-metrics-reports-service"))
+  implementation(project(":polaris-extensions-metrics-reports-spi"))
+
+  implementation(platform(libs.iceberg.bom))
+  implementation("org.apache.iceberg:iceberg-api")
+
+  implementation(libs.jakarta.enterprise.cdi.api)
+  implementation(libs.jakarta.inject.api)
+  implementation(libs.jakarta.ws.rs.api)
+  implementation(libs.smallrye.common.annotation)
+  implementation(libs.slf4j.api)
+  compileOnly(libs.jakarta.annotation.api)
+
+  implementation(platform(libs.jackson.bom))
+  implementation("com.fasterxml.jackson.core:jackson-annotations")
+
+  testImplementation(platform(libs.junit.bom))
+  testImplementation("org.junit.jupiter:junit-jupiter")
+  testImplementation(libs.assertj.core)
+  testImplementation(libs.mockito.core)
+
+  // Provides jakarta.ws.rs.ext.RuntimeDelegate needed to build Response objects in unit tests
+  testRuntimeOnly(enforcedPlatform(libs.quarkus.bom))
+  testRuntimeOnly("io.quarkus.resteasy.reactive:resteasy-reactive")
+}

extensions/metrics-reports/impl/src/main/java/org/apache/polaris/extension/metrics/reports/CommitMetricsReport.java

@@ -0,0 +1,66 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.polaris.extension.metrics.reports;
+
+import com.fasterxml.jackson.annotation.JsonInclude;
+import com.fasterxml.jackson.annotation.JsonProperty;
+
+/** A single commit metrics report entry returned by the list API. */
+public record CommitMetricsReport(
+    @JsonProperty("id") String id,
+    @JsonProperty("timestampMs") long timestampMs,
+    @JsonProperty("actor") MetricsReportActor actor,
+    @JsonProperty("request") MetricsReportRequest request,
+    @JsonProperty("object") TableObject object,
+    @JsonProperty("payload") Payload payload) {
+
+  /** The Iceberg object (table snapshot) associated with the commit. */
+  public record TableObject(@JsonProperty("snapshotId") long snapshotId) {}
+
+  /** The commit metrics payload envelope. */
+  public record Payload(
+      @JsonProperty("type") String type,
+      @JsonProperty("version") int version,
+      @JsonProperty("data") Data data) {
+
+    /** Iceberg commit metrics data fields. */
+    @JsonInclude(JsonInclude.Include.NON_NULL)
+    public record Data(
+        @JsonProperty("sequenceNumber") Long sequenceNumber,
+        @JsonProperty("operation") String operation,
+        @JsonProperty("addedDataFiles") long addedDataFiles,
+        @JsonProperty("removedDataFiles") long removedDataFiles,
+        @JsonProperty("totalDataFiles") long totalDataFiles,
+        @JsonProperty("addedDeleteFiles") long addedDeleteFiles,
+        @JsonProperty("removedDeleteFiles") long removedDeleteFiles,
+        @JsonProperty("totalDeleteFiles") long totalDeleteFiles,
+        @JsonProperty("addedEqualityDeleteFiles") long addedEqualityDeleteFiles,
+        @JsonProperty("removedEqualityDeleteFiles") long removedEqualityDeleteFiles,
+        @JsonProperty("addedPositionalDeleteFiles") long addedPositionalDeleteFiles,
+        @JsonProperty("removedPositionalDeleteFiles") long removedPositionalDeleteFiles,
+        @JsonProperty("addedRecords") long addedRecords,
+        @JsonProperty("removedRecords") long removedRecords,
+        @JsonProperty("totalRecords") long totalRecords,
+        @JsonProperty("addedFileSizeBytes") long addedFileSizeBytes,
+        @JsonProperty("removedFileSizeBytes") long removedFileSizeBytes,
+        @JsonProperty("totalFileSizeBytes") long totalFileSizeBytes,
+        @JsonProperty("totalDurationMs") Long totalDurationMs,
+        @JsonProperty("attempts") int attempts) {}
+  }
+}

runtime/service/src/main/java/org/apache/polaris/service/reporting/DefaultMetricsReporter.java → extensions/metrics-reports/impl/src/main/java/org/apache/polaris/extension/metrics/reports/LoggingMetricsReporter.java

@@ -16,37 +16,30 @@
  * specific language governing permissions and limitations
  * under the License.
  */
-package org.apache.polaris.service.reporting;
+package org.apache.polaris.extension.metrics.reports;
 
-import com.google.common.annotations.VisibleForTesting;
 import io.smallrye.common.annotation.Identifier;
 import jakarta.enterprise.context.ApplicationScoped;
 import java.time.Instant;
 import org.apache.iceberg.catalog.TableIdentifier;
 import org.apache.iceberg.metrics.MetricsReport;
+import org.apache.polaris.core.metrics.IcebergMetricsReporter;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
 /**
- * Default implementation of {@link PolarisMetricsReporter} that logs metrics to the configured
- * logger.
+ * Log-only implementation of {@link IcebergMetricsReporter}.
  *
- * <p>This implementation is selected when {@code polaris.iceberg-metrics.reporting.type} is set to
- * {@code "default"} (the default value).
+ * <p>Selected when {@code polaris.iceberg-metrics.reporting.type} is set to {@code "log"}.
  *
- * <p>By default, logging is disabled. To enable metrics logging, set the logger level for {@code
- * org.apache.polaris.service.reporting} to {@code INFO} in your logging configuration.
- *
- * @see PolarisMetricsReporter
+ * <p>Logging is at INFO level. Enable it by setting the logger level for {@code
+ * org.apache.polaris.extension.metrics.reports} to {@code INFO}.
  */
 @ApplicationScoped
 @Identifier("default")
-public class DefaultMetricsReporter implements PolarisMetricsReporter {
-  private static final Logger LOGGER = LoggerFactory.getLogger(DefaultMetricsReporter.class);
-
-  private final ReportConsumer reportConsumer;
+public class LoggingMetricsReporter implements IcebergMetricsReporter {
+  private static final Logger LOGGER = LoggerFactory.getLogger(LoggingMetricsReporter.class);
 
-  /** Functional interface for consuming metrics reports with timestamp. */
   @FunctionalInterface
   interface ReportConsumer {
     void accept(
@@ -58,15 +51,15 @@ void accept(
         Instant receivedTimestamp);
   }
 
-  /** Creates a new DefaultMetricsReporter that logs metrics to the class logger. */
-  public DefaultMetricsReporter() {
+  private final ReportConsumer reportConsumer;
+
+  public LoggingMetricsReporter() {
     this(
         (catalogName, catalogId, table, tableId, metricsReport, receivedTimestamp) ->
             LOGGER.info("{}.{} (ts={}): {}", catalogName, table, receivedTimestamp, metricsReport));
   }
 
-  @VisibleForTesting
-  DefaultMetricsReporter(ReportConsumer reportConsumer) {
+  LoggingMetricsReporter(ReportConsumer reportConsumer) {
     this.reportConsumer = reportConsumer;
   }
 

extensions/metrics-reports/impl/src/main/java/org/apache/polaris/extension/metrics/reports/MetricsListResponse.java

@@ -0,0 +1,29 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.polaris.extension.metrics.reports;
+
+import com.fasterxml.jackson.annotation.JsonInclude;
+import com.fasterxml.jackson.annotation.JsonProperty;
+import java.util.List;
+
+/** Paginated list response for metrics reports. */
+public record MetricsListResponse<T>(
+    @JsonProperty("metricType") String metricType,
+    @JsonInclude(JsonInclude.Include.NON_NULL) @JsonProperty("nextPageToken") String nextPageToken,
+    @JsonProperty("reports") List<T> reports) {}

extensions/metrics-reports/impl/src/main/java/org/apache/polaris/extension/metrics/reports/MetricsReportActor.java

@@ -0,0 +1,24 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.polaris.extension.metrics.reports;
+
+import com.fasterxml.jackson.annotation.JsonProperty;
+
+/** The principal that triggered the metrics report. */
+public record MetricsReportActor(@JsonProperty("principalName") String principalName) {}