diff --git a/CHANGELOG.md b/CHANGELOG.md index 9d2b1046d55..23f48a1cd85 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -109,13 +109,14 @@ request adding CHANGELOG notes for breaking (!) changes and possibly other secti - The (Before/After)CommitViewEvent has been removed. - The (Before/After)CommitTableEvent has been removed. -- The `PolarisMetricsReporter.reportMetric()` method signature has been extended to include a `receivedTimestamp` parameter of type `java.time.Instant`. +- The `PolarisMetricsReporter` SPI (previously in `runtime/service`) has been replaced by `IcebergMetricsReporter` in `extensions/metrics-reports/spi`. Custom reporters must implement `org.apache.polaris.extension.metrics.IcebergMetricsReporter` and update the `@Identifier` annotation for CDI selection via `polaris.iceberg-metrics.reporting.type`. The method signature now includes a `receivedTimestamp` parameter of type `java.time.Instant`. - The `ExternalCatalogFactory.createCatalog()` and `createGenericCatalog()` method signatures have been extended to include a `catalogProperties` parameter of type `Map` for passing through proxy and timeout settings to federated catalog HTTP clients. - Metrics reporting now requires the `TABLE_READ_DATA` privilege on the target table for read (scan) metrics and `TABLE_WRITE_DATA` for write (commit) metrics. - The `REVOKE_CATALOG_ROLE_FROM_PRINCIPAL_ROLE` operation no longer requires the `PRINCIPAL_ROLE_MANAGE_GRANTS_FOR_GRANTEE` privilege. Only `CATALOG_ROLE_MANAGE_GRANTS_ON_SECURABLE` on the catalog role is now required, making revoke symmetric with assign. This allows catalog administrators to fully manage catalog role assignments without requiring elevated privileges on principal roles. ### New Features +- Added a **beta** Table Metrics REST API (`/api/metrics-reports/v1/`) for querying Iceberg scan and commit metrics. In this release the read path returns an empty result set; durable storage backing is provided by the `polaris-extensions-metrics-reports-jdbc` extension in a follow-up release. Querying requires the new `TABLE_READ_METRICS` privilege on the target table. - Added `deploymentAnnotations` support in Helm chart. - Added KMS properties (optional) to catalog storage config to enable S3 data encryption. - Added `topologySpreadConstraints` support in Helm chart. diff --git a/api/metrics-reports-service/build.gradle.kts b/api/metrics-reports-service/build.gradle.kts new file mode 100644 index 00000000000..ce80aceb238 --- /dev/null +++ b/api/metrics-reports-service/build.gradle.kts @@ -0,0 +1,93 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +import org.openapitools.generator.gradle.plugin.tasks.GenerateTask + +plugins { + alias(libs.plugins.openapi.generator) + id("polaris-client") + id("org.kordamp.gradle.jandex") +} + +dependencies { + implementation(project(":polaris-core")) + + compileOnly(platform(libs.jackson.bom)) + compileOnly("com.fasterxml.jackson.core:jackson-annotations") + + compileOnly(libs.jakarta.annotation.api) + compileOnly(libs.jakarta.inject.api) + compileOnly(libs.jakarta.validation.api) + compileOnly(libs.microprofile.fault.tolerance.api) + compileOnly(libs.swagger.annotations) + + implementation(libs.jakarta.servlet.api) + implementation(libs.jakarta.ws.rs.api) + + compileOnly(platform(libs.micrometer.bom)) + compileOnly("io.micrometer:micrometer-core") + + implementation(libs.slf4j.api) +} + +val rootDir = rootProject.layout.projectDirectory +val specsDir = rootDir.dir("spec") +val templatesDir = rootDir.dir("server-templates") +val generatedDir = project.layout.buildDirectory.dir("generated-openapi") +val generatedOpenApiSrcDir = project.layout.buildDirectory.dir("generated-openapi/src/main/java") + +openApiGenerate { + inputSpec = provider { specsDir.file("metrics-reports-service.yml").asFile.absolutePath } + generatorName = "jaxrs-resteasy" + outputDir = provider { generatedDir.get().asFile.absolutePath } + apiPackage = "org.apache.polaris.service.metrics.api" + modelPackage = "org.apache.polaris.core.metrics.api.model" + ignoreFileOverride.set(provider { rootDir.file(".openapi-generator-ignore").asFile.absolutePath }) + removeOperationIdPrefix.set(true) + templateDir.set(provider { templatesDir.asFile.absolutePath }) + globalProperties.put("apis", "") + globalProperties.put("models", "") + globalProperties.put("apiDocs", "false") + globalProperties.put("modelTests", "false") + configOptions.put("openApiNullable", "false") + configOptions.put("useBeanValidation", "true") + configOptions.put("sourceFolder", "src/main/java") + configOptions.put("useJakartaEe", "true") + configOptions.put("generateBuilders", "true") + configOptions.put("generateConstructorWithAllArgs", "true") + configOptions.put("hideGenerationTimestamp", "true") + additionalProperties.put("apiNamePrefix", "Polaris") + additionalProperties.put("apiNameSuffix", "Api") + additionalProperties.put("metricsPrefix", "polaris") + serverVariables.put("basePath", "api/metrics-reports/v1") +} + +listOf("sourcesJar", "compileJava", "processResources").forEach { task -> + tasks.named(task) { dependsOn("openApiGenerate") } +} + +sourceSets { main { java { srcDir(generatedOpenApiSrcDir) } } } + +tasks.named("openApiGenerate") { + inputs.dir(templatesDir) + inputs.dir(specsDir) + actions.addFirst { delete { delete(generatedDir) } } +} + +tasks.named("javadoc") { dependsOn("jandex") } diff --git a/bom/build.gradle.kts b/bom/build.gradle.kts index 6a67f1e9488..7ad385ab46e 100644 --- a/bom/build.gradle.kts +++ b/bom/build.gradle.kts @@ -30,6 +30,7 @@ dependencies { api(project(":polaris-api-iceberg-service")) api(project(":polaris-api-management-model")) api(project(":polaris-api-management-service")) + api(project(":polaris-api-metrics-reports-service")) api(project(":polaris-container-spec-helper")) api(project(":polaris-azurite-testcontainer")) @@ -101,6 +102,8 @@ dependencies { api(project(":polaris-extensions-federation-bigquery")) api(project(":polaris-extensions-federation-hadoop")) api(project(":polaris-extensions-federation-hive")) + api(project(":polaris-extensions-metrics-reports-jdbc")) + api(project(":polaris-extensions-metrics-reports-spi")) api(project(":polaris-hms-testcontainer")) api(project(":polaris-spark-3.5_2.12")) @@ -112,6 +115,7 @@ dependencies { api(project(":polaris-spark-4.0_2.13")) api(project(":polaris-spark-integration-4.0_2.13")) } + api(project(":polaris-extensions-metrics-reports")) api(project(":polaris-admin")) api(project(":polaris-runtime-common")) diff --git a/extensions/metrics-reports/impl/build.gradle.kts b/extensions/metrics-reports/impl/build.gradle.kts new file mode 100644 index 00000000000..e60711aaa62 --- /dev/null +++ b/extensions/metrics-reports/impl/build.gradle.kts @@ -0,0 +1,39 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +plugins { + id("polaris-server") + id("org.kordamp.gradle.jandex") +} + +dependencies { + implementation(project(":polaris-extensions-metrics-reports-spi")) + + implementation(platform(libs.iceberg.bom)) + implementation("org.apache.iceberg:iceberg-api") + + implementation(libs.jakarta.enterprise.cdi.api) + implementation(libs.jakarta.inject.api) + implementation(libs.smallrye.common.annotation) + implementation(libs.slf4j.api) + + testImplementation(platform(libs.junit.bom)) + testImplementation("org.junit.jupiter:junit-jupiter") + testImplementation(libs.mockito.core) +} diff --git a/runtime/service/src/main/java/org/apache/polaris/service/reporting/DefaultMetricsReporter.java b/extensions/metrics-reports/impl/src/main/java/org/apache/polaris/extension/metrics/reports/LoggingMetricsReporter.java similarity index 66% rename from runtime/service/src/main/java/org/apache/polaris/service/reporting/DefaultMetricsReporter.java rename to extensions/metrics-reports/impl/src/main/java/org/apache/polaris/extension/metrics/reports/LoggingMetricsReporter.java index e58f740c3c0..ae8b6751928 100644 --- a/runtime/service/src/main/java/org/apache/polaris/service/reporting/DefaultMetricsReporter.java +++ b/extensions/metrics-reports/impl/src/main/java/org/apache/polaris/extension/metrics/reports/LoggingMetricsReporter.java @@ -16,37 +16,30 @@ * specific language governing permissions and limitations * under the License. */ -package org.apache.polaris.service.reporting; +package org.apache.polaris.extension.metrics.reports; -import com.google.common.annotations.VisibleForTesting; import io.smallrye.common.annotation.Identifier; import jakarta.enterprise.context.ApplicationScoped; import java.time.Instant; import org.apache.iceberg.catalog.TableIdentifier; import org.apache.iceberg.metrics.MetricsReport; +import org.apache.polaris.extension.metrics.IcebergMetricsReporter; import org.slf4j.Logger; import org.slf4j.LoggerFactory; /** - * Default implementation of {@link PolarisMetricsReporter} that logs metrics to the configured - * logger. + * Log-only implementation of {@link IcebergMetricsReporter}. * - *

This implementation is selected when {@code polaris.iceberg-metrics.reporting.type} is set to - * {@code "default"} (the default value). + *

Selected when {@code polaris.iceberg-metrics.reporting.type} is set to {@code "log"}. * - *

By default, logging is disabled. To enable metrics logging, set the logger level for {@code - * org.apache.polaris.service.reporting} to {@code INFO} in your logging configuration. - * - * @see PolarisMetricsReporter + *

Logging is at INFO level. Enable it by setting the logger level for {@code + * org.apache.polaris.extension.metrics.reports} to {@code INFO}. */ @ApplicationScoped -@Identifier("default") -public class DefaultMetricsReporter implements PolarisMetricsReporter { - private static final Logger LOGGER = LoggerFactory.getLogger(DefaultMetricsReporter.class); - - private final ReportConsumer reportConsumer; +@Identifier("log") +public class LoggingMetricsReporter implements IcebergMetricsReporter { + private static final Logger LOGGER = LoggerFactory.getLogger(LoggingMetricsReporter.class); - /** Functional interface for consuming metrics reports with timestamp. */ @FunctionalInterface interface ReportConsumer { void accept( @@ -58,15 +51,15 @@ void accept( Instant receivedTimestamp); } - /** Creates a new DefaultMetricsReporter that logs metrics to the class logger. */ - public DefaultMetricsReporter() { + private final ReportConsumer reportConsumer; + + public LoggingMetricsReporter() { this( (catalogName, catalogId, table, tableId, metricsReport, receivedTimestamp) -> LOGGER.info("{}.{} (ts={}): {}", catalogName, table, receivedTimestamp, metricsReport)); } - @VisibleForTesting - DefaultMetricsReporter(ReportConsumer reportConsumer) { + LoggingMetricsReporter(ReportConsumer reportConsumer) { this.reportConsumer = reportConsumer; } diff --git a/extensions/metrics-reports/impl/src/main/java/org/apache/polaris/extension/metrics/reports/NoOpMetricsReporter.java b/extensions/metrics-reports/impl/src/main/java/org/apache/polaris/extension/metrics/reports/NoOpMetricsReporter.java new file mode 100644 index 00000000000..0e0133166d1 --- /dev/null +++ b/extensions/metrics-reports/impl/src/main/java/org/apache/polaris/extension/metrics/reports/NoOpMetricsReporter.java @@ -0,0 +1,45 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.polaris.extension.metrics.reports; + +import io.smallrye.common.annotation.Identifier; +import jakarta.enterprise.context.ApplicationScoped; +import java.time.Instant; +import org.apache.iceberg.catalog.TableIdentifier; +import org.apache.iceberg.metrics.MetricsReport; +import org.apache.polaris.extension.metrics.IcebergMetricsReporter; + +/** + * No-op implementation of {@link IcebergMetricsReporter} that silently discards all metrics. + * + *

Selected when {@code polaris.iceberg-metrics.reporting.type} is set to {@code "no-op"}. + */ +@ApplicationScoped +@Identifier("no-op") +public class NoOpMetricsReporter implements IcebergMetricsReporter { + + @Override + public void reportMetric( + String catalogName, + long catalogId, + TableIdentifier table, + long tableId, + MetricsReport metricsReport, + Instant receivedTimestamp) {} +} diff --git a/runtime/service/src/test/java/org/apache/polaris/service/reporting/DefaultMetricsReporterTest.java b/extensions/metrics-reports/impl/src/test/java/org/apache/polaris/extension/metrics/reports/LoggingMetricsReporterTest.java similarity index 85% rename from runtime/service/src/test/java/org/apache/polaris/service/reporting/DefaultMetricsReporterTest.java rename to extensions/metrics-reports/impl/src/test/java/org/apache/polaris/extension/metrics/reports/LoggingMetricsReporterTest.java index 2240f41c921..cc47c55d9fd 100644 --- a/runtime/service/src/test/java/org/apache/polaris/service/reporting/DefaultMetricsReporterTest.java +++ b/extensions/metrics-reports/impl/src/test/java/org/apache/polaris/extension/metrics/reports/LoggingMetricsReporterTest.java @@ -16,7 +16,7 @@ * specific language governing permissions and limitations * under the License. */ -package org.apache.polaris.service.reporting; +package org.apache.polaris.extension.metrics.reports; import static org.mockito.Mockito.mock; import static org.mockito.Mockito.verify; @@ -26,13 +26,13 @@ import org.apache.iceberg.metrics.MetricsReport; import org.junit.jupiter.api.Test; -public class DefaultMetricsReporterTest { +public class LoggingMetricsReporterTest { @Test void testLogging() { - DefaultMetricsReporter.ReportConsumer mockConsumer = - mock(DefaultMetricsReporter.ReportConsumer.class); - DefaultMetricsReporter reporter = new DefaultMetricsReporter(mockConsumer); + LoggingMetricsReporter.ReportConsumer mockConsumer = + mock(LoggingMetricsReporter.ReportConsumer.class); + LoggingMetricsReporter reporter = new LoggingMetricsReporter(mockConsumer); String warehouse = "testWarehouse"; long catalogId = 12345L; TableIdentifier table = TableIdentifier.of("testNamespace", "testTable"); diff --git a/extensions/metrics-reports/persistence/relational-jdbc/build.gradle.kts b/extensions/metrics-reports/persistence/relational-jdbc/build.gradle.kts new file mode 100644 index 00000000000..7e0d365adc0 --- /dev/null +++ b/extensions/metrics-reports/persistence/relational-jdbc/build.gradle.kts @@ -0,0 +1,64 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +plugins { + id("polaris-server") + id("org.kordamp.gradle.jandex") +} + +dependencies { + implementation(project(":polaris-core")) + implementation(project(":polaris-relational-jdbc")) + implementation(project(":polaris-extensions-metrics-reports")) + implementation(project(":polaris-extensions-metrics-reports-spi")) + + implementation(platform(libs.iceberg.bom)) + implementation("org.apache.iceberg:iceberg-api") + implementation("org.apache.iceberg:iceberg-core") + + compileOnly(libs.jspecify) + + compileOnly(libs.postgresql) + + implementation(libs.jakarta.enterprise.cdi.api) + implementation(libs.jakarta.inject.api) + implementation(libs.smallrye.common.annotation) + implementation(libs.slf4j.api) + + compileOnly(platform(libs.opentelemetry.instrumentation.bom.alpha)) + compileOnly("io.opentelemetry:opentelemetry-api") + compileOnly(libs.jakarta.annotation.api) + + compileOnly(project(":polaris-immutables")) + annotationProcessor(project(":polaris-immutables", configuration = "processor")) + + implementation(platform(libs.jackson.bom)) + implementation("com.fasterxml.jackson.core:jackson-annotations") + implementation("com.fasterxml.jackson.core:jackson-databind") + + testImplementation(platform(libs.junit.bom)) + testImplementation("org.junit.jupiter:junit-jupiter") + testImplementation(libs.assertj.core) + testImplementation(libs.mockito.core) + testImplementation(libs.h2) + + // Provides RuntimeDelegate for Response construction in unit tests + testRuntimeOnly(enforcedPlatform(libs.quarkus.bom)) + testRuntimeOnly("io.quarkus.resteasy.reactive:resteasy-reactive") +} diff --git a/extensions/metrics-reports/persistence/relational-jdbc/src/main/java/org/apache/polaris/extension/metrics/jdbc/JdbcMetricsPersistence.java b/extensions/metrics-reports/persistence/relational-jdbc/src/main/java/org/apache/polaris/extension/metrics/jdbc/JdbcMetricsPersistence.java new file mode 100644 index 00000000000..84b574bceaf --- /dev/null +++ b/extensions/metrics-reports/persistence/relational-jdbc/src/main/java/org/apache/polaris/extension/metrics/jdbc/JdbcMetricsPersistence.java @@ -0,0 +1,237 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.polaris.extension.metrics.jdbc; + +import static org.apache.polaris.persistence.relational.jdbc.QueryGenerator.PreparedQuery; + +import jakarta.enterprise.context.ApplicationScoped; +import jakarta.inject.Inject; +import java.sql.SQLException; +import java.util.ArrayList; +import java.util.List; +import java.util.function.Function; +import org.apache.polaris.core.context.RealmContext; +import org.apache.polaris.core.persistence.metrics.CommitMetricsRecord; +import org.apache.polaris.core.persistence.metrics.MetricsPersistence; +import org.apache.polaris.core.persistence.metrics.MetricsQuerySpi; +import org.apache.polaris.core.persistence.metrics.ScanMetricsRecord; +import org.apache.polaris.core.persistence.pagination.Page; +import org.apache.polaris.core.persistence.pagination.PageToken; +import org.apache.polaris.persistence.relational.jdbc.DatasourceOperations; +import org.apache.polaris.persistence.relational.jdbc.QueryGenerator; +import org.apache.polaris.persistence.relational.jdbc.models.ModelCommitMetricsReport; +import org.apache.polaris.persistence.relational.jdbc.models.ModelScanMetricsReport; +import org.apache.polaris.persistence.relational.jdbc.pagination.MetricsReportToken; +import org.jspecify.annotations.NonNull; +import org.jspecify.annotations.Nullable; + +/** + * JDBC implementation of {@link MetricsPersistence}. + * + *

Writes and reads scan/commit metrics reports using the shared {@link DatasourceOperations} + * connection pool. The metrics tables ({@code SCAN_METRICS_REPORT}, {@code COMMIT_METRICS_REPORT}) + * are part of the standard JDBC schema (schema-v4). + * + *

This bean is contributed by the {@code polaris-extensions-metrics-reports-jdbc} extension + * module. When this module is on the classpath the {@code "persisting"} reporter wires to it; when + * absent the reporter falls back to no-op behavior. + */ +@ApplicationScoped +public class JdbcMetricsPersistence implements MetricsPersistence, MetricsQuerySpi { + + private final DatasourceOperations datasourceOperations; + private final RealmContext realmContext; + + @Inject + public JdbcMetricsPersistence( + DatasourceOperations datasourceOperations, RealmContext realmContext) { + this.datasourceOperations = datasourceOperations; + this.realmContext = realmContext; + } + + @Override + public void writeScanReport(@NonNull ScanMetricsRecord record) { + String realmId = realmContext.getRealmIdentifier(); + ModelScanMetricsReport model = ModelScanMetricsReport.fromRecord(record, realmId); + PreparedQuery pq = + QueryGenerator.generateInsertQuery( + ModelScanMetricsReport.ALL_COLUMNS, + ModelScanMetricsReport.TABLE_NAME, + model.toMap(datasourceOperations.getDatabaseType()).values().stream().toList(), + realmId); + try { + datasourceOperations.executeUpdate(pq); + } catch (SQLException e) { + throw new RuntimeException("Failed to write scan metrics report: " + e.getMessage(), e); + } + } + + @Override + public void writeCommitReport(@NonNull CommitMetricsRecord record) { + String realmId = realmContext.getRealmIdentifier(); + ModelCommitMetricsReport model = ModelCommitMetricsReport.fromRecord(record, realmId); + PreparedQuery pq = + QueryGenerator.generateInsertQuery( + ModelCommitMetricsReport.ALL_COLUMNS, + ModelCommitMetricsReport.TABLE_NAME, + model.toMap(datasourceOperations.getDatabaseType()).values().stream().toList(), + realmId); + try { + datasourceOperations.executeUpdate(pq); + } catch (SQLException e) { + throw new RuntimeException("Failed to write commit metrics report: " + e.getMessage(), e); + } + } + + @Override + public Page listScanReports( + long catalogId, + long tableId, + @Nullable Long snapshotId, + @Nullable String principalName, + @Nullable Long timestampFrom, + @Nullable Long timestampTo, + @NonNull PageToken pageToken) { + String realmId = realmContext.getRealmIdentifier(); + try { + PreparedQuery query = + buildMetricsQuery( + ModelScanMetricsReport.TABLE_NAME, + realmId, + catalogId, + tableId, + snapshotId, + principalName, + timestampFrom, + timestampTo, + pageToken); + List rows = + datasourceOperations.executeSelect(query, ModelScanMetricsReport.CONVERTER); + return Page.mapped( + pageToken, + rows.stream().map(ModelScanMetricsReport::toRecord), + Function.identity(), + MetricsReportToken::fromRecord); + } catch (SQLException e) { + throw new RuntimeException("Failed to list scan metrics reports: " + e.getMessage(), e); + } + } + + @Override + public Page listCommitReports( + long catalogId, + long tableId, + @Nullable Long snapshotId, + @Nullable String principalName, + @Nullable Long timestampFrom, + @Nullable Long timestampTo, + @NonNull PageToken pageToken) { + String realmId = realmContext.getRealmIdentifier(); + try { + PreparedQuery query = + buildMetricsQuery( + ModelCommitMetricsReport.TABLE_NAME, + realmId, + catalogId, + tableId, + snapshotId, + principalName, + timestampFrom, + timestampTo, + pageToken); + List rows = + datasourceOperations.executeSelect(query, ModelCommitMetricsReport.CONVERTER); + return Page.mapped( + pageToken, + rows.stream().map(ModelCommitMetricsReport::toRecord), + Function.identity(), + MetricsReportToken::fromRecord); + } catch (SQLException e) { + throw new RuntimeException("Failed to list commit metrics reports: " + e.getMessage(), e); + } + } + + /** + * Builds a parameterized SELECT query for a metrics report table using keyset pagination. + * + *

Rows are ordered by {@code (timestamp_ms DESC, report_id DESC)}. The cursor from {@link + * MetricsReportToken} drives the keyset predicate: {@code (timestamp_ms < cursorTs) OR + * (timestamp_ms = cursorTs AND report_id < cursorId)}. + */ + private PreparedQuery buildMetricsQuery( + String tableName, + String realmId, + long catalogId, + long tableId, + @Nullable Long snapshotId, + @Nullable String principalName, + @Nullable Long timestampFrom, + @Nullable Long timestampTo, + PageToken pageToken) { + StringBuilder sql = new StringBuilder("SELECT * FROM "); + sql.append(QueryGenerator.getFullyQualifiedTableName(tableName)); + sql.append(" WHERE realm_id = ? AND catalog_id = ? AND table_id = ?"); + + List params = new ArrayList<>(); + params.add(realmId); + params.add(catalogId); + params.add(tableId); + + if (snapshotId != null) { + sql.append(" AND snapshot_id = ?"); + params.add(snapshotId); + } + if (principalName != null) { + sql.append(" AND principal_name = ?"); + params.add(principalName); + } + if (timestampFrom != null) { + sql.append(" AND timestamp_ms >= ?"); + params.add(timestampFrom); + } + if (timestampTo != null) { + sql.append(" AND timestamp_ms < ?"); + params.add(timestampTo); + } + + if (pageToken.paginationRequested()) { + if (pageToken.value().isPresent() && pageToken.valueAs(MetricsReportToken.class).isEmpty()) { + throw new IllegalArgumentException( + "pageToken contains a cursor of an unexpected type; expected MetricsReportToken"); + } + pageToken + .valueAs(MetricsReportToken.class) + .ifPresent( + cursor -> { + sql.append(" AND (timestamp_ms < ? OR (timestamp_ms = ? AND report_id < ?))"); + params.add(cursor.timestampMs()); + params.add(cursor.timestampMs()); + params.add(cursor.reportId()); + }); + } + + sql.append(" ORDER BY timestamp_ms DESC, report_id DESC"); + + int limit = pageToken.pageSize().orElse(100); + sql.append(" LIMIT ?"); + params.add(limit + 1); + + return new PreparedQuery(sql.toString(), params); + } +} diff --git a/polaris-core/src/main/java/org/apache/polaris/core/metrics/iceberg/MetricsRecordConverter.java b/extensions/metrics-reports/persistence/relational-jdbc/src/main/java/org/apache/polaris/extension/metrics/jdbc/MetricsRecordConverter.java similarity index 80% rename from polaris-core/src/main/java/org/apache/polaris/core/metrics/iceberg/MetricsRecordConverter.java rename to extensions/metrics-reports/persistence/relational-jdbc/src/main/java/org/apache/polaris/extension/metrics/jdbc/MetricsRecordConverter.java index 111f77a286e..7bccf2a036f 100644 --- a/polaris-core/src/main/java/org/apache/polaris/core/metrics/iceberg/MetricsRecordConverter.java +++ b/extensions/metrics-reports/persistence/relational-jdbc/src/main/java/org/apache/polaris/extension/metrics/jdbc/MetricsRecordConverter.java @@ -16,7 +16,7 @@ * specific language governing permissions and limitations * under the License. */ -package org.apache.polaris.core.metrics.iceberg; +package org.apache.polaris.extension.metrics.jdbc; import java.time.Instant; import java.util.Collections; @@ -35,9 +35,6 @@ /** * Converts Iceberg metrics reports to SPI record types using a fluent builder API. * - *

This converter extracts all relevant metrics from Iceberg's {@link ScanReport} and {@link - * CommitReport} and combines them with context information to create persistence-ready records. - * *

Example usage: * *

{@code
@@ -49,26 +46,12 @@
  */
 public final class MetricsRecordConverter {
 
-  private MetricsRecordConverter() {
-    // Utility class
-  }
+  private MetricsRecordConverter() {}
 
-  /**
-   * Creates a builder for converting a ScanReport to a ScanMetricsRecord.
-   *
-   * @param scanReport the Iceberg scan report
-   * @return builder for configuring the conversion
-   */
   public static ScanReportBuilder forScanReport(ScanReport scanReport) {
     return new ScanReportBuilder(scanReport);
   }
 
-  /**
-   * Creates a builder for converting a CommitReport to a CommitMetricsRecord.
-   *
-   * @param commitReport the Iceberg commit report
-   * @return builder for configuring the conversion
-   */
   public static CommitReportBuilder forCommitReport(CommitReport commitReport) {
     return new CommitReportBuilder(commitReport);
   }
@@ -96,52 +79,31 @@ public ScanReportBuilder catalogId(long catalogId) {
       return this;
     }
 
-    /**
-     * Sets the table entity ID.
-     *
-     * 

This is the internal Polaris entity ID for the table. - * - * @param tableId the table entity ID - * @return this builder - */ public ScanReportBuilder tableId(long tableId) { this.tableId = tableId; return this; } - /** - * Sets the timestamp for the metrics record. - * - *

This should be the time the metrics report was received by the server, which may differ - * from the time it was recorded by the client. - * - * @param timestamp the timestamp - * @return this builder - */ public ScanReportBuilder timestamp(Instant timestamp) { this.timestamp = timestamp; return this; } - /** Sets the principal name for request context. */ public ScanReportBuilder principalName(String principalName) { this.principalName = principalName; return this; } - /** Sets the server-generated request ID. */ public ScanReportBuilder requestId(String requestId) { this.requestId = requestId; return this; } - /** Sets the OpenTelemetry trace ID. */ public ScanReportBuilder otelTraceId(String otelTraceId) { this.otelTraceId = otelTraceId; return this; } - /** Sets the OpenTelemetry span ID. */ public ScanReportBuilder otelSpanId(String otelSpanId) { this.otelSpanId = otelSpanId; return this; @@ -216,52 +178,31 @@ public CommitReportBuilder catalogId(long catalogId) { return this; } - /** - * Sets the table entity ID. - * - *

This is the internal Polaris entity ID for the table. - * - * @param tableId the table entity ID - * @return this builder - */ public CommitReportBuilder tableId(long tableId) { this.tableId = tableId; return this; } - /** - * Sets the timestamp for the metrics record. - * - *

This should be the time the metrics report was received by the server, which may differ - * from the time it was recorded by the client. - * - * @param timestamp the timestamp - * @return this builder - */ public CommitReportBuilder timestamp(Instant timestamp) { this.timestamp = timestamp; return this; } - /** Sets the principal name for request context. */ public CommitReportBuilder principalName(String principalName) { this.principalName = principalName; return this; } - /** Sets the server-generated request ID. */ public CommitReportBuilder requestId(String requestId) { this.requestId = requestId; return this; } - /** Sets the OpenTelemetry trace ID. */ public CommitReportBuilder otelTraceId(String otelTraceId) { this.otelTraceId = otelTraceId; return this; } - /** Sets the OpenTelemetry span ID. */ public CommitReportBuilder otelSpanId(String otelSpanId) { this.otelSpanId = otelSpanId; return this; @@ -307,33 +248,23 @@ public CommitMetricsRecord build() { } } - // === Helper Methods === - private static long getCounterValue(CounterResult counter) { - if (counter == null) { - return 0L; - } + if (counter == null) return 0L; return counter.value(); } private static int getCounterValueInt(CounterResult counter) { - if (counter == null) { - return 0; - } + if (counter == null) return 0; return (int) counter.value(); } private static long getTimerValueMs(TimerResult timer) { - if (timer == null || timer.totalDuration() == null) { - return 0L; - } + if (timer == null || timer.totalDuration() == null) return 0L; return timer.totalDuration().toMillis(); } private static Optional getTimerValueMsOpt(TimerResult timer) { - if (timer == null || timer.totalDuration() == null) { - return Optional.empty(); - } + if (timer == null || timer.totalDuration() == null) return Optional.empty(); return Optional.of(timer.totalDuration().toMillis()); } } diff --git a/runtime/service/src/main/java/org/apache/polaris/service/reporting/PersistingMetricsReporter.java b/extensions/metrics-reports/persistence/relational-jdbc/src/main/java/org/apache/polaris/extension/metrics/jdbc/PersistingMetricsReporter.java similarity index 67% rename from runtime/service/src/main/java/org/apache/polaris/service/reporting/PersistingMetricsReporter.java rename to extensions/metrics-reports/persistence/relational-jdbc/src/main/java/org/apache/polaris/extension/metrics/jdbc/PersistingMetricsReporter.java index 8703d3b8db5..b4b6a2ec090 100644 --- a/runtime/service/src/main/java/org/apache/polaris/service/reporting/PersistingMetricsReporter.java +++ b/extensions/metrics-reports/persistence/relational-jdbc/src/main/java/org/apache/polaris/extension/metrics/jdbc/PersistingMetricsReporter.java @@ -16,7 +16,7 @@ * specific language governing permissions and limitations * under the License. */ -package org.apache.polaris.service.reporting; +package org.apache.polaris.extension.metrics.jdbc; import io.opentelemetry.api.trace.Span; import io.opentelemetry.api.trace.SpanContext; @@ -30,52 +30,41 @@ import org.apache.iceberg.metrics.MetricsReport; import org.apache.iceberg.metrics.ScanReport; import org.apache.polaris.core.auth.PolarisPrincipal; -import org.apache.polaris.core.context.CallContext; import org.apache.polaris.core.context.RequestIdSupplier; -import org.apache.polaris.core.metrics.iceberg.MetricsRecordConverter; -import org.apache.polaris.core.persistence.PolarisMetaStoreManager; import org.apache.polaris.core.persistence.metrics.CommitMetricsRecord; +import org.apache.polaris.core.persistence.metrics.MetricsPersistence; import org.apache.polaris.core.persistence.metrics.ScanMetricsRecord; +import org.apache.polaris.extension.metrics.IcebergMetricsReporter; import org.slf4j.Logger; import org.slf4j.LoggerFactory; /** - * Implementation of {@link PolarisMetricsReporter} that persists metrics using the {@link - * PolarisMetaStoreManager} from the current {@link CallContext}. + * Persisting implementation of {@link IcebergMetricsReporter}. * - *

This reporter is selected when {@code polaris.iceberg-metrics.reporting.type} is set to {@code - * "persisting"}. + *

Selected when {@code polaris.iceberg-metrics.reporting.type} is set to {@code "persisting"}. + * Requires a {@link MetricsPersistence} bean on the classpath (provided by the {@code + * polaris-extensions-metrics-reports-jdbc} extension module). * - *

The reporter uses {@link PolarisMetaStoreManager} to persist metrics, following the same - * abstraction pattern as other Polaris operations. If the underlying persistence does not support - * metrics, they are silently discarded. - * - *

The reporter receives catalog and table IDs from the caller (already resolved during - * authorization), avoiding redundant entity lookups. It uses {@link MetricsRecordConverter} to - * convert Iceberg metrics reports to SPI records before persisting them. - * - * @see PolarisMetricsReporter - * @see PolarisMetaStoreManager - * @see MetricsRecordConverter + *

Converts Iceberg {@link ScanReport}/{@link CommitReport} objects to backend-agnostic SPI + * records via {@link MetricsRecordConverter}, then delegates to {@link MetricsPersistence} for + * durable storage. */ @RequestScoped @Identifier("persisting") -public class PersistingMetricsReporter implements PolarisMetricsReporter { +public class PersistingMetricsReporter implements IcebergMetricsReporter { + private static final Logger LOGGER = LoggerFactory.getLogger(PersistingMetricsReporter.class); - private final CallContext callContext; - private final PolarisMetaStoreManager metaStoreManager; + private final MetricsPersistence metricsPersistence; private final Instance polarisPrincipal; private final Instance requestIdSupplier; @Inject public PersistingMetricsReporter( - CallContext callContext, - PolarisMetaStoreManager metaStoreManager, + MetricsPersistence metricsPersistence, Instance polarisPrincipal, Instance requestIdSupplier) { - this.callContext = callContext; - this.metaStoreManager = metaStoreManager; + this.metricsPersistence = metricsPersistence; this.polarisPrincipal = polarisPrincipal; this.requestIdSupplier = requestIdSupplier; } @@ -89,13 +78,11 @@ public void reportMetric( MetricsReport metricsReport, Instant receivedTimestamp) { - // Resolve request context String principalName = resolvePrincipalName(); String requestId = resolveRequestId(); String otelTraceId = null; String otelSpanId = null; - // Get OpenTelemetry context if available SpanContext spanContext = Span.current().getSpanContext(); if (spanContext.isValid()) { otelTraceId = spanContext.getTraceId(); @@ -113,7 +100,7 @@ public void reportMetric( .otelTraceId(otelTraceId) .otelSpanId(otelSpanId) .build(); - metaStoreManager.writeScanMetrics(callContext.getPolarisCallContext(), record); + metricsPersistence.writeScanReport(record); LOGGER.debug( "Persisted scan metrics for {}.{} (reportId={})", catalogName, table, record.reportId()); } else if (metricsReport instanceof CommitReport commitReport) { @@ -127,7 +114,7 @@ public void reportMetric( .otelTraceId(otelTraceId) .otelSpanId(otelSpanId) .build(); - metaStoreManager.writeCommitMetrics(callContext.getPolarisCallContext(), record); + metricsPersistence.writeCommitReport(record); LOGGER.debug( "Persisted commit metrics for {}.{} (reportId={})", catalogName, @@ -135,23 +122,23 @@ public void reportMetric( record.reportId()); } else { LOGGER.warn( - "Unknown metrics report type: {}. Metrics will not be stored.", + "Unknown metrics report type: {}. Report will not be stored.", metricsReport.getClass().getName()); } } private String resolvePrincipalName() { if (polarisPrincipal.isResolvable()) { - PolarisPrincipal principal = polarisPrincipal.get(); - return principal != null ? principal.getName() : null; + PolarisPrincipal p = polarisPrincipal.get(); + return p != null ? p.getName() : null; } return null; } private String resolveRequestId() { if (requestIdSupplier.isResolvable()) { - RequestIdSupplier supplier = requestIdSupplier.get(); - return supplier != null ? supplier.getRequestId() : null; + RequestIdSupplier s = requestIdSupplier.get(); + return s != null ? s.getRequestId() : null; } return null; } diff --git a/extensions/metrics-reports/persistence/relational-jdbc/src/main/java/org/apache/polaris/persistence/relational/jdbc/models/MetricsModelUtils.java b/extensions/metrics-reports/persistence/relational-jdbc/src/main/java/org/apache/polaris/persistence/relational/jdbc/models/MetricsModelUtils.java new file mode 100644 index 00000000000..44c97f9f844 --- /dev/null +++ b/extensions/metrics-reports/persistence/relational-jdbc/src/main/java/org/apache/polaris/persistence/relational/jdbc/models/MetricsModelUtils.java @@ -0,0 +1,48 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.polaris.persistence.relational.jdbc.models; + +import com.fasterxml.jackson.core.JsonProcessingException; +import com.fasterxml.jackson.core.type.TypeReference; +import com.fasterxml.jackson.databind.ObjectMapper; +import java.util.Map; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +/** Shared utilities for metrics model classes. */ +public final class MetricsModelUtils { + + private static final Logger LOGGER = LoggerFactory.getLogger(MetricsModelUtils.class); + + public static final ObjectMapper OBJECT_MAPPER = new ObjectMapper(); + + private MetricsModelUtils() {} + + public static Map parseMetadata(String json) { + if (json == null || json.isEmpty() || json.equals("{}")) { + return Map.of(); + } + try { + return OBJECT_MAPPER.readValue(json, new TypeReference>() {}); + } catch (JsonProcessingException e) { + LOGGER.warn("Failed to parse metadata JSON: {}", e.getMessage()); + return Map.of(); + } + } +} diff --git a/persistence/relational-jdbc/src/main/java/org/apache/polaris/persistence/relational/jdbc/models/ModelCommitMetricsReport.java b/extensions/metrics-reports/persistence/relational-jdbc/src/main/java/org/apache/polaris/persistence/relational/jdbc/models/ModelCommitMetricsReport.java similarity index 85% rename from persistence/relational-jdbc/src/main/java/org/apache/polaris/persistence/relational/jdbc/models/ModelCommitMetricsReport.java rename to extensions/metrics-reports/persistence/relational-jdbc/src/main/java/org/apache/polaris/persistence/relational/jdbc/models/ModelCommitMetricsReport.java index 03a9096fae1..c2006d93eb4 100644 --- a/persistence/relational-jdbc/src/main/java/org/apache/polaris/persistence/relational/jdbc/models/ModelCommitMetricsReport.java +++ b/extensions/metrics-reports/persistence/relational-jdbc/src/main/java/org/apache/polaris/persistence/relational/jdbc/models/ModelCommitMetricsReport.java @@ -19,23 +19,23 @@ package org.apache.polaris.persistence.relational.jdbc.models; import com.fasterxml.jackson.core.JsonProcessingException; -import com.fasterxml.jackson.databind.ObjectMapper; import java.sql.ResultSet; import java.sql.SQLException; +import java.time.Instant; import java.util.LinkedHashMap; import java.util.List; import java.util.Map; +import java.util.Optional; import org.apache.polaris.core.persistence.metrics.CommitMetricsRecord; import org.apache.polaris.immutables.PolarisImmutable; import org.apache.polaris.persistence.relational.jdbc.DatabaseType; import org.jspecify.annotations.Nullable; -/** Model class for commit_metrics_report table - stores commit metrics as first-class entities. */ +/** JDBC model for the {@code COMMIT_METRICS_REPORT} table. */ @PolarisImmutable public interface ModelCommitMetricsReport extends Converter { String TABLE_NAME = "COMMIT_METRICS_REPORT"; - // Column names String REPORT_ID = "report_id"; String REALM_ID = "realm_id"; String CATALOG_ID = "catalog_id"; @@ -103,7 +103,6 @@ public interface ModelCommitMetricsReport extends Converter toMap(DatabaseType databaseType) { map.put(TOTAL_FILE_SIZE_BYTES, getTotalFileSizeBytes()); map.put(TOTAL_DURATION_MS, getTotalDurationMs()); map.put(ATTEMPTS, getAttempts()); - if (databaseType.equals(DatabaseType.POSTGRES)) { map.put(METADATA, toJsonbPGobject(getMetadata() != null ? getMetadata() : "{}")); } else { @@ -248,24 +246,8 @@ default Map toMap(DatabaseType databaseType) { return map; } - // === Static conversion methods (following ModelEntity pattern) === - - ObjectMapper OBJECT_MAPPER = new ObjectMapper(); - - /** - * Converts a CommitMetricsRecord (SPI) to ModelCommitMetricsReport (JDBC). - * - *

Request context fields (principalName, requestId, otelTraceId, otelSpanId) are read directly - * from the record, which should have been populated by the service layer. - * - * @param record the SPI record containing all metrics and request context - * @param realmId the realm ID for multi-tenancy - * @return the JDBC model - */ static ModelCommitMetricsReport fromRecord(CommitMetricsRecord record, String realmId) { - // Extract client-provided report trace ID from metadata String reportTraceId = record.metadata().get("report-trace-id"); - return ImmutableModelCommitMetricsReport.builder() .reportId(record.reportId()) .realmId(realmId) @@ -302,20 +284,43 @@ static ModelCommitMetricsReport fromRecord(CommitMetricsRecord record, String re .build(); } - // === Helper Methods === - - private static String toJsonString(Map map) { - if (map == null || map.isEmpty()) { - return "{}"; - } - try { - return OBJECT_MAPPER.writeValueAsString(map); - } catch (JsonProcessingException e) { - return "{}"; - } + default CommitMetricsRecord toRecord() { + return CommitMetricsRecord.builder() + .reportId(getReportId()) + .catalogId(getCatalogId()) + .tableId(getTableId()) + .timestamp(Instant.ofEpochMilli(getTimestampMs())) + .metadata(MetricsModelUtils.parseMetadata(getMetadata())) + .principalName(getPrincipalName()) + .requestId(getRequestId()) + .otelTraceId(getOtelTraceId()) + .otelSpanId(getOtelSpanId()) + .snapshotId(getSnapshotId()) + .sequenceNumber(Optional.ofNullable(getSequenceNumber())) + .operation(getOperation()) + .addedDataFiles(getAddedDataFiles()) + .removedDataFiles(getRemovedDataFiles()) + .totalDataFiles(getTotalDataFiles()) + .addedDeleteFiles(getAddedDeleteFiles()) + .removedDeleteFiles(getRemovedDeleteFiles()) + .totalDeleteFiles(getTotalDeleteFiles()) + .addedEqualityDeleteFiles(getAddedEqualityDeleteFiles()) + .removedEqualityDeleteFiles(getRemovedEqualityDeleteFiles()) + .addedPositionalDeleteFiles(getAddedPositionalDeleteFiles()) + .removedPositionalDeleteFiles(getRemovedPositionalDeleteFiles()) + .addedRecords(getAddedRecords()) + .removedRecords(getRemovedRecords()) + .totalRecords(getTotalRecords()) + .addedFileSizeBytes(getAddedFileSizeBytes()) + .removedFileSizeBytes(getRemovedFileSizeBytes()) + .totalFileSizeBytes(getTotalFileSizeBytes()) + // total_duration_ms is NOT NULL in DB; 0 means unknown duration + .totalDurationMs( + getTotalDurationMs() == 0L ? Optional.empty() : Optional.of(getTotalDurationMs())) + .attempts(getAttempts()) + .build(); } - /** Dummy instance to be used as a Converter when calling fromResultSet(). */ ModelCommitMetricsReport CONVERTER = ImmutableModelCommitMetricsReport.builder() .reportId("") @@ -344,4 +349,13 @@ private static String toJsonString(Map map) { .totalDurationMs(0L) .attempts(1) .build(); + + private static String toJsonString(Map map) { + if (map == null || map.isEmpty()) return "{}"; + try { + return MetricsModelUtils.OBJECT_MAPPER.writeValueAsString(map); + } catch (JsonProcessingException e) { + return "{}"; + } + } } diff --git a/persistence/relational-jdbc/src/main/java/org/apache/polaris/persistence/relational/jdbc/models/ModelScanMetricsReport.java b/extensions/metrics-reports/persistence/relational-jdbc/src/main/java/org/apache/polaris/persistence/relational/jdbc/models/ModelScanMetricsReport.java similarity index 80% rename from persistence/relational-jdbc/src/main/java/org/apache/polaris/persistence/relational/jdbc/models/ModelScanMetricsReport.java rename to extensions/metrics-reports/persistence/relational-jdbc/src/main/java/org/apache/polaris/persistence/relational/jdbc/models/ModelScanMetricsReport.java index 93115aeac17..da36f97c839 100644 --- a/persistence/relational-jdbc/src/main/java/org/apache/polaris/persistence/relational/jdbc/models/ModelScanMetricsReport.java +++ b/extensions/metrics-reports/persistence/relational-jdbc/src/main/java/org/apache/polaris/persistence/relational/jdbc/models/ModelScanMetricsReport.java @@ -19,24 +19,26 @@ package org.apache.polaris.persistence.relational.jdbc.models; import com.fasterxml.jackson.core.JsonProcessingException; -import com.fasterxml.jackson.databind.ObjectMapper; import java.sql.ResultSet; import java.sql.SQLException; +import java.time.Instant; +import java.util.Arrays; +import java.util.Collections; import java.util.LinkedHashMap; import java.util.List; import java.util.Map; +import java.util.Optional; import java.util.stream.Collectors; import org.apache.polaris.core.persistence.metrics.ScanMetricsRecord; import org.apache.polaris.immutables.PolarisImmutable; import org.apache.polaris.persistence.relational.jdbc.DatabaseType; import org.jspecify.annotations.Nullable; -/** Model class for scan_metrics_report table - stores scan metrics as first-class entities. */ +/** JDBC model for the {@code SCAN_METRICS_REPORT} table. */ @PolarisImmutable public interface ModelScanMetricsReport extends Converter { String TABLE_NAME = "SCAN_METRICS_REPORT"; - // Column names String REPORT_ID = "report_id"; String REALM_ID = "realm_id"; String CATALOG_ID = "catalog_id"; @@ -104,7 +106,6 @@ public interface ModelScanMetricsReport extends Converter toMap(DatabaseType databaseType) { map.put(POSITIONAL_DELETE_FILES, getPositionalDeleteFiles()); map.put(INDEXED_DELETE_FILES, getIndexedDeleteFiles()); map.put(TOTAL_DELETE_FILE_SIZE_BYTES, getTotalDeleteFileSizeBytes()); - if (databaseType.equals(DatabaseType.POSTGRES)) { map.put(METADATA, toJsonbPGobject(getMetadata() != null ? getMetadata() : "{}")); } else { @@ -249,24 +249,8 @@ default Map toMap(DatabaseType databaseType) { return map; } - // === Static conversion methods (following ModelEntity pattern) === - - ObjectMapper OBJECT_MAPPER = new ObjectMapper(); - - /** - * Converts a ScanMetricsRecord (SPI) to ModelScanMetricsReport (JDBC). - * - *

Request context fields (principalName, requestId, otelTraceId, otelSpanId) are read directly - * from the record, which should have been populated by the service layer. - * - * @param record the SPI record containing all metrics and request context - * @param realmId the realm ID for multi-tenancy - * @return the JDBC model - */ static ModelScanMetricsReport fromRecord(ScanMetricsRecord record, String realmId) { - // Extract client-provided report trace ID from metadata String reportTraceId = record.metadata().get("report-trace-id"); - return ImmutableModelScanMetricsReport.builder() .reportId(record.reportId()) .realmId(realmId) @@ -303,27 +287,65 @@ static ModelScanMetricsReport fromRecord(ScanMetricsRecord record, String realmI .build(); } - // === Helper Methods === - - private static String toCommaSeparated(List list) { - if (list == null || list.isEmpty()) { - return null; - } - return list.stream().map(Object::toString).collect(Collectors.joining(",")); - } - - private static String toJsonString(Map map) { - if (map == null || map.isEmpty()) { - return "{}"; - } - try { - return OBJECT_MAPPER.writeValueAsString(map); - } catch (JsonProcessingException e) { - return "{}"; - } + default ScanMetricsRecord toRecord() { + String rawFieldIds = getProjectedFieldIds(); + String rawFieldNames = getProjectedFieldNames(); + List fieldIds = + rawFieldIds == null || rawFieldIds.isEmpty() + ? Collections.emptyList() + : Arrays.stream(rawFieldIds.split(",")) + .map(String::trim) + .filter(s -> !s.isEmpty()) + .flatMap( + s -> { + try { + return java.util.stream.Stream.of(Integer.parseInt(s)); + } catch (NumberFormatException e) { + return java.util.stream.Stream.empty(); + } + }) + .collect(Collectors.toList()); + List fieldNames = + rawFieldNames == null || rawFieldNames.isEmpty() + ? Collections.emptyList() + : Arrays.stream(rawFieldNames.split(",")) + .map(String::trim) + .filter(s -> !s.isEmpty()) + .collect(Collectors.toList()); + return ScanMetricsRecord.builder() + .reportId(getReportId()) + .catalogId(getCatalogId()) + .tableId(getTableId()) + .timestamp(Instant.ofEpochMilli(getTimestampMs())) + .metadata(MetricsModelUtils.parseMetadata(getMetadata())) + .principalName(getPrincipalName()) + .requestId(getRequestId()) + .otelTraceId(getOtelTraceId()) + .otelSpanId(getOtelSpanId()) + .snapshotId(Optional.ofNullable(getSnapshotId())) + .schemaId(Optional.ofNullable(getSchemaId())) + .filterExpression(Optional.ofNullable(getFilterExpression())) + .projectedFieldIds(fieldIds) + .projectedFieldNames(fieldNames) + .resultDataFiles(getResultDataFiles()) + .resultDeleteFiles(getResultDeleteFiles()) + .totalFileSizeBytes(getTotalFileSizeBytes()) + .totalDataManifests(getTotalDataManifests()) + .totalDeleteManifests(getTotalDeleteManifests()) + .scannedDataManifests(getScannedDataManifests()) + .scannedDeleteManifests(getScannedDeleteManifests()) + .skippedDataManifests(getSkippedDataManifests()) + .skippedDeleteManifests(getSkippedDeleteManifests()) + .skippedDataFiles(getSkippedDataFiles()) + .skippedDeleteFiles(getSkippedDeleteFiles()) + .totalPlanningDurationMs(getTotalPlanningDurationMs()) + .equalityDeleteFiles(getEqualityDeleteFiles()) + .positionalDeleteFiles(getPositionalDeleteFiles()) + .indexedDeleteFiles(getIndexedDeleteFiles()) + .totalDeleteFileSizeBytes(getTotalDeleteFileSizeBytes()) + .build(); } - /** Dummy instance to be used as a Converter when calling fromResultSet(). */ ModelScanMetricsReport CONVERTER = ImmutableModelScanMetricsReport.builder() .reportId("") @@ -348,4 +370,18 @@ private static String toJsonString(Map map) { .indexedDeleteFiles(0L) .totalDeleteFileSizeBytes(0L) .build(); + + private static String toCommaSeparated(List list) { + if (list == null || list.isEmpty()) return null; + return list.stream().map(Object::toString).collect(Collectors.joining(",")); + } + + private static String toJsonString(Map map) { + if (map == null || map.isEmpty()) return "{}"; + try { + return MetricsModelUtils.OBJECT_MAPPER.writeValueAsString(map); + } catch (JsonProcessingException e) { + return "{}"; + } + } } diff --git a/extensions/metrics-reports/persistence/relational-jdbc/src/main/java/org/apache/polaris/persistence/relational/jdbc/pagination/MetricsReportToken.java b/extensions/metrics-reports/persistence/relational-jdbc/src/main/java/org/apache/polaris/persistence/relational/jdbc/pagination/MetricsReportToken.java new file mode 100644 index 00000000000..1e794c83f42 --- /dev/null +++ b/extensions/metrics-reports/persistence/relational-jdbc/src/main/java/org/apache/polaris/persistence/relational/jdbc/pagination/MetricsReportToken.java @@ -0,0 +1,73 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.polaris.persistence.relational.jdbc.pagination; + +import com.fasterxml.jackson.annotation.JsonProperty; +import com.fasterxml.jackson.databind.annotation.JsonDeserialize; +import com.fasterxml.jackson.databind.annotation.JsonSerialize; +import jakarta.annotation.Nullable; +import org.apache.polaris.core.persistence.metrics.MetricsRecordIdentity; +import org.apache.polaris.core.persistence.pagination.Token; +import org.apache.polaris.immutables.PolarisImmutable; + +/** + * Pagination {@linkplain Token token} for metrics reports, backed by {@code (timestamp_ms, + * report_id)}. + * + *

Metrics reports are sorted by {@code (timestamp_ms DESC, report_id DESC)}. The cursor encodes + * the last-seen pair, enabling stable keyset pagination under concurrent inserts. + */ +@PolarisImmutable +@JsonSerialize(as = ImmutableMetricsReportToken.class) +@JsonDeserialize(as = ImmutableMetricsReportToken.class) +public interface MetricsReportToken extends Token { + // Registered token type IDs: "e" = EntityIdToken, "m" = MetricsReportToken + String ID = "m"; + + @JsonProperty("ts") + long timestampMs(); + + @JsonProperty("id") + String reportId(); + + @Override + default String getT() { + return ID; + } + + static @Nullable MetricsReportToken fromRecord(@Nullable MetricsRecordIdentity record) { + if (record == null) return null; + return ImmutableMetricsReportToken.builder() + .timestampMs(record.timestamp().toEpochMilli()) + .reportId(record.reportId()) + .build(); + } + + final class MetricsReportTokenType implements TokenType { + @Override + public String id() { + return ID; + } + + @Override + public Class javaType() { + return MetricsReportToken.class; + } + } +} diff --git a/extensions/metrics-reports/persistence/relational-jdbc/src/main/resources/META-INF/services/org.apache.polaris.core.persistence.pagination.Token$TokenType b/extensions/metrics-reports/persistence/relational-jdbc/src/main/resources/META-INF/services/org.apache.polaris.core.persistence.pagination.Token$TokenType new file mode 100644 index 00000000000..05b16a4e39e --- /dev/null +++ b/extensions/metrics-reports/persistence/relational-jdbc/src/main/resources/META-INF/services/org.apache.polaris.core.persistence.pagination.Token$TokenType @@ -0,0 +1,20 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# + +org.apache.polaris.persistence.relational.jdbc.pagination.MetricsReportToken$MetricsReportTokenType diff --git a/persistence/relational-jdbc/src/test/java/org/apache/polaris/persistence/relational/jdbc/SpiModelConverterTest.java b/extensions/metrics-reports/persistence/relational-jdbc/src/test/java/org/apache/polaris/persistence/relational/jdbc/SpiModelConverterTest.java similarity index 74% rename from persistence/relational-jdbc/src/test/java/org/apache/polaris/persistence/relational/jdbc/SpiModelConverterTest.java rename to extensions/metrics-reports/persistence/relational-jdbc/src/test/java/org/apache/polaris/persistence/relational/jdbc/SpiModelConverterTest.java index 4c889ddd291..27b30d7e7c2 100644 --- a/persistence/relational-jdbc/src/test/java/org/apache/polaris/persistence/relational/jdbc/SpiModelConverterTest.java +++ b/extensions/metrics-reports/persistence/relational-jdbc/src/test/java/org/apache/polaris/persistence/relational/jdbc/SpiModelConverterTest.java @@ -29,8 +29,8 @@ import org.apache.polaris.persistence.relational.jdbc.models.ModelScanMetricsReport; import org.junit.jupiter.api.Test; -/** Unit tests for metrics model conversion methods (SPI record -> JDBC model). */ -public class SpiModelConverterTest { +/** Unit tests for metrics model conversion methods (SPI record ↔ JDBC model). */ +class SpiModelConverterTest { private static final String TEST_REPORT_ID = "report-123"; private static final String TEST_REALM_ID = "realm-1"; @@ -42,7 +42,6 @@ public class SpiModelConverterTest { @Test void testFromScanRecord() { ScanMetricsRecord record = createTestScanRecord(); - ModelScanMetricsReport model = ModelScanMetricsReport.fromRecord(record, TEST_REALM_ID); assertThat(model.getReportId()).isEqualTo(TEST_REPORT_ID); @@ -57,91 +56,50 @@ void testFromScanRecord() { assertThat(model.getProjectedFieldNames()).isEqualTo("id,name,value"); assertThat(model.getResultDataFiles()).isEqualTo(10L); assertThat(model.getResultDeleteFiles()).isEqualTo(2L); - assertThat(model.getTotalFileSizeBytes()).isEqualTo(1024000L); assertThat(model.getMetadata()).isEqualTo("{\"custom\":\"value\"}"); } @Test void testFromCommitRecord() { CommitMetricsRecord record = createTestCommitRecord(); - ModelCommitMetricsReport model = ModelCommitMetricsReport.fromRecord(record, TEST_REALM_ID); assertThat(model.getReportId()).isEqualTo(TEST_REPORT_ID); assertThat(model.getRealmId()).isEqualTo(TEST_REALM_ID); assertThat(model.getCatalogId()).isEqualTo(TEST_CATALOG_ID); assertThat(model.getTableId()).isEqualTo(TEST_TABLE_ID); - assertThat(model.getTimestampMs()).isEqualTo(TEST_TIMESTAMP_MS); assertThat(model.getSnapshotId()).isEqualTo(987654321L); assertThat(model.getSequenceNumber()).isEqualTo(5L); assertThat(model.getOperation()).isEqualTo("append"); assertThat(model.getAddedDataFiles()).isEqualTo(10L); - assertThat(model.getRemovedDataFiles()).isEqualTo(2L); - assertThat(model.getTotalDataFiles()).isEqualTo(100L); assertThat(model.getAttempts()).isEqualTo(1); } @Test - void testNullOptionalFields() { - ScanMetricsRecord record = - ScanMetricsRecord.builder() - .reportId(TEST_REPORT_ID) - .catalogId(TEST_CATALOG_ID) - .tableId(TEST_TABLE_ID) - .timestamp(TEST_TIMESTAMP) - .resultDataFiles(0L) - .resultDeleteFiles(0L) - .totalFileSizeBytes(0L) - .totalDataManifests(0L) - .totalDeleteManifests(0L) - .scannedDataManifests(0L) - .scannedDeleteManifests(0L) - .skippedDataManifests(0L) - .skippedDeleteManifests(0L) - .skippedDataFiles(0L) - .skippedDeleteFiles(0L) - .totalPlanningDurationMs(0L) - .equalityDeleteFiles(0L) - .positionalDeleteFiles(0L) - .indexedDeleteFiles(0L) - .totalDeleteFileSizeBytes(0L) - .build(); + void scanRoundTrip() { + ScanMetricsRecord original = createTestScanRecord(); + ModelScanMetricsReport model = ModelScanMetricsReport.fromRecord(original, TEST_REALM_ID); + ScanMetricsRecord restored = model.toRecord(); + + assertThat(restored.reportId()).isEqualTo(original.reportId()); + assertThat(restored.catalogId()).isEqualTo(original.catalogId()); + assertThat(restored.tableId()).isEqualTo(original.tableId()); + assertThat(restored.snapshotId()).isEqualTo(original.snapshotId()); + assertThat(restored.projectedFieldIds()).isEqualTo(original.projectedFieldIds()); + assertThat(restored.projectedFieldNames()).isEqualTo(original.projectedFieldNames()); + assertThat(restored.resultDataFiles()).isEqualTo(original.resultDataFiles()); + } + @Test + void nullOptionalFields() { + ScanMetricsRecord record = minimalScanRecord(); ModelScanMetricsReport model = ModelScanMetricsReport.fromRecord(record, TEST_REALM_ID); + assertThat(model.getSnapshotId()).isNull(); assertThat(model.getSchemaId()).isNull(); assertThat(model.getFilterExpression()).isNull(); assertThat(model.getProjectedFieldIds()).isNull(); assertThat(model.getProjectedFieldNames()).isNull(); - } - - @Test - void testEmptyMetadata() { - ScanMetricsRecord record = - ScanMetricsRecord.builder() - .reportId(TEST_REPORT_ID) - .catalogId(TEST_CATALOG_ID) - .tableId(TEST_TABLE_ID) - .timestamp(TEST_TIMESTAMP) - .resultDataFiles(0L) - .resultDeleteFiles(0L) - .totalFileSizeBytes(0L) - .totalDataManifests(0L) - .totalDeleteManifests(0L) - .scannedDataManifests(0L) - .scannedDeleteManifests(0L) - .skippedDataManifests(0L) - .skippedDeleteManifests(0L) - .skippedDataFiles(0L) - .skippedDeleteFiles(0L) - .totalPlanningDurationMs(0L) - .equalityDeleteFiles(0L) - .positionalDeleteFiles(0L) - .indexedDeleteFiles(0L) - .totalDeleteFileSizeBytes(0L) - .build(); - - ModelScanMetricsReport model = ModelScanMetricsReport.fromRecord(record, TEST_REALM_ID); assertThat(model.getMetadata()).isEqualTo("{}"); } @@ -206,4 +164,29 @@ private CommitMetricsRecord createTestCommitRecord() { .metadata(Map.of("custom", "value")) .build(); } + + private ScanMetricsRecord minimalScanRecord() { + return ScanMetricsRecord.builder() + .reportId(TEST_REPORT_ID) + .catalogId(TEST_CATALOG_ID) + .tableId(TEST_TABLE_ID) + .timestamp(TEST_TIMESTAMP) + .resultDataFiles(0L) + .resultDeleteFiles(0L) + .totalFileSizeBytes(0L) + .totalDataManifests(0L) + .totalDeleteManifests(0L) + .scannedDataManifests(0L) + .scannedDeleteManifests(0L) + .skippedDataManifests(0L) + .skippedDeleteManifests(0L) + .skippedDataFiles(0L) + .skippedDeleteFiles(0L) + .totalPlanningDurationMs(0L) + .equalityDeleteFiles(0L) + .positionalDeleteFiles(0L) + .indexedDeleteFiles(0L) + .totalDeleteFileSizeBytes(0L) + .build(); + } } diff --git a/extensions/metrics-reports/spi/build.gradle.kts b/extensions/metrics-reports/spi/build.gradle.kts new file mode 100644 index 00000000000..1f11a45d55f --- /dev/null +++ b/extensions/metrics-reports/spi/build.gradle.kts @@ -0,0 +1,27 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +plugins { + id("polaris-server") +} + +dependencies { + implementation(platform(libs.iceberg.bom)) + implementation("org.apache.iceberg:iceberg-api") +} diff --git a/runtime/service/src/main/java/org/apache/polaris/service/reporting/PolarisMetricsReporter.java b/extensions/metrics-reports/spi/src/main/java/org/apache/polaris/extension/metrics/IcebergMetricsReporter.java similarity index 60% rename from runtime/service/src/main/java/org/apache/polaris/service/reporting/PolarisMetricsReporter.java rename to extensions/metrics-reports/spi/src/main/java/org/apache/polaris/extension/metrics/IcebergMetricsReporter.java index c621be2a5d8..3422731bdd3 100644 --- a/runtime/service/src/main/java/org/apache/polaris/service/reporting/PolarisMetricsReporter.java +++ b/extensions/metrics-reports/spi/src/main/java/org/apache/polaris/extension/metrics/IcebergMetricsReporter.java @@ -16,38 +16,33 @@ * specific language governing permissions and limitations * under the License. */ -package org.apache.polaris.service.reporting; +package org.apache.polaris.extension.metrics; import java.time.Instant; import org.apache.iceberg.catalog.TableIdentifier; import org.apache.iceberg.metrics.MetricsReport; /** - * SPI interface for reporting Iceberg metrics received by Polaris. + * SPI for reporting Iceberg metrics received by Polaris. * - *

Implementations can be used to send metrics to external systems for analysis and monitoring. - * Custom implementations can be annotated with appropriate {@code Quarkus} scope and {@link - * io.smallrye.common.annotation.Identifier @Identifier("my-reporter-type")} for CDI discovery. + *

Implementations receive a resolved table context (catalog name/id and table name/id) along + * with the raw Iceberg {@link MetricsReport}. Custom implementations should be annotated with the + * appropriate CDI scope and {@code @Identifier("my-type")} for selection via the {@code + * polaris.iceberg-metrics.reporting.type} configuration property. * - *

The implementation to use is selected via the {@code polaris.iceberg-metrics.reporting.type} - * configuration property, which defaults to {@code "default"}. - * - *

Implementations can inject other CDI beans for context. - * - * @see DefaultMetricsReporter - * @see MetricsReportingConfiguration + *

This interface is intentionally runtime/framework-agnostic. CDI and configuration concerns + * belong in the implementing class, not here. */ -public interface PolarisMetricsReporter { +public interface IcebergMetricsReporter { /** - * Reports an Iceberg metrics report for a specific table. + * Reports an Iceberg metrics report for a resolved table. * * @param catalogName the name of the catalog containing the table * @param catalogId the internal Polaris ID of the catalog * @param table the identifier of the table the metrics are for * @param tableId the internal Polaris ID of the table entity - * @param metricsReport the Iceberg metrics report (e.g., {@link - * org.apache.iceberg.metrics.ScanReport} or {@link org.apache.iceberg.metrics.CommitReport}) + * @param metricsReport the Iceberg metrics report * @param receivedTimestamp the timestamp when the metrics were received by Polaris */ void reportMetric( diff --git a/gradle/projects.main.properties b/gradle/projects.main.properties index 322510c5578..3e13a2d4186 100644 --- a/gradle/projects.main.properties +++ b/gradle/projects.main.properties @@ -23,6 +23,7 @@ polaris-core=polaris-core polaris-api-iceberg-service=api/iceberg-service polaris-api-management-model=api/management-model polaris-api-management-service=api/management-service +polaris-api-metrics-reports-service=api/metrics-reports-service polaris-api-catalog-service=api/polaris-catalog-service polaris-runtime-defaults=runtime/defaults polaris-runtime-service=runtime/service @@ -49,8 +50,11 @@ polaris-extensions-federation-hive=extensions/federation/hive polaris-extensions-federation-bigquery=extensions/federation/bigquery polaris-extensions-auth-opa=extensions/auth/opa/impl polaris-extensions-auth-opa-tests=extensions/auth/opa/tests +polaris-extensions-metrics-reports-jdbc=extensions/metrics-reports/persistence/relational-jdbc +polaris-extensions-metrics-reports-spi=extensions/metrics-reports/spi polaris-extensions-auth-ranger=extensions/auth/ranger/impl polaris-extensions-auth-ranger-tests=extensions/auth/ranger/tests +polaris-extensions-metrics-reports=extensions/metrics-reports/impl polaris-config-docs-annotations=tools/config-docs/annotations polaris-config-docs-generator=tools/config-docs/generator diff --git a/persistence/relational-jdbc/src/main/java/org/apache/polaris/persistence/relational/jdbc/DatasourceOperations.java b/persistence/relational-jdbc/src/main/java/org/apache/polaris/persistence/relational/jdbc/DatasourceOperations.java index cea7f58b95a..f8870c19dfd 100644 --- a/persistence/relational-jdbc/src/main/java/org/apache/polaris/persistence/relational/jdbc/DatasourceOperations.java +++ b/persistence/relational-jdbc/src/main/java/org/apache/polaris/persistence/relational/jdbc/DatasourceOperations.java @@ -94,7 +94,7 @@ public DatasourceOperations( } } - DatabaseType getDatabaseType() { + public DatabaseType getDatabaseType() { return databaseType; } diff --git a/persistence/relational-jdbc/src/main/java/org/apache/polaris/persistence/relational/jdbc/JdbcBasePersistenceImpl.java b/persistence/relational-jdbc/src/main/java/org/apache/polaris/persistence/relational/jdbc/JdbcBasePersistenceImpl.java index 112ee7a9a82..410d5400898 100644 --- a/persistence/relational-jdbc/src/main/java/org/apache/polaris/persistence/relational/jdbc/JdbcBasePersistenceImpl.java +++ b/persistence/relational-jdbc/src/main/java/org/apache/polaris/persistence/relational/jdbc/JdbcBasePersistenceImpl.java @@ -57,9 +57,6 @@ import org.apache.polaris.core.persistence.PolicyMappingAlreadyExistsException; import org.apache.polaris.core.persistence.PrincipalSecretsGenerator; import org.apache.polaris.core.persistence.RetryOnConcurrencyException; -import org.apache.polaris.core.persistence.metrics.CommitMetricsRecord; -import org.apache.polaris.core.persistence.metrics.MetricsPersistence; -import org.apache.polaris.core.persistence.metrics.ScanMetricsRecord; import org.apache.polaris.core.persistence.pagination.EntityIdToken; import org.apache.polaris.core.persistence.pagination.Page; import org.apache.polaris.core.persistence.pagination.PageToken; @@ -71,21 +68,18 @@ import org.apache.polaris.core.storage.PolarisStorageIntegrationProvider; import org.apache.polaris.core.storage.StorageLocation; import org.apache.polaris.persistence.relational.jdbc.models.EntityNameLookupRecordConverter; -import org.apache.polaris.persistence.relational.jdbc.models.ModelCommitMetricsReport; import org.apache.polaris.persistence.relational.jdbc.models.ModelEntity; import org.apache.polaris.persistence.relational.jdbc.models.ModelEvent; import org.apache.polaris.persistence.relational.jdbc.models.ModelGrantRecord; import org.apache.polaris.persistence.relational.jdbc.models.ModelPolicyMappingRecord; import org.apache.polaris.persistence.relational.jdbc.models.ModelPrincipalAuthenticationData; -import org.apache.polaris.persistence.relational.jdbc.models.ModelScanMetricsReport; import org.apache.polaris.persistence.relational.jdbc.models.SchemaVersion; import org.jspecify.annotations.NonNull; import org.jspecify.annotations.Nullable; import org.slf4j.Logger; import org.slf4j.LoggerFactory; -public class JdbcBasePersistenceImpl - implements BasePersistence, IntegrationPersistence, MetricsPersistence { +public class JdbcBasePersistenceImpl implements BasePersistence, IntegrationPersistence { private static final Logger LOGGER = LoggerFactory.getLogger(JdbcBasePersistenceImpl.class); @@ -1299,59 +1293,4 @@ public void persistStorageIntegrationIfNeeded( private interface QueryAction { Integer apply(Connection connection, QueryGenerator.PreparedQuery query) throws SQLException; } - - // ============================================================================ - // MetricsPersistence Implementation - // ============================================================================ - - /** Returns the datasource operations to use for metrics persistence. */ - private DatasourceOperations getMetricsDatasource() { - return datasourceOperations; - } - - @Override - public void writeScanReport(@NonNull ScanMetricsRecord record) { - ModelScanMetricsReport model = ModelScanMetricsReport.fromRecord(record, realmId); - writeScanMetricsReport(model); - } - - @Override - public void writeCommitReport(@NonNull CommitMetricsRecord record) { - ModelCommitMetricsReport model = ModelCommitMetricsReport.fromRecord(record, realmId); - writeCommitMetricsReport(model); - } - - // ========== Internal Metrics JDBC methods ========== - - private void writeScanMetricsReport(@NonNull ModelScanMetricsReport report) { - DatasourceOperations metricsOps = getMetricsDatasource(); - try { - PreparedQuery pq = - QueryGenerator.generateInsertQuery( - ModelScanMetricsReport.ALL_COLUMNS, - ModelScanMetricsReport.TABLE_NAME, - report.toMap(metricsOps.getDatabaseType()).values().stream().toList(), - realmId); - metricsOps.executeUpdate(pq); - } catch (SQLException e) { - throw new RuntimeException( - String.format("Failed to write scan metrics report due to %s", e.getMessage()), e); - } - } - - private void writeCommitMetricsReport(@NonNull ModelCommitMetricsReport report) { - DatasourceOperations metricsOps = getMetricsDatasource(); - try { - PreparedQuery pq = - QueryGenerator.generateInsertQuery( - ModelCommitMetricsReport.ALL_COLUMNS, - ModelCommitMetricsReport.TABLE_NAME, - report.toMap(metricsOps.getDatabaseType()).values().stream().toList(), - realmId); - metricsOps.executeUpdate(pq); - } catch (SQLException e) { - throw new RuntimeException( - String.format("Failed to write commit metrics report due to %s", e.getMessage()), e); - } - } } diff --git a/persistence/relational-jdbc/src/main/java/org/apache/polaris/persistence/relational/jdbc/JdbcMetaStoreManagerFactory.java b/persistence/relational-jdbc/src/main/java/org/apache/polaris/persistence/relational/jdbc/JdbcMetaStoreManagerFactory.java index 1a4c7b5ba84..d3bd2f3ea63 100644 --- a/persistence/relational-jdbc/src/main/java/org/apache/polaris/persistence/relational/jdbc/JdbcMetaStoreManagerFactory.java +++ b/persistence/relational-jdbc/src/main/java/org/apache/polaris/persistence/relational/jdbc/JdbcMetaStoreManagerFactory.java @@ -198,7 +198,8 @@ public synchronized Map bootstrapRealms( PolarisMetaStoreManager metaStoreManager = createNewMetaStoreManager(); JdbcBasePersistenceImpl metaStore = createSession(realm, bootstrapOptions.rootCredentialsSet(), true); - PolarisCallContext polarisContext = new PolarisCallContext(realmContext, metaStore); + PolarisCallContext polarisContext = + new PolarisCallContext(realmContext, metaStore, new MetricsPersistence() {}); PrincipalSecretsResult secretsResult = createPolarisPrincipalForRealm(metaStoreManager, polarisContext); @@ -219,7 +220,8 @@ public synchronized Map purgeRealms(Iterable realms) PolarisMetaStoreManager metaStoreManager = createNewMetaStoreManager(); JdbcBasePersistenceImpl session = createSession(realm, null, true); - PolarisCallContext callContext = new PolarisCallContext(realmContext, session); + PolarisCallContext callContext = + new PolarisCallContext(realmContext, session, new MetricsPersistence() {}); // Verify the realm is bootstrapped before purging — a non-bootstrapped realm // has no root principal, so purging it is a no-op that should be reported as failure. @@ -255,7 +257,7 @@ public BasePersistence getOrCreateSession(RealmContext realmContext) { @Override public MetricsPersistence getOrCreateMetricsPersistence(RealmContext realmContext) { - return createJdbcPersistence(realmContext); + return new MetricsPersistence() {}; } @Override @@ -283,7 +285,8 @@ private void checkPolarisServiceBootstrappedForRealm( String realmId = realmContext.getRealmIdentifier(); PolarisMetaStoreManager metaStoreManager = createNewMetaStoreManager(); JdbcBasePersistenceImpl metaStore = createSession(realmId, null, fallbackOnDne); - PolarisCallContext polarisContext = new PolarisCallContext(realmContext, metaStore); + PolarisCallContext polarisContext = + new PolarisCallContext(realmContext, metaStore, new MetricsPersistence() {}); Optional rootPrincipal = metaStoreManager.findRootPrincipal(polarisContext); if (rootPrincipal.isEmpty()) { diff --git a/persistence/relational-jdbc/src/main/java/org/apache/polaris/persistence/relational/jdbc/QueryGenerator.java b/persistence/relational-jdbc/src/main/java/org/apache/polaris/persistence/relational/jdbc/QueryGenerator.java index 033835f644c..a2669f29b59 100644 --- a/persistence/relational-jdbc/src/main/java/org/apache/polaris/persistence/relational/jdbc/QueryGenerator.java +++ b/persistence/relational-jdbc/src/main/java/org/apache/polaris/persistence/relational/jdbc/QueryGenerator.java @@ -422,7 +422,7 @@ public static PreparedQuery generateOverlapQuery( return new PreparedQuery(query.sql(), where.parameters()); } - static String getFullyQualifiedTableName(String tableName) { + public static String getFullyQualifiedTableName(String tableName) { // TODO: make schema name configurable. return "POLARIS_SCHEMA." + tableName; } diff --git a/persistence/relational-jdbc/src/test/java/org/apache/polaris/persistence/relational/jdbc/AtomicMetastoreManagerWithJdbcBasePersistenceImplTest.java b/persistence/relational-jdbc/src/test/java/org/apache/polaris/persistence/relational/jdbc/AtomicMetastoreManagerWithJdbcBasePersistenceImplTest.java index 3a78ca39ba5..74a879351b7 100644 --- a/persistence/relational-jdbc/src/test/java/org/apache/polaris/persistence/relational/jdbc/AtomicMetastoreManagerWithJdbcBasePersistenceImplTest.java +++ b/persistence/relational-jdbc/src/test/java/org/apache/polaris/persistence/relational/jdbc/AtomicMetastoreManagerWithJdbcBasePersistenceImplTest.java @@ -40,6 +40,7 @@ import org.apache.polaris.core.persistence.AtomicOperationMetaStoreManager; import org.apache.polaris.core.persistence.BasePolarisMetaStoreManagerTest; import org.apache.polaris.core.persistence.PolarisTestMetaStoreManager; +import org.apache.polaris.core.persistence.metrics.MetricsPersistence; import org.assertj.core.api.Assertions; import org.assertj.core.api.Assumptions; import org.h2.jdbcx.JdbcConnectionPool; @@ -106,7 +107,8 @@ protected PolarisTestMetaStoreManager createPolarisTestMetaStoreManager() { schemaVersion()); AtomicOperationMetaStoreManager metaStoreManager = new AtomicOperationMetaStoreManager(clock, diagServices); - PolarisCallContext callCtx = new PolarisCallContext(realmContext, basePersistence); + PolarisCallContext callCtx = + new PolarisCallContext(realmContext, basePersistence, new MetricsPersistence() {}); return new PolarisTestMetaStoreManager(metaStoreManager, callCtx); } diff --git a/persistence/relational-jdbc/src/test/java/org/apache/polaris/persistence/relational/jdbc/JdbcGrantRecordsIdempotencyTest.java b/persistence/relational-jdbc/src/test/java/org/apache/polaris/persistence/relational/jdbc/JdbcGrantRecordsIdempotencyTest.java index 41e9182296c..f73310fe3cf 100644 --- a/persistence/relational-jdbc/src/test/java/org/apache/polaris/persistence/relational/jdbc/JdbcGrantRecordsIdempotencyTest.java +++ b/persistence/relational-jdbc/src/test/java/org/apache/polaris/persistence/relational/jdbc/JdbcGrantRecordsIdempotencyTest.java @@ -35,6 +35,7 @@ import org.apache.polaris.core.PolarisDefaultDiagServiceImpl; import org.apache.polaris.core.context.RealmContext; import org.apache.polaris.core.entity.PolarisGrantRecord; +import org.apache.polaris.core.persistence.metrics.MetricsPersistence; import org.apache.polaris.core.storage.PolarisStorageIntegrationProvider; import org.h2.jdbcx.JdbcConnectionPool; import org.junit.jupiter.params.ParameterizedTest; @@ -79,7 +80,8 @@ void writeToGrantRecordsIsIdempotent(int schemaVersion) throws SQLException { mock(PolarisStorageIntegrationProvider.class), REALM_CONTEXT.getRealmIdentifier(), schemaVersion); - PolarisCallContext callCtx = new PolarisCallContext(REALM_CONTEXT, basePersistence); + PolarisCallContext callCtx = + new PolarisCallContext(REALM_CONTEXT, basePersistence, new MetricsPersistence() {}); PolarisGrantRecord grant = new PolarisGrantRecord( @@ -122,7 +124,8 @@ void writeToGrantRecords_IdempotencyCheck_OnlyCatchesPKUniqueness(String sqlStat mock(PolarisStorageIntegrationProvider.class), REALM_CONTEXT.getRealmIdentifier(), schemaVersion); - PolarisCallContext callCtx = new PolarisCallContext(REALM_CONTEXT, basePersistence); + PolarisCallContext callCtx = + new PolarisCallContext(REALM_CONTEXT, basePersistence, new MetricsPersistence() {}); PolarisGrantRecord grant = new PolarisGrantRecord( diff --git a/persistence/relational-jdbc/src/test/java/org/apache/polaris/persistence/relational/jdbc/MetricsReportPersistenceTest.java b/persistence/relational-jdbc/src/test/java/org/apache/polaris/persistence/relational/jdbc/MetricsReportPersistenceTest.java deleted file mode 100644 index ec016b8b6f4..00000000000 --- a/persistence/relational-jdbc/src/test/java/org/apache/polaris/persistence/relational/jdbc/MetricsReportPersistenceTest.java +++ /dev/null @@ -1,206 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ -package org.apache.polaris.persistence.relational.jdbc; - -import java.io.InputStream; -import java.sql.SQLException; -import java.time.Instant; -import java.util.List; -import java.util.Optional; -import java.util.UUID; -import javax.sql.DataSource; -import org.apache.polaris.core.PolarisDefaultDiagServiceImpl; -import org.apache.polaris.core.PolarisDiagnostics; -import org.apache.polaris.core.entity.PolarisEntity; -import org.apache.polaris.core.persistence.PrincipalSecretsGenerator; -import org.apache.polaris.core.persistence.metrics.CommitMetricsRecord; -import org.apache.polaris.core.persistence.metrics.ScanMetricsRecord; -import org.apache.polaris.core.storage.PolarisStorageIntegration; -import org.apache.polaris.core.storage.PolarisStorageIntegrationProvider; -import org.h2.jdbcx.JdbcConnectionPool; -import org.junit.jupiter.api.BeforeEach; -import org.junit.jupiter.api.Test; - -/** - * Integration tests for metrics report persistence using JdbcBasePersistenceImpl. Tests the - * SPI-level write operations for scan and commit metrics reports. - */ -class MetricsReportPersistenceTest { - - private JdbcBasePersistenceImpl metricsPersistence; - private DataSource dataSource; - - @BeforeEach - void setUp() throws SQLException { - dataSource = - JdbcConnectionPool.create( - "jdbc:h2:mem:test_metrics_" + UUID.randomUUID() + ";DB_CLOSE_DELAY=-1", "sa", ""); - - DatasourceOperations datasourceOperations = - new DatasourceOperations(dataSource, new TestJdbcConfiguration()); - - // Execute main schema v4 (includes metrics tables) - ClassLoader classLoader = DatasourceOperations.class.getClassLoader(); - InputStream schemaStream = classLoader.getResourceAsStream("h2/schema-v4.sql"); - datasourceOperations.executeScript(schemaStream); - - PolarisDiagnostics diagnostics = new PolarisDefaultDiagServiceImpl(); - PolarisStorageIntegrationProvider storageProvider = - new PolarisStorageIntegrationProvider() { - @Override - public PolarisStorageIntegration getStorageIntegration( - List resolvedEntityPath) { - return null; - } - }; - - metricsPersistence = - new JdbcBasePersistenceImpl( - diagnostics, - datasourceOperations, - PrincipalSecretsGenerator.RANDOM_SECRETS, - storageProvider, - "TEST_REALM", - 4); - } - - @Test - void testWriteScanReport() { - ScanMetricsRecord record = - ScanMetricsRecord.builder() - .reportId(UUID.randomUUID().toString()) - .catalogId(12345L) - .tableId(67890L) - .timestamp(Instant.now()) - .principalName("test-user") - .requestId("req-123") - .otelTraceId("trace-abc") - .otelSpanId("span-xyz") - .resultDataFiles(10L) - .resultDeleteFiles(2L) - .totalFileSizeBytes(1024000L) - .totalDataManifests(5L) - .totalDeleteManifests(1L) - .scannedDataManifests(3L) - .scannedDeleteManifests(1L) - .skippedDataManifests(2L) - .skippedDeleteManifests(0L) - .skippedDataFiles(5L) - .skippedDeleteFiles(0L) - .totalPlanningDurationMs(150L) - .equalityDeleteFiles(1L) - .positionalDeleteFiles(1L) - .indexedDeleteFiles(0L) - .totalDeleteFileSizeBytes(10240L) - .build(); - - // Should not throw - uses SPI method - metricsPersistence.writeScanReport(record); - } - - @Test - void testWriteCommitReport() { - CommitMetricsRecord record = - CommitMetricsRecord.builder() - .reportId(UUID.randomUUID().toString()) - .catalogId(12345L) - .tableId(67890L) - .timestamp(Instant.now()) - .principalName("test-user") - .requestId("req-456") - .otelTraceId("trace-def") - .otelSpanId("span-uvw") - .snapshotId(12345L) - .operation("append") - .addedDataFiles(5L) - .removedDataFiles(0L) - .totalDataFiles(100L) - .addedDeleteFiles(0L) - .removedDeleteFiles(0L) - .totalDeleteFiles(2L) - .addedEqualityDeleteFiles(0L) - .removedEqualityDeleteFiles(0L) - .addedPositionalDeleteFiles(0L) - .removedPositionalDeleteFiles(0L) - .addedRecords(1000L) - .removedRecords(0L) - .totalRecords(50000L) - .addedFileSizeBytes(102400L) - .removedFileSizeBytes(0L) - .totalFileSizeBytes(5120000L) - .attempts(1) - .build(); - - // Should not throw - uses SPI method - metricsPersistence.writeCommitReport(record); - } - - @Test - void testWriteMultipleScanReports() { - for (int i = 0; i < 10; i++) { - ScanMetricsRecord record = - ScanMetricsRecord.builder() - .reportId(UUID.randomUUID().toString()) - .catalogId(12345L) - .tableId(100L + i) - .timestamp(Instant.now()) - .resultDataFiles((long) (i * 10)) - .resultDeleteFiles(0L) - .totalFileSizeBytes((long) (i * 1024)) - .totalDataManifests(1L) - .totalDeleteManifests(0L) - .scannedDataManifests(1L) - .scannedDeleteManifests(0L) - .skippedDataManifests(0L) - .skippedDeleteManifests(0L) - .skippedDataFiles(0L) - .skippedDeleteFiles(0L) - .totalPlanningDurationMs((long) (i * 10)) - .equalityDeleteFiles(0L) - .positionalDeleteFiles(0L) - .indexedDeleteFiles(0L) - .totalDeleteFileSizeBytes(0L) - .build(); - - metricsPersistence.writeScanReport(record); - } - } - - private static class TestJdbcConfiguration implements RelationalJdbcConfiguration { - @Override - public Optional maxRetries() { - return Optional.of(1); - } - - @Override - public Optional maxDurationInMs() { - return Optional.of(100L); - } - - @Override - public Optional initialDelayInMs() { - return Optional.of(10L); - } - - @Override - public Optional databaseType() { - return Optional.empty(); - } - } -} diff --git a/persistence/relational-jdbc/src/test/java/org/apache/polaris/persistence/relational/jdbc/models/ModelCommitMetricsReportTest.java b/persistence/relational-jdbc/src/test/java/org/apache/polaris/persistence/relational/jdbc/models/ModelCommitMetricsReportTest.java deleted file mode 100644 index bdbb540853b..00000000000 --- a/persistence/relational-jdbc/src/test/java/org/apache/polaris/persistence/relational/jdbc/models/ModelCommitMetricsReportTest.java +++ /dev/null @@ -1,276 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ -package org.apache.polaris.persistence.relational.jdbc.models; - -import static org.junit.jupiter.api.Assertions.assertEquals; -import static org.junit.jupiter.api.Assertions.assertNull; -import static org.mockito.Mockito.mock; -import static org.mockito.Mockito.when; - -import java.sql.ResultSet; -import java.sql.SQLException; -import java.util.Map; -import org.apache.polaris.persistence.relational.jdbc.DatabaseType; -import org.junit.jupiter.api.Test; -import org.postgresql.util.PGobject; - -public class ModelCommitMetricsReportTest { - - private static final String TEST_REPORT_ID = "commit-report-123"; - private static final String TEST_REALM_ID = "realm-1"; - private static final long TEST_CATALOG_ID = 12345L; - private static final long TEST_TABLE_ID = 67890L; - private static final long TEST_TIMESTAMP_MS = 1704067200000L; - private static final String TEST_PRINCIPAL = "user@example.com"; - private static final String TEST_REQUEST_ID = "req-456"; - private static final String TEST_OTEL_TRACE_ID = "trace-789"; - private static final String TEST_OTEL_SPAN_ID = "span-012"; - private static final String TEST_REPORT_TRACE_ID = "report-trace-345"; - private static final long TEST_SNAPSHOT_ID = 987654321L; - private static final Long TEST_SEQUENCE_NUMBER = 5L; - private static final String TEST_OPERATION = "append"; - private static final long TEST_ADDED_DATA_FILES = 10L; - private static final long TEST_REMOVED_DATA_FILES = 2L; - private static final long TEST_TOTAL_DATA_FILES = 50L; - private static final long TEST_ADDED_DELETE_FILES = 1L; - private static final long TEST_REMOVED_DELETE_FILES = 0L; - private static final long TEST_TOTAL_DELETE_FILES = 3L; - private static final long TEST_ADDED_EQUALITY_DELETE_FILES = 1L; - private static final long TEST_REMOVED_EQUALITY_DELETE_FILES = 0L; - private static final long TEST_ADDED_POSITIONAL_DELETE_FILES = 0L; - private static final long TEST_REMOVED_POSITIONAL_DELETE_FILES = 0L; - private static final long TEST_ADDED_RECORDS = 1000L; - private static final long TEST_REMOVED_RECORDS = 50L; - private static final long TEST_TOTAL_RECORDS = 10000L; - private static final long TEST_ADDED_FILE_SIZE = 1024000L; - private static final long TEST_REMOVED_FILE_SIZE = 51200L; - private static final long TEST_TOTAL_FILE_SIZE = 10240000L; - private static final long TEST_TOTAL_DURATION = 250L; - private static final int TEST_ATTEMPTS = 1; - private static final String TEST_METADATA = "{\"commit\":\"info\"}"; - - @Test - public void testFromResultSet() throws SQLException { - ResultSet mockResultSet = mock(ResultSet.class); - when(mockResultSet.getString(ModelCommitMetricsReport.REPORT_ID)).thenReturn(TEST_REPORT_ID); - when(mockResultSet.getString(ModelCommitMetricsReport.REALM_ID)).thenReturn(TEST_REALM_ID); - when(mockResultSet.getLong(ModelCommitMetricsReport.CATALOG_ID)).thenReturn(TEST_CATALOG_ID); - when(mockResultSet.getLong(ModelCommitMetricsReport.TABLE_ID_COL)).thenReturn(TEST_TABLE_ID); - when(mockResultSet.getLong(ModelCommitMetricsReport.TIMESTAMP_MS)) - .thenReturn(TEST_TIMESTAMP_MS); - when(mockResultSet.getString(ModelCommitMetricsReport.PRINCIPAL_NAME)) - .thenReturn(TEST_PRINCIPAL); - when(mockResultSet.getString(ModelCommitMetricsReport.REQUEST_ID)).thenReturn(TEST_REQUEST_ID); - when(mockResultSet.getString(ModelCommitMetricsReport.OTEL_TRACE_ID)) - .thenReturn(TEST_OTEL_TRACE_ID); - when(mockResultSet.getString(ModelCommitMetricsReport.OTEL_SPAN_ID)) - .thenReturn(TEST_OTEL_SPAN_ID); - when(mockResultSet.getString(ModelCommitMetricsReport.REPORT_TRACE_ID)) - .thenReturn(TEST_REPORT_TRACE_ID); - when(mockResultSet.getLong(ModelCommitMetricsReport.SNAPSHOT_ID)).thenReturn(TEST_SNAPSHOT_ID); - when(mockResultSet.getObject(ModelCommitMetricsReport.SEQUENCE_NUMBER, Long.class)) - .thenReturn(TEST_SEQUENCE_NUMBER); - when(mockResultSet.getString(ModelCommitMetricsReport.OPERATION)).thenReturn(TEST_OPERATION); - when(mockResultSet.getLong(ModelCommitMetricsReport.ADDED_DATA_FILES)) - .thenReturn(TEST_ADDED_DATA_FILES); - when(mockResultSet.getLong(ModelCommitMetricsReport.REMOVED_DATA_FILES)) - .thenReturn(TEST_REMOVED_DATA_FILES); - when(mockResultSet.getLong(ModelCommitMetricsReport.TOTAL_DATA_FILES)) - .thenReturn(TEST_TOTAL_DATA_FILES); - when(mockResultSet.getLong(ModelCommitMetricsReport.ADDED_DELETE_FILES)) - .thenReturn(TEST_ADDED_DELETE_FILES); - when(mockResultSet.getLong(ModelCommitMetricsReport.REMOVED_DELETE_FILES)) - .thenReturn(TEST_REMOVED_DELETE_FILES); - when(mockResultSet.getLong(ModelCommitMetricsReport.TOTAL_DELETE_FILES)) - .thenReturn(TEST_TOTAL_DELETE_FILES); - when(mockResultSet.getLong(ModelCommitMetricsReport.ADDED_EQUALITY_DELETE_FILES)) - .thenReturn(TEST_ADDED_EQUALITY_DELETE_FILES); - when(mockResultSet.getLong(ModelCommitMetricsReport.REMOVED_EQUALITY_DELETE_FILES)) - .thenReturn(TEST_REMOVED_EQUALITY_DELETE_FILES); - when(mockResultSet.getLong(ModelCommitMetricsReport.ADDED_POSITIONAL_DELETE_FILES)) - .thenReturn(TEST_ADDED_POSITIONAL_DELETE_FILES); - when(mockResultSet.getLong(ModelCommitMetricsReport.REMOVED_POSITIONAL_DELETE_FILES)) - .thenReturn(TEST_REMOVED_POSITIONAL_DELETE_FILES); - when(mockResultSet.getLong(ModelCommitMetricsReport.ADDED_RECORDS)) - .thenReturn(TEST_ADDED_RECORDS); - when(mockResultSet.getLong(ModelCommitMetricsReport.REMOVED_RECORDS)) - .thenReturn(TEST_REMOVED_RECORDS); - when(mockResultSet.getLong(ModelCommitMetricsReport.TOTAL_RECORDS)) - .thenReturn(TEST_TOTAL_RECORDS); - when(mockResultSet.getLong(ModelCommitMetricsReport.ADDED_FILE_SIZE_BYTES)) - .thenReturn(TEST_ADDED_FILE_SIZE); - when(mockResultSet.getLong(ModelCommitMetricsReport.REMOVED_FILE_SIZE_BYTES)) - .thenReturn(TEST_REMOVED_FILE_SIZE); - when(mockResultSet.getLong(ModelCommitMetricsReport.TOTAL_FILE_SIZE_BYTES)) - .thenReturn(TEST_TOTAL_FILE_SIZE); - when(mockResultSet.getLong(ModelCommitMetricsReport.TOTAL_DURATION_MS)) - .thenReturn(TEST_TOTAL_DURATION); - when(mockResultSet.getInt(ModelCommitMetricsReport.ATTEMPTS)).thenReturn(TEST_ATTEMPTS); - when(mockResultSet.getString(ModelCommitMetricsReport.METADATA)).thenReturn(TEST_METADATA); - - ModelCommitMetricsReport result = - ModelCommitMetricsReport.CONVERTER.fromResultSet(mockResultSet); - - assertEquals(TEST_REPORT_ID, result.getReportId()); - assertEquals(TEST_REALM_ID, result.getRealmId()); - assertEquals(TEST_CATALOG_ID, result.getCatalogId()); - assertEquals(TEST_TABLE_ID, result.getTableId()); - assertEquals(TEST_TIMESTAMP_MS, result.getTimestampMs()); - assertEquals(TEST_SNAPSHOT_ID, result.getSnapshotId()); - assertEquals(TEST_OPERATION, result.getOperation()); - assertEquals(TEST_ADDED_DATA_FILES, result.getAddedDataFiles()); - assertEquals(TEST_ADDED_RECORDS, result.getAddedRecords()); - assertEquals(TEST_TOTAL_DURATION, result.getTotalDurationMs()); - assertEquals(TEST_ATTEMPTS, result.getAttempts()); - assertEquals(TEST_METADATA, result.getMetadata()); - } - - @Test - public void testToMapWithH2DatabaseType() { - ModelCommitMetricsReport report = createTestReport(); - - Map resultMap = report.toMap(DatabaseType.H2); - - assertEquals(TEST_REPORT_ID, resultMap.get(ModelCommitMetricsReport.REPORT_ID)); - assertEquals(TEST_SNAPSHOT_ID, resultMap.get(ModelCommitMetricsReport.SNAPSHOT_ID)); - assertEquals(TEST_OPERATION, resultMap.get(ModelCommitMetricsReport.OPERATION)); - assertEquals(TEST_ADDED_DATA_FILES, resultMap.get(ModelCommitMetricsReport.ADDED_DATA_FILES)); - assertEquals(TEST_METADATA, resultMap.get(ModelCommitMetricsReport.METADATA)); - // realm_id is not included in toMap() - it's added by the persistence layer - assertNull(resultMap.get(ModelCommitMetricsReport.REALM_ID)); - } - - @Test - public void testToMapWithPostgresType() { - ModelCommitMetricsReport report = createTestReport(); - - Map resultMap = report.toMap(DatabaseType.POSTGRES); - - assertEquals(TEST_REPORT_ID, resultMap.get(ModelCommitMetricsReport.REPORT_ID)); - PGobject pgObject = (PGobject) resultMap.get(ModelCommitMetricsReport.METADATA); - assertEquals("jsonb", pgObject.getType()); - assertEquals(TEST_METADATA, pgObject.getValue()); - } - - @Test - public void testConverterFromResultSet() throws SQLException { - // Test the CONVERTER constant (used in query methods) - ResultSet mockResultSet = mock(ResultSet.class); - when(mockResultSet.getString(ModelCommitMetricsReport.REPORT_ID)).thenReturn(TEST_REPORT_ID); - when(mockResultSet.getString(ModelCommitMetricsReport.REALM_ID)).thenReturn(TEST_REALM_ID); - when(mockResultSet.getLong(ModelCommitMetricsReport.CATALOG_ID)).thenReturn(TEST_CATALOG_ID); - when(mockResultSet.getLong(ModelCommitMetricsReport.TABLE_ID_COL)).thenReturn(TEST_TABLE_ID); - when(mockResultSet.getLong(ModelCommitMetricsReport.TIMESTAMP_MS)) - .thenReturn(TEST_TIMESTAMP_MS); - when(mockResultSet.getString(ModelCommitMetricsReport.PRINCIPAL_NAME)) - .thenReturn(TEST_PRINCIPAL); - when(mockResultSet.getString(ModelCommitMetricsReport.REQUEST_ID)).thenReturn(TEST_REQUEST_ID); - when(mockResultSet.getString(ModelCommitMetricsReport.OTEL_TRACE_ID)) - .thenReturn(TEST_OTEL_TRACE_ID); - when(mockResultSet.getString(ModelCommitMetricsReport.OTEL_SPAN_ID)) - .thenReturn(TEST_OTEL_SPAN_ID); - when(mockResultSet.getString(ModelCommitMetricsReport.REPORT_TRACE_ID)) - .thenReturn(TEST_REPORT_TRACE_ID); - when(mockResultSet.getObject(ModelCommitMetricsReport.SNAPSHOT_ID, Long.class)) - .thenReturn(TEST_SNAPSHOT_ID); - when(mockResultSet.getObject(ModelCommitMetricsReport.SEQUENCE_NUMBER, Long.class)) - .thenReturn(TEST_SEQUENCE_NUMBER); - when(mockResultSet.getString(ModelCommitMetricsReport.OPERATION)).thenReturn(TEST_OPERATION); - when(mockResultSet.getLong(ModelCommitMetricsReport.ADDED_DATA_FILES)) - .thenReturn(TEST_ADDED_DATA_FILES); - when(mockResultSet.getLong(ModelCommitMetricsReport.REMOVED_DATA_FILES)) - .thenReturn(TEST_REMOVED_DATA_FILES); - when(mockResultSet.getLong(ModelCommitMetricsReport.TOTAL_DATA_FILES)) - .thenReturn(TEST_TOTAL_DATA_FILES); - when(mockResultSet.getLong(ModelCommitMetricsReport.ADDED_DELETE_FILES)) - .thenReturn(TEST_ADDED_DELETE_FILES); - when(mockResultSet.getLong(ModelCommitMetricsReport.REMOVED_DELETE_FILES)) - .thenReturn(TEST_REMOVED_DELETE_FILES); - when(mockResultSet.getLong(ModelCommitMetricsReport.TOTAL_DELETE_FILES)) - .thenReturn(TEST_TOTAL_DELETE_FILES); - when(mockResultSet.getLong(ModelCommitMetricsReport.ADDED_EQUALITY_DELETE_FILES)) - .thenReturn(TEST_ADDED_EQUALITY_DELETE_FILES); - when(mockResultSet.getLong(ModelCommitMetricsReport.REMOVED_EQUALITY_DELETE_FILES)) - .thenReturn(TEST_REMOVED_EQUALITY_DELETE_FILES); - when(mockResultSet.getLong(ModelCommitMetricsReport.ADDED_POSITIONAL_DELETE_FILES)) - .thenReturn(TEST_ADDED_POSITIONAL_DELETE_FILES); - when(mockResultSet.getLong(ModelCommitMetricsReport.REMOVED_POSITIONAL_DELETE_FILES)) - .thenReturn(TEST_REMOVED_POSITIONAL_DELETE_FILES); - when(mockResultSet.getLong(ModelCommitMetricsReport.ADDED_RECORDS)) - .thenReturn(TEST_ADDED_RECORDS); - when(mockResultSet.getLong(ModelCommitMetricsReport.REMOVED_RECORDS)) - .thenReturn(TEST_REMOVED_RECORDS); - when(mockResultSet.getLong(ModelCommitMetricsReport.TOTAL_RECORDS)) - .thenReturn(TEST_TOTAL_RECORDS); - when(mockResultSet.getLong(ModelCommitMetricsReport.ADDED_FILE_SIZE_BYTES)) - .thenReturn(TEST_ADDED_FILE_SIZE); - when(mockResultSet.getLong(ModelCommitMetricsReport.REMOVED_FILE_SIZE_BYTES)) - .thenReturn(TEST_REMOVED_FILE_SIZE); - when(mockResultSet.getLong(ModelCommitMetricsReport.TOTAL_FILE_SIZE_BYTES)) - .thenReturn(TEST_TOTAL_FILE_SIZE); - when(mockResultSet.getObject(ModelCommitMetricsReport.TOTAL_DURATION_MS, Long.class)) - .thenReturn(TEST_TOTAL_DURATION); - when(mockResultSet.getObject(ModelCommitMetricsReport.ATTEMPTS, Integer.class)) - .thenReturn(TEST_ATTEMPTS); - when(mockResultSet.getString(ModelCommitMetricsReport.METADATA)).thenReturn(TEST_METADATA); - - ModelCommitMetricsReport result = - ModelCommitMetricsReport.CONVERTER.fromResultSet(mockResultSet); - - assertEquals(TEST_REPORT_ID, result.getReportId()); - assertEquals(TEST_REALM_ID, result.getRealmId()); - assertEquals(TEST_CATALOG_ID, result.getCatalogId()); - assertEquals(TEST_METADATA, result.getMetadata()); - } - - private ModelCommitMetricsReport createTestReport() { - return ImmutableModelCommitMetricsReport.builder() - .reportId(TEST_REPORT_ID) - .realmId(TEST_REALM_ID) - .catalogId(TEST_CATALOG_ID) - .tableId(TEST_TABLE_ID) - .timestampMs(TEST_TIMESTAMP_MS) - .principalName(TEST_PRINCIPAL) - .requestId(TEST_REQUEST_ID) - .otelTraceId(TEST_OTEL_TRACE_ID) - .snapshotId(TEST_SNAPSHOT_ID) - .sequenceNumber(TEST_SEQUENCE_NUMBER) - .operation(TEST_OPERATION) - .addedDataFiles(TEST_ADDED_DATA_FILES) - .removedDataFiles(TEST_REMOVED_DATA_FILES) - .totalDataFiles(TEST_TOTAL_DATA_FILES) - .addedDeleteFiles(TEST_ADDED_DELETE_FILES) - .removedDeleteFiles(TEST_REMOVED_DELETE_FILES) - .totalDeleteFiles(TEST_TOTAL_DELETE_FILES) - .addedEqualityDeleteFiles(TEST_ADDED_EQUALITY_DELETE_FILES) - .removedEqualityDeleteFiles(TEST_REMOVED_EQUALITY_DELETE_FILES) - .addedPositionalDeleteFiles(TEST_ADDED_POSITIONAL_DELETE_FILES) - .removedPositionalDeleteFiles(TEST_REMOVED_POSITIONAL_DELETE_FILES) - .addedRecords(TEST_ADDED_RECORDS) - .removedRecords(TEST_REMOVED_RECORDS) - .totalRecords(TEST_TOTAL_RECORDS) - .addedFileSizeBytes(TEST_ADDED_FILE_SIZE) - .removedFileSizeBytes(TEST_REMOVED_FILE_SIZE) - .totalFileSizeBytes(TEST_TOTAL_FILE_SIZE) - .totalDurationMs(TEST_TOTAL_DURATION) - .attempts(TEST_ATTEMPTS) - .metadata(TEST_METADATA) - .build(); - } -} diff --git a/persistence/relational-jdbc/src/test/java/org/apache/polaris/persistence/relational/jdbc/models/ModelScanMetricsReportTest.java b/persistence/relational-jdbc/src/test/java/org/apache/polaris/persistence/relational/jdbc/models/ModelScanMetricsReportTest.java deleted file mode 100644 index f8068d9e041..00000000000 --- a/persistence/relational-jdbc/src/test/java/org/apache/polaris/persistence/relational/jdbc/models/ModelScanMetricsReportTest.java +++ /dev/null @@ -1,270 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ -package org.apache.polaris.persistence.relational.jdbc.models; - -import static org.junit.jupiter.api.Assertions.assertEquals; -import static org.junit.jupiter.api.Assertions.assertNull; -import static org.mockito.Mockito.mock; -import static org.mockito.Mockito.when; - -import java.sql.ResultSet; -import java.sql.SQLException; -import java.util.Map; -import org.apache.polaris.persistence.relational.jdbc.DatabaseType; -import org.junit.jupiter.api.Test; -import org.postgresql.util.PGobject; - -public class ModelScanMetricsReportTest { - - private static final String TEST_REPORT_ID = "report-123"; - private static final String TEST_REALM_ID = "realm-1"; - private static final long TEST_CATALOG_ID = 12345L; - private static final long TEST_TABLE_ID = 67890L; - private static final long TEST_TIMESTAMP_MS = 1704067200000L; - private static final String TEST_PRINCIPAL = "user@example.com"; - private static final String TEST_REQUEST_ID = "req-456"; - private static final String TEST_OTEL_TRACE_ID = "trace-789"; - private static final String TEST_OTEL_SPAN_ID = "span-012"; - private static final String TEST_REPORT_TRACE_ID = "report-trace-345"; - private static final Long TEST_SNAPSHOT_ID = 123456789L; - private static final Integer TEST_SCHEMA_ID = 1; - private static final String TEST_FILTER = "id > 100"; - private static final String TEST_PROJECTED_IDS = "1,2,3"; - private static final String TEST_PROJECTED_NAMES = "id,name,value"; - private static final long TEST_RESULT_DATA_FILES = 10L; - private static final long TEST_RESULT_DELETE_FILES = 2L; - private static final long TEST_TOTAL_FILE_SIZE = 1024000L; - private static final long TEST_TOTAL_DATA_MANIFESTS = 5L; - private static final long TEST_TOTAL_DELETE_MANIFESTS = 1L; - private static final long TEST_SCANNED_DATA_MANIFESTS = 3L; - private static final long TEST_SCANNED_DELETE_MANIFESTS = 1L; - private static final long TEST_SKIPPED_DATA_MANIFESTS = 2L; - private static final long TEST_SKIPPED_DELETE_MANIFESTS = 0L; - private static final long TEST_SKIPPED_DATA_FILES = 5L; - private static final long TEST_SKIPPED_DELETE_FILES = 0L; - private static final long TEST_PLANNING_DURATION = 150L; - private static final long TEST_EQUALITY_DELETE_FILES = 1L; - private static final long TEST_POSITIONAL_DELETE_FILES = 1L; - private static final long TEST_INDEXED_DELETE_FILES = 0L; - private static final long TEST_DELETE_FILE_SIZE = 2048L; - private static final String TEST_METADATA = "{\"custom\":\"value\"}"; - - @Test - public void testFromResultSet() throws SQLException { - ResultSet mockResultSet = mock(ResultSet.class); - when(mockResultSet.getString(ModelScanMetricsReport.REPORT_ID)).thenReturn(TEST_REPORT_ID); - when(mockResultSet.getString(ModelScanMetricsReport.REALM_ID)).thenReturn(TEST_REALM_ID); - when(mockResultSet.getLong(ModelScanMetricsReport.CATALOG_ID)).thenReturn(TEST_CATALOG_ID); - when(mockResultSet.getLong(ModelScanMetricsReport.TABLE_ID_COL)).thenReturn(TEST_TABLE_ID); - when(mockResultSet.getLong(ModelScanMetricsReport.TIMESTAMP_MS)).thenReturn(TEST_TIMESTAMP_MS); - when(mockResultSet.getString(ModelScanMetricsReport.PRINCIPAL_NAME)).thenReturn(TEST_PRINCIPAL); - when(mockResultSet.getString(ModelScanMetricsReport.REQUEST_ID)).thenReturn(TEST_REQUEST_ID); - when(mockResultSet.getString(ModelScanMetricsReport.OTEL_TRACE_ID)) - .thenReturn(TEST_OTEL_TRACE_ID); - when(mockResultSet.getString(ModelScanMetricsReport.OTEL_SPAN_ID)) - .thenReturn(TEST_OTEL_SPAN_ID); - when(mockResultSet.getString(ModelScanMetricsReport.REPORT_TRACE_ID)) - .thenReturn(TEST_REPORT_TRACE_ID); - when(mockResultSet.getObject(ModelScanMetricsReport.SNAPSHOT_ID, Long.class)) - .thenReturn(TEST_SNAPSHOT_ID); - when(mockResultSet.getObject(ModelScanMetricsReport.SCHEMA_ID, Integer.class)) - .thenReturn(TEST_SCHEMA_ID); - when(mockResultSet.getString(ModelScanMetricsReport.FILTER_EXPRESSION)).thenReturn(TEST_FILTER); - when(mockResultSet.getString(ModelScanMetricsReport.PROJECTED_FIELD_IDS)) - .thenReturn(TEST_PROJECTED_IDS); - when(mockResultSet.getString(ModelScanMetricsReport.PROJECTED_FIELD_NAMES)) - .thenReturn(TEST_PROJECTED_NAMES); - when(mockResultSet.getLong(ModelScanMetricsReport.RESULT_DATA_FILES)) - .thenReturn(TEST_RESULT_DATA_FILES); - when(mockResultSet.getLong(ModelScanMetricsReport.RESULT_DELETE_FILES)) - .thenReturn(TEST_RESULT_DELETE_FILES); - when(mockResultSet.getLong(ModelScanMetricsReport.TOTAL_FILE_SIZE_BYTES)) - .thenReturn(TEST_TOTAL_FILE_SIZE); - when(mockResultSet.getLong(ModelScanMetricsReport.TOTAL_DATA_MANIFESTS)) - .thenReturn(TEST_TOTAL_DATA_MANIFESTS); - when(mockResultSet.getLong(ModelScanMetricsReport.TOTAL_DELETE_MANIFESTS)) - .thenReturn(TEST_TOTAL_DELETE_MANIFESTS); - when(mockResultSet.getLong(ModelScanMetricsReport.SCANNED_DATA_MANIFESTS)) - .thenReturn(TEST_SCANNED_DATA_MANIFESTS); - when(mockResultSet.getLong(ModelScanMetricsReport.SCANNED_DELETE_MANIFESTS)) - .thenReturn(TEST_SCANNED_DELETE_MANIFESTS); - when(mockResultSet.getLong(ModelScanMetricsReport.SKIPPED_DATA_MANIFESTS)) - .thenReturn(TEST_SKIPPED_DATA_MANIFESTS); - when(mockResultSet.getLong(ModelScanMetricsReport.SKIPPED_DELETE_MANIFESTS)) - .thenReturn(TEST_SKIPPED_DELETE_MANIFESTS); - when(mockResultSet.getLong(ModelScanMetricsReport.SKIPPED_DATA_FILES)) - .thenReturn(TEST_SKIPPED_DATA_FILES); - when(mockResultSet.getLong(ModelScanMetricsReport.SKIPPED_DELETE_FILES)) - .thenReturn(TEST_SKIPPED_DELETE_FILES); - when(mockResultSet.getLong(ModelScanMetricsReport.TOTAL_PLANNING_DURATION_MS)) - .thenReturn(TEST_PLANNING_DURATION); - when(mockResultSet.getLong(ModelScanMetricsReport.EQUALITY_DELETE_FILES)) - .thenReturn(TEST_EQUALITY_DELETE_FILES); - when(mockResultSet.getLong(ModelScanMetricsReport.POSITIONAL_DELETE_FILES)) - .thenReturn(TEST_POSITIONAL_DELETE_FILES); - when(mockResultSet.getLong(ModelScanMetricsReport.INDEXED_DELETE_FILES)) - .thenReturn(TEST_INDEXED_DELETE_FILES); - when(mockResultSet.getLong(ModelScanMetricsReport.TOTAL_DELETE_FILE_SIZE_BYTES)) - .thenReturn(TEST_DELETE_FILE_SIZE); - when(mockResultSet.getString(ModelScanMetricsReport.METADATA)).thenReturn(TEST_METADATA); - - ModelScanMetricsReport result = ModelScanMetricsReport.CONVERTER.fromResultSet(mockResultSet); - - assertEquals(TEST_REPORT_ID, result.getReportId()); - assertEquals(TEST_REALM_ID, result.getRealmId()); - assertEquals(TEST_CATALOG_ID, result.getCatalogId()); - assertEquals(TEST_TABLE_ID, result.getTableId()); - assertEquals(TEST_TIMESTAMP_MS, result.getTimestampMs()); - assertEquals(TEST_PRINCIPAL, result.getPrincipalName()); - assertEquals(TEST_REQUEST_ID, result.getRequestId()); - assertEquals(TEST_OTEL_TRACE_ID, result.getOtelTraceId()); - assertEquals(TEST_SNAPSHOT_ID, result.getSnapshotId()); - assertEquals(TEST_RESULT_DATA_FILES, result.getResultDataFiles()); - assertEquals(TEST_TOTAL_FILE_SIZE, result.getTotalFileSizeBytes()); - assertEquals(TEST_PLANNING_DURATION, result.getTotalPlanningDurationMs()); - assertEquals(TEST_METADATA, result.getMetadata()); - } - - @Test - public void testToMapWithH2DatabaseType() { - ModelScanMetricsReport report = createTestReport(); - - Map resultMap = report.toMap(DatabaseType.H2); - - assertEquals(TEST_REPORT_ID, resultMap.get(ModelScanMetricsReport.REPORT_ID)); - assertEquals(TEST_CATALOG_ID, resultMap.get(ModelScanMetricsReport.CATALOG_ID)); - assertEquals(TEST_TABLE_ID, resultMap.get(ModelScanMetricsReport.TABLE_ID_COL)); - assertEquals(TEST_TIMESTAMP_MS, resultMap.get(ModelScanMetricsReport.TIMESTAMP_MS)); - assertEquals(TEST_RESULT_DATA_FILES, resultMap.get(ModelScanMetricsReport.RESULT_DATA_FILES)); - assertEquals(TEST_METADATA, resultMap.get(ModelScanMetricsReport.METADATA)); - // realm_id is not included in toMap() - it's added by the persistence layer - assertNull(resultMap.get(ModelScanMetricsReport.REALM_ID)); - } - - @Test - public void testToMapWithPostgresType() { - ModelScanMetricsReport report = createTestReport(); - - Map resultMap = report.toMap(DatabaseType.POSTGRES); - - assertEquals(TEST_REPORT_ID, resultMap.get(ModelScanMetricsReport.REPORT_ID)); - PGobject pgObject = (PGobject) resultMap.get(ModelScanMetricsReport.METADATA); - assertEquals("jsonb", pgObject.getType()); - assertEquals(TEST_METADATA, pgObject.getValue()); - } - - @Test - public void testConverterFromResultSet() throws SQLException { - // Test the CONVERTER constant (used in query methods) - ResultSet mockResultSet = mock(ResultSet.class); - when(mockResultSet.getString(ModelScanMetricsReport.REPORT_ID)).thenReturn(TEST_REPORT_ID); - when(mockResultSet.getString(ModelScanMetricsReport.REALM_ID)).thenReturn(TEST_REALM_ID); - when(mockResultSet.getLong(ModelScanMetricsReport.CATALOG_ID)).thenReturn(TEST_CATALOG_ID); - when(mockResultSet.getLong(ModelScanMetricsReport.TABLE_ID_COL)).thenReturn(TEST_TABLE_ID); - when(mockResultSet.getLong(ModelScanMetricsReport.TIMESTAMP_MS)).thenReturn(TEST_TIMESTAMP_MS); - when(mockResultSet.getString(ModelScanMetricsReport.PRINCIPAL_NAME)).thenReturn(TEST_PRINCIPAL); - when(mockResultSet.getString(ModelScanMetricsReport.REQUEST_ID)).thenReturn(TEST_REQUEST_ID); - when(mockResultSet.getString(ModelScanMetricsReport.OTEL_TRACE_ID)) - .thenReturn(TEST_OTEL_TRACE_ID); - when(mockResultSet.getString(ModelScanMetricsReport.OTEL_SPAN_ID)) - .thenReturn(TEST_OTEL_SPAN_ID); - when(mockResultSet.getString(ModelScanMetricsReport.REPORT_TRACE_ID)) - .thenReturn(TEST_REPORT_TRACE_ID); - when(mockResultSet.getObject(ModelScanMetricsReport.SNAPSHOT_ID, Long.class)) - .thenReturn(TEST_SNAPSHOT_ID); - when(mockResultSet.getObject(ModelScanMetricsReport.SCHEMA_ID, Integer.class)) - .thenReturn(TEST_SCHEMA_ID); - when(mockResultSet.getString(ModelScanMetricsReport.FILTER_EXPRESSION)).thenReturn(TEST_FILTER); - when(mockResultSet.getString(ModelScanMetricsReport.PROJECTED_FIELD_IDS)) - .thenReturn(TEST_PROJECTED_IDS); - when(mockResultSet.getString(ModelScanMetricsReport.PROJECTED_FIELD_NAMES)) - .thenReturn(TEST_PROJECTED_NAMES); - when(mockResultSet.getLong(ModelScanMetricsReport.RESULT_DATA_FILES)) - .thenReturn(TEST_RESULT_DATA_FILES); - when(mockResultSet.getLong(ModelScanMetricsReport.RESULT_DELETE_FILES)) - .thenReturn(TEST_RESULT_DELETE_FILES); - when(mockResultSet.getLong(ModelScanMetricsReport.TOTAL_FILE_SIZE_BYTES)) - .thenReturn(TEST_TOTAL_FILE_SIZE); - when(mockResultSet.getLong(ModelScanMetricsReport.TOTAL_DATA_MANIFESTS)) - .thenReturn(TEST_TOTAL_DATA_MANIFESTS); - when(mockResultSet.getLong(ModelScanMetricsReport.TOTAL_DELETE_MANIFESTS)) - .thenReturn(TEST_TOTAL_DELETE_MANIFESTS); - when(mockResultSet.getLong(ModelScanMetricsReport.SCANNED_DATA_MANIFESTS)) - .thenReturn(TEST_SCANNED_DATA_MANIFESTS); - when(mockResultSet.getLong(ModelScanMetricsReport.SCANNED_DELETE_MANIFESTS)) - .thenReturn(TEST_SCANNED_DELETE_MANIFESTS); - when(mockResultSet.getLong(ModelScanMetricsReport.SKIPPED_DATA_MANIFESTS)) - .thenReturn(TEST_SKIPPED_DATA_MANIFESTS); - when(mockResultSet.getLong(ModelScanMetricsReport.SKIPPED_DELETE_MANIFESTS)) - .thenReturn(TEST_SKIPPED_DELETE_MANIFESTS); - when(mockResultSet.getLong(ModelScanMetricsReport.SKIPPED_DATA_FILES)) - .thenReturn(TEST_SKIPPED_DATA_FILES); - when(mockResultSet.getLong(ModelScanMetricsReport.SKIPPED_DELETE_FILES)) - .thenReturn(TEST_SKIPPED_DELETE_FILES); - when(mockResultSet.getLong(ModelScanMetricsReport.TOTAL_PLANNING_DURATION_MS)) - .thenReturn(TEST_PLANNING_DURATION); - when(mockResultSet.getLong(ModelScanMetricsReport.EQUALITY_DELETE_FILES)) - .thenReturn(TEST_EQUALITY_DELETE_FILES); - when(mockResultSet.getLong(ModelScanMetricsReport.POSITIONAL_DELETE_FILES)) - .thenReturn(TEST_POSITIONAL_DELETE_FILES); - when(mockResultSet.getLong(ModelScanMetricsReport.INDEXED_DELETE_FILES)) - .thenReturn(TEST_INDEXED_DELETE_FILES); - when(mockResultSet.getLong(ModelScanMetricsReport.TOTAL_DELETE_FILE_SIZE_BYTES)) - .thenReturn(TEST_DELETE_FILE_SIZE); - when(mockResultSet.getString(ModelScanMetricsReport.METADATA)).thenReturn(TEST_METADATA); - - ModelScanMetricsReport result = ModelScanMetricsReport.CONVERTER.fromResultSet(mockResultSet); - - assertEquals(TEST_REPORT_ID, result.getReportId()); - assertEquals(TEST_REALM_ID, result.getRealmId()); - assertEquals(TEST_CATALOG_ID, result.getCatalogId()); - assertEquals(TEST_METADATA, result.getMetadata()); - } - - private ModelScanMetricsReport createTestReport() { - return ImmutableModelScanMetricsReport.builder() - .reportId(TEST_REPORT_ID) - .realmId(TEST_REALM_ID) - .catalogId(TEST_CATALOG_ID) - .tableId(TEST_TABLE_ID) - .timestampMs(TEST_TIMESTAMP_MS) - .principalName(TEST_PRINCIPAL) - .requestId(TEST_REQUEST_ID) - .otelTraceId(TEST_OTEL_TRACE_ID) - .snapshotId(TEST_SNAPSHOT_ID) - .resultDataFiles(TEST_RESULT_DATA_FILES) - .resultDeleteFiles(TEST_RESULT_DELETE_FILES) - .totalFileSizeBytes(TEST_TOTAL_FILE_SIZE) - .totalDataManifests(TEST_TOTAL_DATA_MANIFESTS) - .totalDeleteManifests(TEST_TOTAL_DELETE_MANIFESTS) - .scannedDataManifests(TEST_SCANNED_DATA_MANIFESTS) - .scannedDeleteManifests(TEST_SCANNED_DELETE_MANIFESTS) - .skippedDataManifests(TEST_SKIPPED_DATA_MANIFESTS) - .skippedDeleteManifests(TEST_SKIPPED_DELETE_MANIFESTS) - .skippedDataFiles(TEST_SKIPPED_DATA_FILES) - .skippedDeleteFiles(TEST_SKIPPED_DELETE_FILES) - .totalPlanningDurationMs(TEST_PLANNING_DURATION) - .equalityDeleteFiles(TEST_EQUALITY_DELETE_FILES) - .positionalDeleteFiles(TEST_POSITIONAL_DELETE_FILES) - .indexedDeleteFiles(TEST_INDEXED_DELETE_FILES) - .totalDeleteFileSizeBytes(TEST_DELETE_FILE_SIZE) - .metadata(TEST_METADATA) - .build(); - } -} diff --git a/polaris-core/src/main/java/org/apache/polaris/core/auth/PolarisAuthorizableOperation.java b/polaris-core/src/main/java/org/apache/polaris/core/auth/PolarisAuthorizableOperation.java index c016749459b..39c92153c5c 100644 --- a/polaris-core/src/main/java/org/apache/polaris/core/auth/PolarisAuthorizableOperation.java +++ b/polaris-core/src/main/java/org/apache/polaris/core/auth/PolarisAuthorizableOperation.java @@ -56,6 +56,7 @@ public enum PolarisAuthorizableOperation { VIEW_EXISTS, RENAME_VIEW, REPORT_READ_METRICS, + LIST_TABLE_METRICS, REPORT_WRITE_METRICS, SEND_NOTIFICATIONS, LIST_CATALOGS, diff --git a/polaris-core/src/main/java/org/apache/polaris/core/auth/PolarisAuthorizerImpl.java b/polaris-core/src/main/java/org/apache/polaris/core/auth/PolarisAuthorizerImpl.java index 25fa6ae774d..0e4fe62801a 100644 --- a/polaris-core/src/main/java/org/apache/polaris/core/auth/PolarisAuthorizerImpl.java +++ b/polaris-core/src/main/java/org/apache/polaris/core/auth/PolarisAuthorizerImpl.java @@ -98,6 +98,7 @@ import static org.apache.polaris.core.entity.PolarisPrivilege.TABLE_MANAGE_GRANTS_ON_SECURABLE; import static org.apache.polaris.core.entity.PolarisPrivilege.TABLE_MANAGE_STRUCTURE; import static org.apache.polaris.core.entity.PolarisPrivilege.TABLE_READ_DATA; +import static org.apache.polaris.core.entity.PolarisPrivilege.TABLE_READ_METRICS; import static org.apache.polaris.core.entity.PolarisPrivilege.TABLE_READ_PROPERTIES; import static org.apache.polaris.core.entity.PolarisPrivilege.TABLE_REMOVE_PARTITION_SPECS; import static org.apache.polaris.core.entity.PolarisPrivilege.TABLE_REMOVE_PROPERTIES; @@ -456,6 +457,9 @@ public class PolarisAuthorizerImpl implements PolarisAuthorizer { SUPER_PRIVILEGES.putAll( TABLE_READ_DATA, List.of(CATALOG_MANAGE_CONTENT, TABLE_READ_DATA, TABLE_WRITE_DATA)); SUPER_PRIVILEGES.putAll(TABLE_WRITE_DATA, List.of(CATALOG_MANAGE_CONTENT, TABLE_WRITE_DATA)); + SUPER_PRIVILEGES.putAll( + TABLE_READ_METRICS, + List.of(CATALOG_MANAGE_CONTENT, TABLE_FULL_METADATA, TABLE_READ_DATA, TABLE_READ_METRICS)); SUPER_PRIVILEGES.putAll( NAMESPACE_FULL_METADATA, List.of(CATALOG_MANAGE_CONTENT, CATALOG_MANAGE_METADATA, NAMESPACE_FULL_METADATA)); diff --git a/polaris-core/src/main/java/org/apache/polaris/core/auth/RbacOperationSemantics.java b/polaris-core/src/main/java/org/apache/polaris/core/auth/RbacOperationSemantics.java index f63999959dd..4880f8dddad 100644 --- a/polaris-core/src/main/java/org/apache/polaris/core/auth/RbacOperationSemantics.java +++ b/polaris-core/src/main/java/org/apache/polaris/core/auth/RbacOperationSemantics.java @@ -77,6 +77,7 @@ import static org.apache.polaris.core.auth.PolarisAuthorizableOperation.LIST_PRINCIPAL_ROLES; import static org.apache.polaris.core.auth.PolarisAuthorizableOperation.LIST_PRINCIPAL_ROLES_ASSIGNED; import static org.apache.polaris.core.auth.PolarisAuthorizableOperation.LIST_TABLES; +import static org.apache.polaris.core.auth.PolarisAuthorizableOperation.LIST_TABLE_METRICS; import static org.apache.polaris.core.auth.PolarisAuthorizableOperation.LIST_VIEWS; import static org.apache.polaris.core.auth.PolarisAuthorizableOperation.LOAD_NAMESPACE_METADATA; import static org.apache.polaris.core.auth.PolarisAuthorizableOperation.LOAD_POLICY; @@ -193,6 +194,7 @@ import static org.apache.polaris.core.entity.PolarisPrivilege.TABLE_LIST; import static org.apache.polaris.core.entity.PolarisPrivilege.TABLE_MANAGE_GRANTS_ON_SECURABLE; import static org.apache.polaris.core.entity.PolarisPrivilege.TABLE_READ_DATA; +import static org.apache.polaris.core.entity.PolarisPrivilege.TABLE_READ_METRICS; import static org.apache.polaris.core.entity.PolarisPrivilege.TABLE_READ_PROPERTIES; import static org.apache.polaris.core.entity.PolarisPrivilege.TABLE_REMOVE_PARTITION_SPECS; import static org.apache.polaris.core.entity.PolarisPrivilege.TABLE_REMOVE_PROPERTIES; @@ -325,6 +327,7 @@ private static void register( // Metrics and notifications register(REPORT_READ_METRICS, TABLE_READ_DATA); + register(LIST_TABLE_METRICS, TABLE_READ_METRICS); register(REPORT_WRITE_METRICS, TABLE_WRITE_DATA); register( SEND_NOTIFICATIONS, diff --git a/polaris-core/src/main/java/org/apache/polaris/core/entity/PolarisPrivilege.java b/polaris-core/src/main/java/org/apache/polaris/core/entity/PolarisPrivilege.java index ff7e029ba54..483d18b2e45 100644 --- a/polaris-core/src/main/java/org/apache/polaris/core/entity/PolarisPrivilege.java +++ b/polaris-core/src/main/java/org/apache/polaris/core/entity/PolarisPrivilege.java @@ -256,6 +256,15 @@ public enum PolarisPrivilege { PolarisEntityType.TABLE_LIKE, List.of(PolarisEntitySubType.ICEBERG_TABLE, PolarisEntitySubType.GENERIC_TABLE), PolarisEntityType.CATALOG_ROLE), + /** + * Read-only access to table scan and commit metrics reports. Does not grant access to table data. + * Implied by TABLE_READ_DATA and TABLE_FULL_METADATA. + */ + TABLE_READ_METRICS( + 103, + PolarisEntityType.TABLE_LIKE, + List.of(PolarisEntitySubType.ICEBERG_TABLE, PolarisEntitySubType.GENERIC_TABLE), + PolarisEntityType.CATALOG_ROLE), ; /** diff --git a/polaris-core/src/main/java/org/apache/polaris/core/persistence/BasePersistence.java b/polaris-core/src/main/java/org/apache/polaris/core/persistence/BasePersistence.java index 59081179f98..6aaf0dd5bd0 100644 --- a/polaris-core/src/main/java/org/apache/polaris/core/persistence/BasePersistence.java +++ b/polaris-core/src/main/java/org/apache/polaris/core/persistence/BasePersistence.java @@ -52,11 +52,6 @@ *

Note that APIs to the actual persistence store are very basic, often point read or write to * the underlying data store. The goal is to make it really easy to back this using databases like * Postgres or simpler KV store. - * - *

Metrics-related persistence is intentionally decoupled and lives in {@code - * MetricsPersistence}. A concrete backend may implement both SPIs on the same class, but callers - * that only need metrics persistence should depend on {@code MetricsPersistence} directly rather - * than on {@link BasePersistence}. */ public interface BasePersistence extends PolicyMappingPersistence { /** diff --git a/polaris-core/src/main/java/org/apache/polaris/core/persistence/PolarisMetaStoreManager.java b/polaris-core/src/main/java/org/apache/polaris/core/persistence/PolarisMetaStoreManager.java index 35c8c72221c..939d1f76258 100644 --- a/polaris-core/src/main/java/org/apache/polaris/core/persistence/PolarisMetaStoreManager.java +++ b/polaris-core/src/main/java/org/apache/polaris/core/persistence/PolarisMetaStoreManager.java @@ -48,7 +48,6 @@ import org.apache.polaris.core.persistence.dao.entity.ListEntitiesResult; import org.apache.polaris.core.persistence.dao.entity.ResolvedEntitiesResult; import org.apache.polaris.core.persistence.dao.entity.ResolvedEntityResult; -import org.apache.polaris.core.persistence.metrics.PolarisMetricsManager; import org.apache.polaris.core.persistence.pagination.Page; import org.apache.polaris.core.persistence.pagination.PageToken; import org.apache.polaris.core.policy.PolarisPolicyMappingManager; @@ -63,8 +62,7 @@ public interface PolarisMetaStoreManager extends PolarisSecretsManager, PolarisGrantManager, PolarisPolicyMappingManager, - PolarisEventManager, - PolarisMetricsManager { + PolarisEventManager { /** * Bootstrap the Polaris service, creating the root catalog, root principal, and associated diff --git a/polaris-core/src/main/java/org/apache/polaris/core/persistence/metrics/CommitMetricsRecord.java b/polaris-core/src/main/java/org/apache/polaris/core/persistence/metrics/CommitMetricsRecord.java index 6d67408cbab..a6441f11329 100644 --- a/polaris-core/src/main/java/org/apache/polaris/core/persistence/metrics/CommitMetricsRecord.java +++ b/polaris-core/src/main/java/org/apache/polaris/core/persistence/metrics/CommitMetricsRecord.java @@ -25,14 +25,6 @@ /** * Backend-agnostic representation of an Iceberg commit metrics report. * - *

This record captures all relevant metrics from an Iceberg {@code CommitReport} along with - * contextual information such as catalog identification and table location. - * - *

Common identification fields are inherited from {@link MetricsRecordIdentity}. - * - *

Note: Realm ID is not included in this record. Multi-tenancy realm context should be obtained - * from the CDI-injected {@code RealmContext} at persistence time. - * *

Note: This type is part of the experimental Metrics Persistence SPI and may change in * future releases. */ @@ -40,86 +32,48 @@ @PolarisImmutable public interface CommitMetricsRecord extends MetricsRecordIdentity { - // === Commit Context === - - /** Snapshot ID created by this commit. */ long snapshotId(); - /** Sequence number of the snapshot. */ Optional sequenceNumber(); - /** Operation type (e.g., "append", "overwrite", "delete"). */ String operation(); - // === File Metrics - Data Files === - - /** Number of data files added. */ long addedDataFiles(); - /** Number of data files removed. */ long removedDataFiles(); - /** Total number of data files after commit. */ long totalDataFiles(); - // === File Metrics - Delete Files === - - /** Number of delete files added. */ long addedDeleteFiles(); - /** Number of delete files removed. */ long removedDeleteFiles(); - /** Total number of delete files after commit. */ long totalDeleteFiles(); - /** Number of equality delete files added. */ long addedEqualityDeleteFiles(); - /** Number of equality delete files removed. */ long removedEqualityDeleteFiles(); - /** Number of positional delete files added. */ long addedPositionalDeleteFiles(); - /** Number of positional delete files removed. */ long removedPositionalDeleteFiles(); - // === Record Metrics === - - /** Number of records added. */ long addedRecords(); - /** Number of records removed. */ long removedRecords(); - /** Total number of records after commit. */ long totalRecords(); - // === Size Metrics === - - /** Size of added files in bytes. */ long addedFileSizeBytes(); - /** Size of removed files in bytes. */ long removedFileSizeBytes(); - /** Total file size in bytes after commit. */ long totalFileSizeBytes(); - // === Timing === - - /** Total duration of the commit in milliseconds. */ Optional totalDurationMs(); - /** Number of commit attempts. */ int attempts(); - /** - * Creates a new builder for CommitMetricsRecord. - * - * @return a new builder instance - */ static ImmutableCommitMetricsRecord.Builder builder() { return ImmutableCommitMetricsRecord.builder(); } diff --git a/polaris-core/src/main/java/org/apache/polaris/core/persistence/metrics/MetricsQuerySpi.java b/polaris-core/src/main/java/org/apache/polaris/core/persistence/metrics/MetricsQuerySpi.java new file mode 100644 index 00000000000..58b6e28e1f0 --- /dev/null +++ b/polaris-core/src/main/java/org/apache/polaris/core/persistence/metrics/MetricsQuerySpi.java @@ -0,0 +1,64 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.polaris.core.persistence.metrics; + +import com.google.common.annotations.Beta; +import org.apache.polaris.core.persistence.pagination.Page; +import org.apache.polaris.core.persistence.pagination.PageToken; +import org.jspecify.annotations.NonNull; +import org.jspecify.annotations.Nullable; + +/** + * SPI for querying persisted Iceberg metrics reports. + * + *

Implementations are provided by persistence-backend extension modules (e.g. {@code + * polaris-extensions-metrics-reports-jdbc}). When no implementation is on the classpath, the read + * path returns HTTP 501 Not Implemented. + * + * @see MetricsPersistence for the corresponding write SPI + */ +@Beta +public interface MetricsQuerySpi { + + /** + * Lists persisted scan metrics reports for the given table, applying the supplied filters and + * returning at most one page of results. + */ + Page listScanReports( + long catalogId, + long tableId, + @Nullable Long snapshotId, + @Nullable String principalName, + @Nullable Long timestampFrom, + @Nullable Long timestampTo, + @NonNull PageToken pageToken); + + /** + * Lists persisted commit metrics reports for the given table, applying the supplied filters and + * returning at most one page of results. + */ + Page listCommitReports( + long catalogId, + long tableId, + @Nullable Long snapshotId, + @Nullable String principalName, + @Nullable Long timestampFrom, + @Nullable Long timestampTo, + @NonNull PageToken pageToken); +} diff --git a/polaris-core/src/main/java/org/apache/polaris/core/persistence/metrics/MetricsRecordIdentity.java b/polaris-core/src/main/java/org/apache/polaris/core/persistence/metrics/MetricsRecordIdentity.java index 7d4e427677b..fa98a537bfa 100644 --- a/polaris-core/src/main/java/org/apache/polaris/core/persistence/metrics/MetricsRecordIdentity.java +++ b/polaris-core/src/main/java/org/apache/polaris/core/persistence/metrics/MetricsRecordIdentity.java @@ -26,105 +26,39 @@ /** * Base interface containing common identification fields shared by all metrics records. * - *

This interface defines the common fields that identify the source of a metrics report, - * including the report ID, catalog ID, table ID, timestamp, and metadata. - * *

Both {@link ScanMetricsRecord} and {@link CommitMetricsRecord} extend this interface to * inherit these common fields while adding their own specific metrics. * - *

Design Decisions

- * - *

Entity IDs only (no names): We store only catalog ID and table ID, not their names or - * namespace paths. Names can change over time (via rename operations), which would make querying - * historical metrics by name challenging and lead to correctness issues. Queries should resolve - * names to IDs using the current catalog state. The table ID uniquely identifies the table, and the - * namespace can be derived from the table entity if needed. - * - *

Realm ID: Realm ID is intentionally not included in this interface. Multi-tenancy realm - * context should be obtained from the CDI-injected {@code RealmContext} at persistence time. This - * keeps catalog-specific code from needing to manage realm concerns. - * *

Note: This type is part of the experimental Metrics Persistence SPI and may change in * future releases. */ @Beta public interface MetricsRecordIdentity { - /** - * Unique identifier for this report (UUID). - * - *

This ID is generated when the record is created and serves as the primary key for the - * metrics record in persistence storage. - */ + /** Unique identifier for this report (UUID). */ String reportId(); - /** - * Internal catalog ID. - * - *

This matches the catalog entity ID in Polaris persistence, as defined by {@code - * PolarisEntityCore#getId()}. The catalog name is not stored since it can change over time; - * queries should resolve names to IDs using the current catalog state. - */ + /** Internal catalog ID. */ long catalogId(); - /** - * Internal table entity ID. - * - *

This matches the table entity ID in Polaris persistence, as defined by {@code - * PolarisEntityCore#getId()}. The table name is not stored since it can change over time; queries - * should resolve names to IDs using the current catalog state. The namespace can be derived from - * the table entity if needed. - */ + /** Internal table entity ID. */ long tableId(); - /** - * Timestamp when the report was received. - * - *

This is the server-side timestamp when the metrics report was processed, not the client-side - * timestamp when the operation occurred. - */ + /** Timestamp when the report was received. */ Instant timestamp(); - /** - * Additional metadata as key-value pairs. - * - *

This map can contain additional contextual information from the original Iceberg report, - * including client-provided trace IDs or other correlation data. Persistence implementations can - * store and index specific metadata fields as needed. - */ + /** Additional metadata as key-value pairs. */ Map metadata(); - // === Request Context === - - /** - * Name of the principal who made the request. - * - *

This is resolved from the authenticated principal at the time the metrics report was - * received. May be null if authentication is not enabled or the principal cannot be determined. - */ + /** Name of the principal who made the request. */ @Nullable String principalName(); - /** - * Server-generated request ID for correlation. - * - *

This ID is generated by the server for each incoming request and can be used to correlate - * metrics with server logs and traces. - */ + /** Server-generated request ID for correlation. */ @Nullable String requestId(); - /** - * OpenTelemetry trace ID for distributed tracing correlation. - * - *

When OpenTelemetry is configured, this provides the trace ID from the current span, enabling - * correlation of metrics with distributed traces. - */ + /** OpenTelemetry trace ID for distributed tracing correlation. */ @Nullable String otelTraceId(); - /** - * OpenTelemetry span ID for distributed tracing correlation. - * - *

When OpenTelemetry is configured, this provides the span ID from the current span, enabling - * correlation of metrics with specific spans in a trace. - */ + /** OpenTelemetry span ID for distributed tracing correlation. */ @Nullable String otelSpanId(); } diff --git a/polaris-core/src/main/java/org/apache/polaris/core/persistence/metrics/PolarisMetricsManager.java b/polaris-core/src/main/java/org/apache/polaris/core/persistence/metrics/PolarisMetricsManager.java index 1d2885d0d94..874fee5efb5 100644 --- a/polaris-core/src/main/java/org/apache/polaris/core/persistence/metrics/PolarisMetricsManager.java +++ b/polaris-core/src/main/java/org/apache/polaris/core/persistence/metrics/PolarisMetricsManager.java @@ -34,11 +34,6 @@ *

Request context (principal name, request ID, OTEL trace/span IDs) should be populated in the * record by the caller before invoking these methods. This keeps the SPI simple with a single * method parameter containing all the data needed for persistence. - * - *

Since {@link org.apache.polaris.core.persistence.BasePersistence} now extends {@link - * MetricsPersistence} with default no-op implementations, all persistence backends automatically - * support this interface. Backends that want actual metrics persistence (e.g., JDBC) override the - * methods; others use the default no-op behavior. */ public interface PolarisMetricsManager { @@ -48,9 +43,6 @@ public interface PolarisMetricsManager { *

Delegates to the underlying {@link MetricsPersistence#writeScanReport} method. If the * persistence backend doesn't override the default implementation, this is a no-op. * - *

The record should contain all request context fields (principalName, requestId, otelTraceId, - * otelSpanId) populated by the caller. - * * @param callCtx the call context containing the persistence layer * @param record the scan metrics record to persist (including request context) */ @@ -65,9 +57,6 @@ default void writeScanMetrics( *

Delegates to the underlying {@link MetricsPersistence#writeCommitReport} method. If the * persistence backend doesn't override the default implementation, this is a no-op. * - *

The record should contain all request context fields (principalName, requestId, otelTraceId, - * otelSpanId) populated by the caller. - * * @param callCtx the call context containing the persistence layer * @param record the commit metrics record to persist (including request context) */ diff --git a/polaris-core/src/main/java/org/apache/polaris/core/persistence/metrics/ScanMetricsRecord.java b/polaris-core/src/main/java/org/apache/polaris/core/persistence/metrics/ScanMetricsRecord.java index 44947d8f758..8fbfb01bbca 100644 --- a/polaris-core/src/main/java/org/apache/polaris/core/persistence/metrics/ScanMetricsRecord.java +++ b/polaris-core/src/main/java/org/apache/polaris/core/persistence/metrics/ScanMetricsRecord.java @@ -26,14 +26,6 @@ /** * Backend-agnostic representation of an Iceberg scan metrics report. * - *

This record captures all relevant metrics from an Iceberg {@code ScanReport} along with - * contextual information such as catalog identification and table location. - * - *

Common identification fields are inherited from {@link MetricsRecordIdentity}. - * - *

Note: Realm ID is not included in this record. Multi-tenancy realm context should be obtained - * from the CDI-injected {@code RealmContext} at persistence time. - * *

Note: This type is part of the experimental Metrics Persistence SPI and may change in * future releases. */ @@ -41,84 +33,48 @@ @PolarisImmutable public interface ScanMetricsRecord extends MetricsRecordIdentity { - // === Scan Context === - - /** Snapshot ID that was scanned. */ Optional snapshotId(); - /** Schema ID used for the scan. */ Optional schemaId(); - /** Filter expression applied to the scan (as string). */ Optional filterExpression(); - /** List of projected field IDs. */ List projectedFieldIds(); - /** List of projected field names. */ List projectedFieldNames(); - // === Scan Metrics - File Counts === - - /** Number of data files in the result. */ long resultDataFiles(); - /** Number of delete files in the result. */ long resultDeleteFiles(); - /** Total size of files in bytes. */ long totalFileSizeBytes(); - // === Scan Metrics - Manifest Counts === - - /** Total number of data manifests. */ long totalDataManifests(); - /** Total number of delete manifests. */ long totalDeleteManifests(); - /** Number of data manifests that were scanned. */ long scannedDataManifests(); - /** Number of delete manifests that were scanned. */ long scannedDeleteManifests(); - /** Number of data manifests that were skipped. */ long skippedDataManifests(); - /** Number of delete manifests that were skipped. */ long skippedDeleteManifests(); - /** Number of data files that were skipped. */ long skippedDataFiles(); - /** Number of delete files that were skipped. */ long skippedDeleteFiles(); - // === Scan Metrics - Timing === - - /** Total planning duration in milliseconds. */ long totalPlanningDurationMs(); - // === Scan Metrics - Delete Files === - - /** Number of equality delete files. */ long equalityDeleteFiles(); - /** Number of positional delete files. */ long positionalDeleteFiles(); - /** Number of indexed delete files. */ long indexedDeleteFiles(); - /** Total size of delete files in bytes. */ long totalDeleteFileSizeBytes(); - /** - * Creates a new builder for ScanMetricsRecord. - * - * @return a new builder instance - */ static ImmutableScanMetricsRecord.Builder builder() { return ImmutableScanMetricsRecord.builder(); } diff --git a/polaris-core/src/test/java/org/apache/polaris/core/entity/PolarisPrivilegeTest.java b/polaris-core/src/test/java/org/apache/polaris/core/entity/PolarisPrivilegeTest.java index 14596911fd7..048f9ce8893 100644 --- a/polaris-core/src/test/java/org/apache/polaris/core/entity/PolarisPrivilegeTest.java +++ b/polaris-core/src/test/java/org/apache/polaris/core/entity/PolarisPrivilegeTest.java @@ -131,7 +131,8 @@ static Stream polarisPrivileges() { Arguments.of(100, PolarisPrivilege.TABLE_REMOVE_STATISTICS), Arguments.of(101, PolarisPrivilege.TABLE_REMOVE_PARTITION_SPECS), Arguments.of(102, PolarisPrivilege.TABLE_MANAGE_STRUCTURE), - Arguments.of(103, null)); + Arguments.of(103, PolarisPrivilege.TABLE_READ_METRICS), + Arguments.of(104, null)); } @ParameterizedTest diff --git a/runtime/defaults/src/main/resources/application-it.properties b/runtime/defaults/src/main/resources/application-it.properties index 108a9585b9c..0110420705f 100644 --- a/runtime/defaults/src/main/resources/application-it.properties +++ b/runtime/defaults/src/main/resources/application-it.properties @@ -20,6 +20,8 @@ # Configuration common to ALL integration tests (executed with "it" profile – Gradle "intTest" tasks). # Note: Quarkus integration tests cannot use QuarkusTestProfile. +polaris.iceberg-metrics.reporting.type=no-op + quarkus.http.limits.max-body-size=1000000 quarkus.http.port=0 diff --git a/runtime/defaults/src/main/resources/application-test.properties b/runtime/defaults/src/main/resources/application-test.properties index 4f31702e270..ca38e1280bf 100644 --- a/runtime/defaults/src/main/resources/application-test.properties +++ b/runtime/defaults/src/main/resources/application-test.properties @@ -20,6 +20,8 @@ # Configuration common to ALL unit tests (executed with "test" profile – Gradle "test" tasks). # Per-test specific configuration should use QuarkusTestProfile +polaris.iceberg-metrics.reporting.type=no-op + quarkus.datasource.devservices.enabled=false quarkus.keycloak.devservices.enabled=false quarkus.mongodb.devservices.enabled=false diff --git a/runtime/server/build.gradle.kts b/runtime/server/build.gradle.kts index f577e2c84ef..eb9b69b16c2 100644 --- a/runtime/server/build.gradle.kts +++ b/runtime/server/build.gradle.kts @@ -40,6 +40,8 @@ dependencies { runtimeOnly(project(":polaris-extensions-federation-hadoop")) runtimeOnly(project(":polaris-extensions-auth-opa")) runtimeOnly(project(":polaris-extensions-auth-ranger")) + runtimeOnly(project(":polaris-extensions-metrics-reports")) + runtimeOnly(project(":polaris-api-metrics-reports-service")) if ((project.findProperty("NonRESTCatalogs") as String?)?.contains("HIVE") == true) { runtimeOnly(project(":polaris-extensions-federation-hive")) diff --git a/runtime/service/build.gradle.kts b/runtime/service/build.gradle.kts index d3608738c7a..ae0afc8a7e3 100644 --- a/runtime/service/build.gradle.kts +++ b/runtime/service/build.gradle.kts @@ -30,9 +30,12 @@ dependencies { implementation(project(":polaris-api-management-service")) implementation(project(":polaris-api-iceberg-service")) implementation(project(":polaris-api-catalog-service")) + // compileOnly — MetricsReportsService implements PolarisCatalogsApiService from this module but + // we keep it off the transitive dep graph; runtime/server adds it to the runtime classpath. + compileOnly(project(":polaris-api-metrics-reports-service")) + implementation(project(":polaris-extensions-metrics-reports-spi")) runtimeOnly(project(":polaris-relational-jdbc")) - implementation(project(":polaris-runtime-defaults")) implementation(project(":polaris-runtime-common")) @@ -126,6 +129,7 @@ dependencies { } testImplementation(project(":polaris-api-management-model")) + testImplementation(project(":polaris-api-metrics-reports-service")) testImplementation(project(":polaris-relational-jdbc")) testImplementation(project(":polaris-minio-testcontainer")) @@ -171,6 +175,7 @@ dependencies { testImplementation(project(":polaris-persistence-nosql-impl")) testFixturesImplementation(project(":polaris-core")) + testFixturesImplementation(project(":polaris-extensions-metrics-reports-spi")) testFixturesImplementation(project(":polaris-api-management-model")) testFixturesImplementation(project(":polaris-api-management-service")) testFixturesImplementation(project(":polaris-api-iceberg-service")) @@ -204,6 +209,10 @@ dependencies { testFixturesImplementation("com.azure:azure-storage-blob") testFixturesImplementation("com.azure:azure-storage-file-datalake") + // Provides jakarta.ws.rs.ext.RuntimeDelegate needed to build Response objects in plain unit tests + testRuntimeOnly(enforcedPlatform(libs.quarkus.bom)) + testRuntimeOnly("io.quarkus.resteasy.reactive:resteasy-reactive") + // This dependency brings in RESTEasy Classic, which conflicts with Quarkus RESTEasy Reactive; // it must not be present during Quarkus augmentation otherwise Quarkus tests won't start. intTestRuntimeOnly(libs.keycloak.admin.client) diff --git a/runtime/service/src/main/java/org/apache/polaris/service/catalog/iceberg/IcebergCatalogHandler.java b/runtime/service/src/main/java/org/apache/polaris/service/catalog/iceberg/IcebergCatalogHandler.java index 46d33faa930..af9447e188f 100644 --- a/runtime/service/src/main/java/org/apache/polaris/service/catalog/iceberg/IcebergCatalogHandler.java +++ b/runtime/service/src/main/java/org/apache/polaris/service/catalog/iceberg/IcebergCatalogHandler.java @@ -117,6 +117,7 @@ import org.apache.polaris.core.storage.PolarisStorageActions; import org.apache.polaris.core.storage.StorageAccessConfig; import org.apache.polaris.core.storage.StorageUtil; +import org.apache.polaris.extension.metrics.IcebergMetricsReporter; import org.apache.polaris.immutables.PolarisImmutable; import org.apache.polaris.service.catalog.AccessDelegationMode; import org.apache.polaris.service.catalog.AccessDelegationModeResolver; @@ -130,7 +131,6 @@ import org.apache.polaris.service.events.EventAttributes; import org.apache.polaris.service.http.IcebergHttpUtil; import org.apache.polaris.service.http.IfNoneMatch; -import org.apache.polaris.service.reporting.PolarisMetricsReporter; import org.apache.polaris.service.types.NotificationRequest; import org.jspecify.annotations.NonNull; import org.jspecify.annotations.Nullable; @@ -209,7 +209,7 @@ public abstract class IcebergCatalogHandler extends CatalogHandler implements Au protected abstract EventAttributeMap eventAttributeMap(); - protected abstract PolarisMetricsReporter metricsReporter(); + protected abstract IcebergMetricsReporter metricsReporter(); protected abstract Clock clock(); diff --git a/runtime/service/src/main/java/org/apache/polaris/service/catalog/iceberg/IcebergCatalogHandlerFactory.java b/runtime/service/src/main/java/org/apache/polaris/service/catalog/iceberg/IcebergCatalogHandlerFactory.java index 31ba19550a1..08d71558bbb 100644 --- a/runtime/service/src/main/java/org/apache/polaris/service/catalog/iceberg/IcebergCatalogHandlerFactory.java +++ b/runtime/service/src/main/java/org/apache/polaris/service/catalog/iceberg/IcebergCatalogHandlerFactory.java @@ -33,12 +33,12 @@ import org.apache.polaris.core.persistence.PolarisMetaStoreManager; import org.apache.polaris.core.persistence.resolver.ResolutionManifestFactory; import org.apache.polaris.core.persistence.resolver.ResolverFactory; +import org.apache.polaris.extension.metrics.IcebergMetricsReporter; import org.apache.polaris.service.catalog.AccessDelegationModeResolver; import org.apache.polaris.service.catalog.CatalogPrefixParser; import org.apache.polaris.service.catalog.io.StorageAccessConfigProvider; import org.apache.polaris.service.config.ReservedProperties; import org.apache.polaris.service.events.EventAttributeMap; -import org.apache.polaris.service.reporting.PolarisMetricsReporter; @RequestScoped public class IcebergCatalogHandlerFactory { @@ -57,7 +57,7 @@ public class IcebergCatalogHandlerFactory { @Inject @Any Instance federatedCatalogFactories; @Inject StorageAccessConfigProvider storageAccessConfigProvider; @Inject EventAttributeMap eventAttributeMap; - @Inject PolarisMetricsReporter metricsReporter; + @Inject IcebergMetricsReporter metricsReporter; @Inject Clock clock; @Inject AccessDelegationModeResolver accessDelegationModeResolver; diff --git a/runtime/service/src/main/java/org/apache/polaris/service/config/ServiceProducers.java b/runtime/service/src/main/java/org/apache/polaris/service/config/ServiceProducers.java index 8a5f9a71526..209b4e4e635 100644 --- a/runtime/service/src/main/java/org/apache/polaris/service/config/ServiceProducers.java +++ b/runtime/service/src/main/java/org/apache/polaris/service/config/ServiceProducers.java @@ -62,6 +62,7 @@ import org.apache.polaris.core.secrets.UserSecretsManagerFactory; import org.apache.polaris.core.storage.cache.StorageCredentialCache; import org.apache.polaris.core.storage.cache.StorageCredentialCacheConfig; +import org.apache.polaris.extension.metrics.IcebergMetricsReporter; import org.apache.polaris.service.auth.AuthenticationConfiguration; import org.apache.polaris.service.auth.AuthenticationRealmConfiguration; import org.apache.polaris.service.auth.AuthenticationType; @@ -83,7 +84,6 @@ import org.apache.polaris.service.ratelimiter.TokenBucketConfiguration; import org.apache.polaris.service.ratelimiter.TokenBucketFactory; import org.apache.polaris.service.reporting.MetricsReportingConfiguration; -import org.apache.polaris.service.reporting.PolarisMetricsReporter; import org.apache.polaris.service.secrets.SecretsManagerConfiguration; import org.apache.polaris.service.storage.StorageConfiguration; import org.apache.polaris.service.storage.aws.S3AccessConfig; @@ -457,9 +457,19 @@ public RequestIdSupplier requestIdSupplier() { @Produces @ApplicationScoped - public PolarisMetricsReporter metricsReporter( - MetricsReportingConfiguration config, @Any Instance reporters) { - return reporters.select(Identifier.Literal.of(config.type())).get(); + public IcebergMetricsReporter metricsReporter( + MetricsReportingConfiguration config, @Any Instance reporters) { + var selected = reporters.select(Identifier.Literal.of(config.type())); + if (selected.isUnsatisfied()) { + // NoOpMetricsReporter and LoggingMetricsReporter live in polaris-extensions-metrics-reports + // (not SPI). If that module is absent from the classpath, fall back to a silent no-op so + // core service startup is not blocked by a missing metrics extension. + LOGGER.warn( + "No IcebergMetricsReporter found for type '{}'; Iceberg metrics will be dropped", + config.type()); + return (catalogName, catalogId, table, tableId, metricsReport, receivedTimestamp) -> {}; + } + return selected.get(); } @Produces diff --git a/runtime/service/src/main/java/org/apache/polaris/service/metrics/MetricsReportsService.java b/runtime/service/src/main/java/org/apache/polaris/service/metrics/MetricsReportsService.java new file mode 100644 index 00000000000..23e86af2df8 --- /dev/null +++ b/runtime/service/src/main/java/org/apache/polaris/service/metrics/MetricsReportsService.java @@ -0,0 +1,273 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.polaris.service.metrics; + +import jakarta.enterprise.context.RequestScoped; +import jakarta.enterprise.inject.Any; +import jakarta.enterprise.inject.Instance; +import jakarta.inject.Inject; +import jakarta.ws.rs.core.Response; +import jakarta.ws.rs.core.SecurityContext; +import java.util.Arrays; +import java.util.List; +import org.apache.iceberg.catalog.Namespace; +import org.apache.iceberg.catalog.TableIdentifier; +import org.apache.iceberg.exceptions.NotFoundException; +import org.apache.polaris.core.auth.PolarisAuthorizableOperation; +import org.apache.polaris.core.auth.PolarisAuthorizer; +import org.apache.polaris.core.auth.PolarisPrincipal; +import org.apache.polaris.core.catalog.PolarisCatalogHelpers; +import org.apache.polaris.core.context.RealmContext; +import org.apache.polaris.core.entity.CatalogEntity; +import org.apache.polaris.core.entity.PolarisEntitySubType; +import org.apache.polaris.core.entity.PolarisEntityType; +import org.apache.polaris.core.metrics.api.model.CommitMetricsObject; +import org.apache.polaris.core.metrics.api.model.CommitMetricsReport; +import org.apache.polaris.core.metrics.api.model.CommitPayload; +import org.apache.polaris.core.metrics.api.model.CommitPayloadData; +import org.apache.polaris.core.metrics.api.model.ListCommitMetricsResponse; +import org.apache.polaris.core.metrics.api.model.ListScanMetricsResponse; +import org.apache.polaris.core.metrics.api.model.MetricsActor; +import org.apache.polaris.core.metrics.api.model.MetricsRequest; +import org.apache.polaris.core.metrics.api.model.ScanMetricsObject; +import org.apache.polaris.core.metrics.api.model.ScanMetricsReport; +import org.apache.polaris.core.metrics.api.model.ScanPayload; +import org.apache.polaris.core.metrics.api.model.ScanPayloadData; +import org.apache.polaris.core.persistence.PolarisResolvedPathWrapper; +import org.apache.polaris.core.persistence.metrics.CommitMetricsRecord; +import org.apache.polaris.core.persistence.metrics.MetricsQuerySpi; +import org.apache.polaris.core.persistence.metrics.ScanMetricsRecord; +import org.apache.polaris.core.persistence.pagination.Page; +import org.apache.polaris.core.persistence.pagination.PageToken; +import org.apache.polaris.core.persistence.resolver.PolarisResolutionManifest; +import org.apache.polaris.core.persistence.resolver.ResolutionManifestFactory; +import org.apache.polaris.core.persistence.resolver.ResolvedPathKey; +import org.apache.polaris.core.persistence.resolver.ResolverPath; +import org.apache.polaris.core.persistence.resolver.ResolverStatus; +import org.apache.polaris.core.rest.NamespaceUtils; +import org.apache.polaris.service.metrics.api.PolarisCatalogsApiService; +import org.jspecify.annotations.NonNull; + +/** Service implementation for the Metrics Reports API. */ +@RequestScoped +public class MetricsReportsService implements PolarisCatalogsApiService { + + private final PolarisAuthorizer authorizer; + private final PolarisPrincipal polarisPrincipal; + private final ResolutionManifestFactory resolutionManifestFactory; + private final Instance queryProvider; + + @Inject + public MetricsReportsService( + @NonNull PolarisAuthorizer authorizer, + @NonNull PolarisPrincipal polarisPrincipal, + @NonNull ResolutionManifestFactory resolutionManifestFactory, + @Any Instance queryProvider) { + this.authorizer = authorizer; + this.polarisPrincipal = polarisPrincipal; + this.resolutionManifestFactory = resolutionManifestFactory; + this.queryProvider = queryProvider; + } + + @Override + public Response listTableMetrics( + String catalogName, + String namespace, + String table, + String metricType, + String pageToken, + Integer pageSize, + Long snapshotId, + String principalName, + Long timestampFrom, + Long timestampTo, + RealmContext realmContext, + SecurityContext securityContext) { + + Namespace ns = decodeNamespace(namespace); + TableIdentifier identifier = TableIdentifier.of(ns, table); + + PolarisResolutionManifest manifest = resolveAndAuthorizeTableMetrics(catalogName, identifier); + + if (!queryProvider.isResolvable()) { + return Response.status(Response.Status.NOT_IMPLEMENTED) + .entity( + "Durable metrics query is not available in this deployment. " + + "Install the polaris-extensions-metrics-reports-jdbc extension to enable it.") + .build(); + } + + CatalogEntity catalogEntity = manifest.getResolvedCatalogEntity(); + long catalogId = catalogEntity != null ? catalogEntity.getId() : -1L; + PolarisResolvedPathWrapper tableWrapper = + manifest.getResolvedPath( + ResolvedPathKey.ofTableLike(identifier), PolarisEntitySubType.ANY_SUBTYPE, true); + long tableId = tableWrapper.getRawLeafEntity().getId(); + + PageToken pt = PageToken.build(pageToken, pageSize, () -> true); + MetricsQuerySpi provider = queryProvider.get(); + + if ("commit".equalsIgnoreCase(metricType)) { + Page page = + provider.listCommitReports( + catalogId, tableId, snapshotId, principalName, timestampFrom, timestampTo, pt); + List reports = + page.items().stream().map(MetricsReportsService::toCommitReport).toList(); + return Response.ok( + new ListCommitMetricsResponse( + page.encodedResponseToken(), + ListCommitMetricsResponse.MetricTypeEnum.COMMIT, + reports)) + .build(); + } + + Page page = + provider.listScanReports( + catalogId, tableId, snapshotId, principalName, timestampFrom, timestampTo, pt); + List reports = + page.items().stream().map(MetricsReportsService::toScanReport).toList(); + return Response.ok( + new ListScanMetricsResponse( + page.encodedResponseToken(), ListScanMetricsResponse.MetricTypeEnum.SCAN, reports)) + .build(); + } + + private PolarisResolutionManifest resolveAndAuthorizeTableMetrics( + String catalogName, TableIdentifier identifier) { + PolarisResolutionManifest manifest = + resolutionManifestFactory.createResolutionManifest(polarisPrincipal, catalogName); + manifest.addPassthroughPath( + new ResolverPath( + Arrays.asList(identifier.namespace().levels()), PolarisEntityType.NAMESPACE)); + manifest.addPassthroughPath( + new ResolverPath( + PolarisCatalogHelpers.tableIdentifierToList(identifier), PolarisEntityType.TABLE_LIKE)); + ResolverStatus status = manifest.resolveAll(); + + if (status.getStatus() == ResolverStatus.StatusEnum.ENTITY_COULD_NOT_BE_RESOLVED) { + throw new NotFoundException( + "TopLevelEntity of type %s does not exist: %s", + status.getFailedToResolvedEntityType(), status.getFailedToResolvedEntityName()); + } + if (status.getStatus() == ResolverStatus.StatusEnum.PATH_COULD_NOT_BE_FULLY_RESOLVED) { + throw new NotFoundException("Table not found: %s", identifier); + } + + PolarisResolvedPathWrapper tableWrapper = + manifest.getResolvedPath( + ResolvedPathKey.ofTableLike(identifier), PolarisEntitySubType.ANY_SUBTYPE, true); + + if (tableWrapper == null) { + throw new NotFoundException("Table not found: %s", identifier); + } + + authorizer.authorizeOrThrow( + polarisPrincipal, + manifest.getAllActivatedCatalogRoleAndPrincipalRoles(), + PolarisAuthorizableOperation.LIST_TABLE_METRICS, + tableWrapper, + null); + + return manifest; + } + + private static ScanMetricsReport toScanReport(ScanMetricsRecord r) { + MetricsActor actor = r.principalName() != null ? new MetricsActor(r.principalName()) : null; + MetricsRequest request = + (r.requestId() != null || r.otelTraceId() != null || r.otelSpanId() != null) + ? new MetricsRequest(r.requestId(), r.otelTraceId(), r.otelSpanId()) + : null; + ScanMetricsObject object = new ScanMetricsObject(r.snapshotId().orElse(null)); + ScanPayloadData data = + ScanPayloadData.builder() + .setSchemaId(r.schemaId().orElse(null)) + .setFilterExpression(r.filterExpression().orElse(null)) + .setProjectedFieldIds(r.projectedFieldIds()) + .setProjectedFieldNames(r.projectedFieldNames()) + .setResultDataFiles(r.resultDataFiles()) + .setResultDeleteFiles(r.resultDeleteFiles()) + .setTotalFileSizeBytes(r.totalFileSizeBytes()) + .setTotalDataManifests(r.totalDataManifests()) + .setTotalDeleteManifests(r.totalDeleteManifests()) + .setScannedDataManifests(r.scannedDataManifests()) + .setScannedDeleteManifests(r.scannedDeleteManifests()) + .setSkippedDataManifests(r.skippedDataManifests()) + .setSkippedDeleteManifests(r.skippedDeleteManifests()) + .setSkippedDataFiles(r.skippedDataFiles()) + .setSkippedDeleteFiles(r.skippedDeleteFiles()) + .setTotalPlanningDurationMs(r.totalPlanningDurationMs()) + .setEqualityDeleteFiles(r.equalityDeleteFiles()) + .setPositionalDeleteFiles(r.positionalDeleteFiles()) + .setIndexedDeleteFiles(r.indexedDeleteFiles()) + .setTotalDeleteFileSizeBytes(r.totalDeleteFileSizeBytes()) + .build(); + ScanPayload payload = + new ScanPayload( + ScanPayload.TypeEnum.ICEBERG_METRICS_SCAN, ScanPayload.VersionEnum.NUMBER_1, data); + return new ScanMetricsReport( + r.reportId(), r.timestamp().toEpochMilli(), actor, request, object, payload); + } + + private static CommitMetricsReport toCommitReport(CommitMetricsRecord r) { + MetricsActor actor = r.principalName() != null ? new MetricsActor(r.principalName()) : null; + MetricsRequest request = + (r.requestId() != null || r.otelTraceId() != null || r.otelSpanId() != null) + ? new MetricsRequest(r.requestId(), r.otelTraceId(), r.otelSpanId()) + : null; + CommitMetricsObject object = new CommitMetricsObject(r.snapshotId()); + CommitPayloadData data = + CommitPayloadData.builder() + .setSequenceNumber(r.sequenceNumber().orElse(null)) + .setOperation(r.operation()) + .setAddedDataFiles(r.addedDataFiles()) + .setRemovedDataFiles(r.removedDataFiles()) + .setTotalDataFiles(r.totalDataFiles()) + .setAddedDeleteFiles(r.addedDeleteFiles()) + .setRemovedDeleteFiles(r.removedDeleteFiles()) + .setTotalDeleteFiles(r.totalDeleteFiles()) + .setAddedEqualityDeleteFiles(r.addedEqualityDeleteFiles()) + .setRemovedEqualityDeleteFiles(r.removedEqualityDeleteFiles()) + .setAddedPositionalDeleteFiles(r.addedPositionalDeleteFiles()) + .setRemovedPositionalDeleteFiles(r.removedPositionalDeleteFiles()) + .setAddedRecords(r.addedRecords()) + .setRemovedRecords(r.removedRecords()) + .setTotalRecords(r.totalRecords()) + .setAddedFileSizeBytes(r.addedFileSizeBytes()) + .setRemovedFileSizeBytes(r.removedFileSizeBytes()) + .setTotalFileSizeBytes(r.totalFileSizeBytes()) + .setTotalDurationMs(r.totalDurationMs().orElse(null)) + .setAttempts(r.attempts()) + .build(); + CommitPayload payload = + new CommitPayload( + CommitPayload.TypeEnum.ICEBERG_METRICS_COMMIT, + CommitPayload.VersionEnum.NUMBER_1, + data); + return new CommitMetricsReport( + r.reportId(), r.timestamp().toEpochMilli(), actor, request, object, payload); + } + + private static Namespace decodeNamespace(String encodedNamespace) { + if (encodedNamespace == null || encodedNamespace.isEmpty()) { + throw new IllegalArgumentException("namespace must not be empty"); + } + return NamespaceUtils.splitNamespace( + encodedNamespace, NamespaceUtils.DEFAULT_NAMESPACE_SEPARATOR); + } +} diff --git a/runtime/service/src/main/java/org/apache/polaris/service/reporting/MetricsReportingConfiguration.java b/runtime/service/src/main/java/org/apache/polaris/service/reporting/MetricsReportingConfiguration.java index 3d60302ab3f..86add73fa4b 100644 --- a/runtime/service/src/main/java/org/apache/polaris/service/reporting/MetricsReportingConfiguration.java +++ b/runtime/service/src/main/java/org/apache/polaris/service/reporting/MetricsReportingConfiguration.java @@ -23,6 +23,6 @@ @ConfigMapping(prefix = "polaris.iceberg-metrics.reporting") public interface MetricsReportingConfiguration { - @WithDefault("default") + @WithDefault("log") String type(); } diff --git a/runtime/service/src/test/java/org/apache/polaris/service/catalog/iceberg/IcebergCatalogHandlerTest.java b/runtime/service/src/test/java/org/apache/polaris/service/catalog/iceberg/IcebergCatalogHandlerTest.java index ebf1feac31b..0d429141de9 100644 --- a/runtime/service/src/test/java/org/apache/polaris/service/catalog/iceberg/IcebergCatalogHandlerTest.java +++ b/runtime/service/src/test/java/org/apache/polaris/service/catalog/iceberg/IcebergCatalogHandlerTest.java @@ -60,12 +60,12 @@ import org.apache.polaris.core.persistence.resolver.ResolutionManifestFactory; import org.apache.polaris.core.persistence.resolver.ResolverFactory; import org.apache.polaris.core.storage.StorageAccessConfig; +import org.apache.polaris.extension.metrics.IcebergMetricsReporter; import org.apache.polaris.service.catalog.AccessDelegationModeResolver; import org.apache.polaris.service.catalog.CatalogPrefixParser; import org.apache.polaris.service.catalog.io.StorageAccessConfigProvider; import org.apache.polaris.service.config.ReservedProperties; import org.apache.polaris.service.events.EventAttributeMap; -import org.apache.polaris.service.reporting.PolarisMetricsReporter; import org.junit.jupiter.api.Test; class IcebergCatalogHandlerTest { @@ -126,7 +126,7 @@ private IcebergCatalogHandler newHandler() { .catalogHandlerUtils(mock(CatalogHandlerUtils.class)) .storageAccessConfigProvider(storageAccessConfigProvider) .eventAttributeMap(mock(EventAttributeMap.class)) - .metricsReporter(mock(PolarisMetricsReporter.class)) + .metricsReporter(mock(IcebergMetricsReporter.class)) .clock(mock(Clock.class)) .accessDelegationModeResolver(accessDelegationModeResolver) .build(); diff --git a/runtime/service/src/test/java/org/apache/polaris/service/metrics/MetricsReportsServiceTest.java b/runtime/service/src/test/java/org/apache/polaris/service/metrics/MetricsReportsServiceTest.java new file mode 100644 index 00000000000..add7e4c3fe7 --- /dev/null +++ b/runtime/service/src/test/java/org/apache/polaris/service/metrics/MetricsReportsServiceTest.java @@ -0,0 +1,255 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.polaris.service.metrics; + +import static org.assertj.core.api.Assertions.assertThat; +import static org.assertj.core.api.Assertions.assertThatThrownBy; +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.ArgumentMatchers.eq; +import static org.mockito.ArgumentMatchers.isNull; +import static org.mockito.Mockito.doNothing; +import static org.mockito.Mockito.doThrow; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.when; + +import jakarta.enterprise.inject.Instance; +import jakarta.ws.rs.ForbiddenException; +import jakarta.ws.rs.core.Response; +import jakarta.ws.rs.core.SecurityContext; +import java.util.List; +import java.util.Set; +import org.apache.iceberg.exceptions.NotFoundException; +import org.apache.polaris.core.auth.PolarisAuthorizableOperation; +import org.apache.polaris.core.auth.PolarisAuthorizer; +import org.apache.polaris.core.auth.PolarisPrincipal; +import org.apache.polaris.core.context.RealmContext; +import org.apache.polaris.core.entity.PolarisEntitySubType; +import org.apache.polaris.core.entity.PolarisEntityType; +import org.apache.polaris.core.persistence.PolarisResolvedPathWrapper; +import org.apache.polaris.core.persistence.metrics.MetricsQuerySpi; +import org.apache.polaris.core.persistence.resolver.PolarisResolutionManifest; +import org.apache.polaris.core.persistence.resolver.ResolutionManifestFactory; +import org.apache.polaris.core.persistence.resolver.ResolvedPathKey; +import org.apache.polaris.core.persistence.resolver.ResolverPath; +import org.apache.polaris.core.persistence.resolver.ResolverStatus; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; + +/** + * Unit tests for {@link MetricsReportsService}. + * + *

The read path currently returns 501 Not Implemented pending the durable extension (#4756). + * These tests cover authorization, resolution error paths, and input validation. + */ +class MetricsReportsServiceTest { + + private static final String CATALOG = "test-catalog"; + private static final String NAMESPACE = "dbschema"; + private static final String TABLE = "events"; + + private PolarisAuthorizer authorizer; + private PolarisResolutionManifest manifest; + private PolarisPrincipal principal; + private ResolutionManifestFactory factory; + private MetricsReportsService service; + private RealmContext realmContext; + private SecurityContext securityContext; + private Instance queryProvider; + + @BeforeEach + void setUp() { + authorizer = mock(PolarisAuthorizer.class); + principal = mock(PolarisPrincipal.class); + + PolarisResolvedPathWrapper tableWrapper = mock(PolarisResolvedPathWrapper.class); + manifest = mock(PolarisResolutionManifest.class); + factory = mock(ResolutionManifestFactory.class); + realmContext = mock(RealmContext.class); + securityContext = mock(SecurityContext.class); + + when(manifest.resolveAll()).thenReturn(new ResolverStatus(ResolverStatus.StatusEnum.SUCCESS)); + when(manifest.getResolvedPath( + any(ResolvedPathKey.class), eq(PolarisEntitySubType.ANY_SUBTYPE), eq(true))) + .thenReturn(tableWrapper); + when(manifest.getAllActivatedCatalogRoleAndPrincipalRoles()).thenReturn(Set.of()); + when(factory.createResolutionManifest(eq(principal), eq(CATALOG))).thenReturn(manifest); + doNothing() + .when(authorizer) + .authorizeOrThrow( + any(PolarisPrincipal.class), + any(Set.class), + any(PolarisAuthorizableOperation.class), + any(PolarisResolvedPathWrapper.class), + (PolarisResolvedPathWrapper) isNull()); + + @SuppressWarnings("unchecked") + Instance noOpProvider = mock(Instance.class); + when(noOpProvider.isResolvable()).thenReturn(false); + queryProvider = noOpProvider; + + service = new MetricsReportsService(authorizer, principal, factory, queryProvider); + realmContext = mock(RealmContext.class); + securityContext = mock(SecurityContext.class); + } + + @Test + void authorizedRequestReturnsNotImplemented() { + Response response = + service.listTableMetrics( + CATALOG, + NAMESPACE, + TABLE, + "scan", + null, + 10, + null, + null, + null, + null, + realmContext, + securityContext); + + assertThat(response.getStatus()).isEqualTo(Response.Status.NOT_IMPLEMENTED.getStatusCode()); + } + + @Test + void unauthorizedRequestThrowsForbiddenException() { + doThrow(new ForbiddenException("denied")) + .when(authorizer) + .authorizeOrThrow( + any(PolarisPrincipal.class), + any(Set.class), + eq(PolarisAuthorizableOperation.LIST_TABLE_METRICS), + any(PolarisResolvedPathWrapper.class), + (PolarisResolvedPathWrapper) isNull()); + + assertThatThrownBy( + () -> + service.listTableMetrics( + CATALOG, + NAMESPACE, + TABLE, + "scan", + null, + 10, + null, + null, + null, + null, + realmContext, + securityContext)) + .isInstanceOf(ForbiddenException.class); + } + + @Test + void tableNotFoundThrowsNotFoundException() { + when(manifest.getResolvedPath( + any(ResolvedPathKey.class), eq(PolarisEntitySubType.ANY_SUBTYPE), eq(true))) + .thenReturn(null); + + assertThatThrownBy( + () -> + service.listTableMetrics( + CATALOG, + NAMESPACE, + TABLE, + "scan", + null, + 10, + null, + null, + null, + null, + realmContext, + securityContext)) + .isInstanceOf(NotFoundException.class) + .hasMessageContaining(TABLE); + } + + @Test + void catalogNotFoundPropagatesNotFoundException() { + when(manifest.resolveAll()).thenReturn(new ResolverStatus(PolarisEntityType.CATALOG, CATALOG)); + + assertThatThrownBy( + () -> + service.listTableMetrics( + CATALOG, + NAMESPACE, + TABLE, + "scan", + null, + 10, + null, + null, + null, + null, + realmContext, + securityContext)) + .isInstanceOf(NotFoundException.class) + .hasMessageContaining(CATALOG); + } + + @Test + void pathNotFoundPropagatesNotFoundException() { + ResolverPath failedPath = new ResolverPath(List.of(NAMESPACE), PolarisEntityType.NAMESPACE); + when(manifest.resolveAll()).thenReturn(new ResolverStatus(failedPath, 0)); + + assertThatThrownBy( + () -> + service.listTableMetrics( + CATALOG, + NAMESPACE, + TABLE, + "scan", + null, + 10, + null, + null, + null, + null, + realmContext, + securityContext)) + .isInstanceOf(NotFoundException.class); + } + + @Test + void multiLevelNamespaceIsSplitCorrectly() { + // JAX-RS decodes %1F -> U+001F before injection; the service must split correctly. + // "dbschema" represents namespace ["db", "schema"]. + String encodedTwoLevel = "dbschema"; + when(factory.createResolutionManifest(eq(principal), eq(CATALOG))).thenReturn(manifest); + + Response response = + service.listTableMetrics( + CATALOG, + encodedTwoLevel, + TABLE, + "scan", + null, + 10, + null, + null, + null, + null, + realmContext, + securityContext); + + assertThat(response.getStatus()).isEqualTo(Response.Status.NOT_IMPLEMENTED.getStatusCode()); + } +} diff --git a/runtime/service/src/test/java/org/apache/polaris/service/reporting/PersistingMetricsReporterTest.java b/runtime/service/src/test/java/org/apache/polaris/service/reporting/PersistingMetricsReporterTest.java deleted file mode 100644 index 88face5b245..00000000000 --- a/runtime/service/src/test/java/org/apache/polaris/service/reporting/PersistingMetricsReporterTest.java +++ /dev/null @@ -1,172 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ -package org.apache.polaris.service.reporting; - -import static org.assertj.core.api.Assertions.assertThat; -import static org.mockito.ArgumentMatchers.any; -import static org.mockito.Mockito.mock; -import static org.mockito.Mockito.never; -import static org.mockito.Mockito.verify; -import static org.mockito.Mockito.when; - -import jakarta.enterprise.inject.Instance; -import java.time.Instant; -import java.util.Map; -import org.apache.iceberg.catalog.Namespace; -import org.apache.iceberg.catalog.TableIdentifier; -import org.apache.iceberg.expressions.Expressions; -import org.apache.iceberg.metrics.CommitMetrics; -import org.apache.iceberg.metrics.CommitMetricsResult; -import org.apache.iceberg.metrics.CommitReport; -import org.apache.iceberg.metrics.ImmutableCommitReport; -import org.apache.iceberg.metrics.ImmutableScanReport; -import org.apache.iceberg.metrics.MetricsReport; -import org.apache.iceberg.metrics.ScanMetrics; -import org.apache.iceberg.metrics.ScanMetricsResult; -import org.apache.iceberg.metrics.ScanReport; -import org.apache.polaris.core.PolarisCallContext; -import org.apache.polaris.core.auth.PolarisPrincipal; -import org.apache.polaris.core.context.CallContext; -import org.apache.polaris.core.context.RequestIdSupplier; -import org.apache.polaris.core.persistence.PolarisMetaStoreManager; -import org.apache.polaris.core.persistence.metrics.CommitMetricsRecord; -import org.apache.polaris.core.persistence.metrics.ScanMetricsRecord; -import org.junit.jupiter.api.BeforeEach; -import org.junit.jupiter.api.Test; -import org.mockito.ArgumentCaptor; - -/** - * Tests for {@link PersistingMetricsReporter}. - * - *

Note: The reporter now receives catalogId and tableId directly from the caller (already - * resolved during authorization in IcebergCatalogHandler), so there's no need to mock entity - * lookups. The reporter uses {@link PolarisMetaStoreManager} to persist metrics. - */ -public class PersistingMetricsReporterTest { - - private static final String CATALOG_NAME = "test-catalog"; - private static final long CATALOG_ID = 12345L; - private static final long TABLE_ID = 67890L; - private static final String TABLE_NAME = "test_table"; - private static final TableIdentifier TABLE_IDENTIFIER = - TableIdentifier.of(Namespace.of("db", "schema"), TABLE_NAME); - - private PolarisMetaStoreManager metaStoreManager; - private PolarisCallContext polarisCallContext; - private PersistingMetricsReporter reporter; - - @SuppressWarnings("unchecked") - @BeforeEach - void setUp() { - // Mock PolarisMetaStoreManager - metaStoreManager = mock(PolarisMetaStoreManager.class); - - // Mock CallContext - CallContext callContext = mock(CallContext.class); - polarisCallContext = mock(PolarisCallContext.class); - when(callContext.getPolarisCallContext()).thenReturn(polarisCallContext); - - // Mock Instance beans (not resolvable in test context) - Instance principalInstance = mock(Instance.class); - when(principalInstance.isResolvable()).thenReturn(false); - - Instance requestIdInstance = mock(Instance.class); - when(requestIdInstance.isResolvable()).thenReturn(false); - - reporter = - new PersistingMetricsReporter( - callContext, metaStoreManager, principalInstance, requestIdInstance); - } - - @Test - void testReportScanMetrics() { - // Create a scan report - ScanReport scanReport = createScanReport(); - - // Call the reporter with pre-resolved IDs - reporter.reportMetric( - CATALOG_NAME, CATALOG_ID, TABLE_IDENTIFIER, TABLE_ID, scanReport, Instant.now()); - - // Verify metaStoreManager was called with correct record - ArgumentCaptor captor = ArgumentCaptor.forClass(ScanMetricsRecord.class); - verify(metaStoreManager).writeScanMetrics(any(PolarisCallContext.class), captor.capture()); - - ScanMetricsRecord record = captor.getValue(); - assertThat(record.catalogId()).isEqualTo(CATALOG_ID); - assertThat(record.tableId()).isEqualTo(TABLE_ID); - assertThat(record.reportId()).isNotNull(); - } - - @Test - void testReportCommitMetrics() { - // Create a commit report - CommitReport commitReport = createCommitReport(); - - // Call the reporter with pre-resolved IDs - reporter.reportMetric( - CATALOG_NAME, CATALOG_ID, TABLE_IDENTIFIER, TABLE_ID, commitReport, Instant.now()); - - // Verify metaStoreManager was called with correct record - ArgumentCaptor captor = ArgumentCaptor.forClass(CommitMetricsRecord.class); - verify(metaStoreManager).writeCommitMetrics(any(PolarisCallContext.class), captor.capture()); - - CommitMetricsRecord record = captor.getValue(); - assertThat(record.catalogId()).isEqualTo(CATALOG_ID); - assertThat(record.tableId()).isEqualTo(TABLE_ID); - assertThat(record.reportId()).isNotNull(); - } - - @Test - void testUnknownReportType() { - // Create an unknown report type (using a mock) - MetricsReport unknownReport = mock(MetricsReport.class); - - // Call the reporter - should not throw - reporter.reportMetric( - CATALOG_NAME, CATALOG_ID, TABLE_IDENTIFIER, TABLE_ID, unknownReport, Instant.now()); - - // Verify metaStoreManager was NOT called since report type is unknown - verify(metaStoreManager, never()).writeScanMetrics(any(), any()); - verify(metaStoreManager, never()).writeCommitMetrics(any(), any()); - } - - private ScanReport createScanReport() { - return ImmutableScanReport.builder() - .tableName("db.schema.test_table") - .snapshotId(123456789L) - .schemaId(1) - .filter(Expressions.alwaysTrue()) - .scanMetrics(ScanMetricsResult.fromScanMetrics(ScanMetrics.noop())) - .build(); - } - - private CommitReport createCommitReport() { - CommitMetrics commitMetrics = - CommitMetrics.of(new org.apache.iceberg.metrics.DefaultMetricsContext()); - CommitMetricsResult metricsResult = CommitMetricsResult.from(commitMetrics, Map.of()); - - return ImmutableCommitReport.builder() - .tableName("db.schema.test_table") - .snapshotId(987654321L) - .sequenceNumber(5L) - .operation("append") - .commitMetrics(metricsResult) - .build(); - } -} diff --git a/runtime/service/src/testFixtures/java/org/apache/polaris/service/TestServices.java b/runtime/service/src/testFixtures/java/org/apache/polaris/service/TestServices.java index ce52a212023..cce013ff38c 100644 --- a/runtime/service/src/testFixtures/java/org/apache/polaris/service/TestServices.java +++ b/runtime/service/src/testFixtures/java/org/apache/polaris/service/TestServices.java @@ -99,7 +99,6 @@ import org.apache.polaris.service.events.listeners.InMemoryEventCollector; import org.apache.polaris.service.identity.provider.DefaultServiceIdentityProvider; import org.apache.polaris.service.persistence.InMemoryPolarisMetaStoreManagerFactory; -import org.apache.polaris.service.reporting.DefaultMetricsReporter; import org.apache.polaris.service.secrets.UnsafeInMemorySecretsManagerFactory; import org.apache.polaris.service.storage.PolarisStorageIntegrationProviderImpl; import org.apache.polaris.service.task.TaskExecutor; @@ -359,7 +358,7 @@ public IcebergCatalogHandler createHandler( .federatedCatalogFactories(federatedCatalogFactory) .storageAccessConfigProvider(storageAccessConfigProvider) .eventAttributeMap(eventAttributeMap) - .metricsReporter(new DefaultMetricsReporter()) + .metricsReporter((_catName, _catId, _table, _tableId, _report, _ts) -> {}) .clock(clock) .accessDelegationModeResolver( new DefaultAccessDelegationModeResolver(realmConfig)) diff --git a/site/content/in-dev/unreleased/configuration/config-sections/smallrye-polaris_iceberg_metrics_reporting.md b/site/content/in-dev/unreleased/configuration/config-sections/smallrye-polaris_iceberg_metrics_reporting.md index eb3a21a0dc8..a118b4720bf 100644 --- a/site/content/in-dev/unreleased/configuration/config-sections/smallrye-polaris_iceberg_metrics_reporting.md +++ b/site/content/in-dev/unreleased/configuration/config-sections/smallrye-polaris_iceberg_metrics_reporting.md @@ -25,4 +25,4 @@ build: | Property | Default Value | Type | Description | |----------|---------------|------|-------------| -| `polaris.iceberg-metrics.reporting.type` | `default` | `string` | | +| `polaris.iceberg-metrics.reporting.type` | `log` | `string` | | diff --git a/site/content/in-dev/unreleased/managing-security/access-control.md b/site/content/in-dev/unreleased/managing-security/access-control.md index 40752651b30..6bb45888649 100644 --- a/site/content/in-dev/unreleased/managing-security/access-control.md +++ b/site/content/in-dev/unreleased/managing-security/access-control.md @@ -115,6 +115,7 @@ To grant the full set of privileges (drop, list, read, write, etc.) on an object | TABLE_WRITE_PROPERTIES | Enables configuring properties for the table. | | TABLE_READ_DATA | Enables reading data from the table by receiving short-lived read-only storage credentials from the catalog. | | TABLE_WRITE_DATA | Enables writing data to the table by receiving short-lived read+write storage credentials from the catalog. | +| TABLE_READ_METRICS | Enables reading persisted Iceberg scan and commit metrics reports for the table via the Metrics Reports API. | | TABLE_FULL_METADATA | Grants all table privileges, except TABLE_READ_DATA and TABLE_WRITE_DATA, which need to be granted individually. | | TABLE_ATTACH_POLICY | Enables attaching policy to a table. | | TABLE_DETACH_POLICY | Enables detaching policy from a table. | diff --git a/site/content/in-dev/unreleased/polaris-api-specs/polaris-metrics-reports-api.md b/site/content/in-dev/unreleased/polaris-api-specs/polaris-metrics-reports-api.md new file mode 100644 index 00000000000..c4cc1f983a0 --- /dev/null +++ b/site/content/in-dev/unreleased/polaris-api-specs/polaris-metrics-reports-api.md @@ -0,0 +1,27 @@ +--- +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# +title: 'Apache Polaris Metrics Reports Service OpenAPI Specification' +linkTitle: 'Metrics Reports API ↗' +weight: 300 +params: + show_page_toc: false +--- + +{{< redoc-polaris "metrics-reports-service.yml" >}} diff --git a/site/content/in-dev/unreleased/telemetry.md b/site/content/in-dev/unreleased/telemetry.md index 59a9b763ee6..1d5bc249412 100644 --- a/site/content/in-dev/unreleased/telemetry.md +++ b/site/content/in-dev/unreleased/telemetry.md @@ -191,6 +191,37 @@ polaris.log.mdc.region=us-west-2 MDC context is propagated across threads, including in `TaskExecutor` threads. +## Iceberg Metrics Reports API + +Polaris collects Iceberg scan and commit metrics reports submitted by clients and routes them to a +configured reporter. Two built-in reporters are provided: + +| Type | Description | +|------|-------------| +| `log` (default) | Logs each report at INFO level via SLF4J. | +| `no-op` | Silently discards all reports. | + +The active reporter is selected with: + +```properties +polaris.iceberg-metrics.reporting.type=log +``` + +### Durable persistence (extension) + +Durable storage of metrics reports and querying via the REST API at +`/api/metrics-reports/v1/catalogs/{catalogName}/namespaces/{namespace}/tables/{table}` requires the +`polaris-extensions-metrics-reports-jdbc` extension. Without it, the query endpoint returns HTTP +501. See the extension documentation for installation and configuration details. + +### Prerequisites for the query API + +The caller must hold the `TABLE_READ_METRICS` privilege on the target table (or a privilege that +implies it, such as `TABLE_READ_DATA`, `TABLE_FULL_METADATA`, or `CATALOG_MANAGE_CONTENT`). + +See the [Metrics Reports API specification]({{% relref "polaris-api-specs/polaris-metrics-reports-api" %}}) +for the full schema reference. + ## Links Visit [Using Polaris with telemetry tools]({{% relref "getting-started/using-polaris/telemetry-tools" %}}) to see sample Polaris config with Prometheus and Jaeger. diff --git a/spec/metrics-reports-service.yml b/spec/metrics-reports-service.yml new file mode 100644 index 00000000000..bbebd05d5eb --- /dev/null +++ b/spec/metrics-reports-service.yml @@ -0,0 +1,484 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# + +openapi: 3.0.3 +info: + title: Apache Polaris Metrics Reports API + description: > + **Beta**: Read-only API for querying Iceberg table metrics (scan and commit reports) from Apache + Polaris. This API is in beta and may change in future releases. Requires TABLE_READ_METRICS + privilege on the target table. Durable persistence backing for this API is provided by the + polaris-extensions-metrics-reports-jdbc extension; without it the endpoint returns HTTP 501. + version: 0.1.0 + license: + name: Apache 2.0 + url: https://www.apache.org/licenses/LICENSE-2.0.html + +servers: + - url: "{scheme}://{host}/api/metrics-reports/v1" + variables: + scheme: + default: https + host: + default: localhost + +paths: + /catalogs/{catalogName}/namespaces/{namespace}/tables/{table}: + parameters: + - $ref: '#/components/parameters/catalogName' + - $ref: '#/components/parameters/namespace' + - $ref: '#/components/parameters/table' + get: + operationId: listTableMetrics + summary: List metrics reports for a table + description: > + Returns persisted metrics reports for the specified table. The required `metricType` + parameter selects between scan reports (produced during table reads) and commit reports + (produced during table writes). Results are ordered by timestamp descending. + Requires TABLE_READ_METRICS privilege on the target table. + tags: + - Metrics + parameters: + - name: metricType + in: query + required: true + description: Type of metrics to retrieve + schema: + type: string + enum: [scan, commit] + - name: pageToken + in: query + required: false + schema: + type: string + description: Opaque cursor from a previous response's nextPageToken field + - name: pageSize + in: query + required: false + schema: + type: integer + minimum: 1 + default: 100 + description: Maximum number of results to return per page + - name: snapshotId + in: query + required: false + schema: + type: integer + format: int64 + description: Filter results to a specific snapshot ID + - name: principalName + in: query + required: false + schema: + type: string + description: Filter results to a specific principal (e.g. service account name) + - name: timestampFrom + in: query + required: false + schema: + type: integer + format: int64 + description: Inclusive lower bound on report timestamp (Unix epoch milliseconds) + - name: timestampTo + in: query + required: false + schema: + type: integer + format: int64 + description: Exclusive upper bound on report timestamp (Unix epoch milliseconds) + responses: + '200': + description: Paginated list of metrics reports + content: + application/json: + schema: + $ref: '#/components/schemas/ListMetricsResponse' + '400': + description: Bad request (missing or invalid parameters) + content: + application/json: + schema: + $ref: '#/components/schemas/ErrorResponse' + '403': + description: Insufficient privileges (TABLE_READ_METRICS required) + content: + application/json: + schema: + $ref: '#/components/schemas/ErrorResponse' + '404': + description: Catalog, namespace, or table not found + content: + application/json: + schema: + $ref: '#/components/schemas/ErrorResponse' + +components: + parameters: + catalogName: + name: catalogName + in: path + required: true + schema: + type: string + namespace: + name: namespace + in: path + required: true + description: > + Namespace path encoded using the same convention as the Polaris Iceberg REST API: + multi-level namespaces use the unit separator (0x1F) between levels, URL-encoded as %1F. + schema: + type: string + table: + name: table + in: path + required: true + schema: + type: string + + schemas: + ListMetricsResponse: + description: > + Polymorphic response for metrics queries. The concrete type is determined by the + metricType discriminator field, which echoes the requested metricType query parameter. + oneOf: + - $ref: '#/components/schemas/ListScanMetricsResponse' + - $ref: '#/components/schemas/ListCommitMetricsResponse' + discriminator: + propertyName: metricType + mapping: + scan: '#/components/schemas/ListScanMetricsResponse' + commit: '#/components/schemas/ListCommitMetricsResponse' + + ListScanMetricsResponse: + type: object + required: + - metricType + - reports + properties: + nextPageToken: + type: string + nullable: true + description: > + Opaque cursor for fetching the next page. Null or absent when no further pages exist. + metricType: + type: string + enum: [scan] + description: Discriminator — always "scan" for this response type + reports: + type: array + items: + $ref: '#/components/schemas/ScanMetricsReport' + + ListCommitMetricsResponse: + type: object + required: + - metricType + - reports + properties: + nextPageToken: + type: string + nullable: true + description: > + Opaque cursor for fetching the next page. Null or absent when no further pages exist. + metricType: + type: string + enum: [commit] + description: Discriminator — always "commit" for this response type + reports: + type: array + items: + $ref: '#/components/schemas/CommitMetricsReport' + + MetricsActor: + type: object + description: Identity of the principal who triggered the operation + properties: + principalName: + type: string + nullable: true + + MetricsRequest: + type: object + description: Request context for correlation with logs and traces + properties: + requestId: + type: string + nullable: true + otelTraceId: + type: string + nullable: true + description: OpenTelemetry trace ID + otelSpanId: + type: string + nullable: true + description: OpenTelemetry span ID + + ScanMetricsObject: + type: object + description: Resource context for the scanned table operation + properties: + snapshotId: + type: integer + format: int64 + nullable: true + + CommitMetricsObject: + type: object + description: Resource context for the committed table operation + required: + - snapshotId + properties: + snapshotId: + type: integer + format: int64 + + ScanPayloadData: + type: object + description: Iceberg scan metrics data + properties: + schemaId: + type: integer + nullable: true + filterExpression: + type: string + nullable: true + projectedFieldIds: + type: array + nullable: true + items: + type: integer + description: Projected field IDs + projectedFieldNames: + type: array + nullable: true + items: + type: string + description: Projected field names + resultDataFiles: + type: integer + format: int64 + resultDeleteFiles: + type: integer + format: int64 + totalFileSizeBytes: + type: integer + format: int64 + totalDataManifests: + type: integer + format: int64 + totalDeleteManifests: + type: integer + format: int64 + scannedDataManifests: + type: integer + format: int64 + scannedDeleteManifests: + type: integer + format: int64 + skippedDataManifests: + type: integer + format: int64 + skippedDeleteManifests: + type: integer + format: int64 + skippedDataFiles: + type: integer + format: int64 + skippedDeleteFiles: + type: integer + format: int64 + totalPlanningDurationMs: + type: integer + format: int64 + equalityDeleteFiles: + type: integer + format: int64 + positionalDeleteFiles: + type: integer + format: int64 + indexedDeleteFiles: + type: integer + format: int64 + totalDeleteFileSizeBytes: + type: integer + format: int64 + + ScanPayload: + type: object + required: + - type + - version + - data + properties: + type: + type: string + enum: [iceberg.metrics.scan] + version: + type: integer + enum: [1] + data: + $ref: '#/components/schemas/ScanPayloadData' + + CommitPayloadData: + type: object + description: Iceberg commit metrics data + properties: + sequenceNumber: + type: integer + format: int64 + nullable: true + operation: + type: string + description: Commit operation (append, overwrite, delete, replace) + addedDataFiles: + type: integer + format: int64 + removedDataFiles: + type: integer + format: int64 + totalDataFiles: + type: integer + format: int64 + addedDeleteFiles: + type: integer + format: int64 + removedDeleteFiles: + type: integer + format: int64 + totalDeleteFiles: + type: integer + format: int64 + addedEqualityDeleteFiles: + type: integer + format: int64 + removedEqualityDeleteFiles: + type: integer + format: int64 + addedPositionalDeleteFiles: + type: integer + format: int64 + removedPositionalDeleteFiles: + type: integer + format: int64 + addedRecords: + type: integer + format: int64 + removedRecords: + type: integer + format: int64 + totalRecords: + type: integer + format: int64 + addedFileSizeBytes: + type: integer + format: int64 + removedFileSizeBytes: + type: integer + format: int64 + totalFileSizeBytes: + type: integer + format: int64 + totalDurationMs: + type: integer + format: int64 + nullable: true + attempts: + type: integer + + CommitPayload: + type: object + required: + - type + - version + - data + properties: + type: + type: string + enum: [iceberg.metrics.commit] + version: + type: integer + enum: [1] + data: + $ref: '#/components/schemas/CommitPayloadData' + + ScanMetricsReport: + type: object + description: Stable envelope for a persisted Iceberg scan metrics report + required: + - id + - timestampMs + - object + - payload + properties: + id: + type: string + description: Unique identifier for this report + timestampMs: + type: integer + format: int64 + description: Server-side timestamp when the report was received (Unix epoch milliseconds) + actor: + $ref: '#/components/schemas/MetricsActor' + nullable: true + request: + $ref: '#/components/schemas/MetricsRequest' + nullable: true + object: + $ref: '#/components/schemas/ScanMetricsObject' + payload: + $ref: '#/components/schemas/ScanPayload' + + CommitMetricsReport: + type: object + description: Stable envelope for a persisted Iceberg commit metrics report + required: + - id + - timestampMs + - object + - payload + properties: + id: + type: string + description: Unique identifier for this report + timestampMs: + type: integer + format: int64 + description: Server-side timestamp when the report was received (Unix epoch milliseconds) + actor: + $ref: '#/components/schemas/MetricsActor' + nullable: true + request: + $ref: '#/components/schemas/MetricsRequest' + nullable: true + object: + $ref: '#/components/schemas/CommitMetricsObject' + payload: + $ref: '#/components/schemas/CommitPayload' + + ErrorResponse: + type: object + properties: + message: + type: string + type: + type: string + code: + type: integer