From 1e3a611d94848b89217b52abd081717b88c846ef Mon Sep 17 00:00:00 2001 From: Robert Stupp Date: Thu, 11 Jun 2026 17:53:30 +0200 Subject: [PATCH 01/22] Build: allow setting environment variables for the Gradle invocation This allows creating a `.env` file and for example enable the Gradle configuration cache via ``` GRADLE_OPTS='-Dorg.gradle.configuration-cache=true' ``` --- .gitignore | 2 ++ build.gradle.kts | 3 +++ gradlew | 1 + 3 files changed, 6 insertions(+) diff --git a/.gitignore b/.gitignore index bfaa0356c11..9a5ebaf9db5 100644 --- a/.gitignore +++ b/.gitignore @@ -111,6 +111,8 @@ venv # And then use `./gradlew run -Dquarkus.profile=local` to run Polaris with dev profile. application-local.properties +/.env + # AI / Agentic Development Tools .agents/ .aider/ diff --git a/build.gradle.kts b/build.gradle.kts index 9c8ff902d08..3e91577bb25 100644 --- a/build.gradle.kts +++ b/build.gradle.kts @@ -63,6 +63,8 @@ tasks.named("rat").configure { excludes.add("ide-name.txt") excludes.add("version.txt") + excludes.add(".env") + excludes.add("**/LICENSE*") excludes.add("**/NOTICE*") @@ -269,6 +271,7 @@ tasks.named("wrapper") { val insertAtLine = scriptLines.indexOf("# Use the maximum available, or set MAX_FD != -1 to use that value.") scriptLines.add(insertAtLine, "") + scriptLines.add(insertAtLine, $$"[ -f \"${APP_HOME}/.env\" ] && . \"${APP_HOME}/.env\"") scriptLines.add(insertAtLine, $$". \"${APP_HOME}/gradle/gradlew-include.sh\"") scriptFile.writeText(scriptLines.joinToString("\n")) diff --git a/gradlew b/gradlew index 47acbf2081a..5da0b6bbcaa 100755 --- a/gradlew +++ b/gradlew @@ -88,6 +88,7 @@ APP_BASE_NAME=${0##*/} # Discard cd standard output in case $CDPATH is set (https://github.com/gradle/gradle/issues/25036) APP_HOME=$( cd -P "${APP_HOME:-./}" > /dev/null && printf '%s\n' "$PWD" ) || exit +[ -f "${APP_HOME}/.env" ] && . "${APP_HOME}/.env" . "${APP_HOME}/gradle/gradlew-include.sh" # Use the maximum available, or set MAX_FD != -1 to use that value. From 02f6f9cdfcd8a632f0ca5a6bbcd0354b33cf97ed Mon Sep 17 00:00:00 2001 From: Robert Stupp Date: Tue, 9 Jun 2026 09:33:36 +0200 Subject: [PATCH 02/22] Add polaris-server-test-runner plugin This new plugin for test purposes mainly buys us separation and faster feedback: - Test projects no longer need to build or embed a full Quarkus test app just to run against Polaris. - Tests can depend on a ready `quarkus-run.jar` via `polarisServer(...)`, so the server lifecycle is externalized from the test JVM. - Less Quarkus application builds across all checks, improving overall test runtime. - Quarkus dependencies stop leaking into test classpaths, which reduces dependency conflicts with Spark, Iceberg, Hadoop, etc. - The plugin makes `enforcedPlatform(libs.quarkus.bom)` safer because Quarkus is mostly constrained to the app/server side instead of the tests. - Each test task can get its own isolated Polaris server instance and config. - Projects can use normal `JvmTestSuite`s, which improves Gradle/IntelliJ behavior and keeps test classpaths scoped. - Custom server startup hooks still allow task-specific setup, like OPA/Testcontainers or RustFS, without baking those concerns into the plugin. --- .gitignore | 2 + .../src/main/kotlin/polaris-root.gradle.kts | 8 +- gradle/server-test-runner/build.gradle.kts | 43 ++++ .../gradle/libs.versions.toml | 22 ++ gradle/server-test-runner/settings.gradle.kts | 24 ++ .../spi/PolarisServerStartupAction.java | 36 +++ .../spi/PolarisServerStartupContext.java | 33 +++ .../PolarisServerTestRunnerExtensions.kt | 107 ++++++++ .../PolarisServerTestRunnerExtension.kt | 136 ++++++++++ .../runner/PolarisServerTestRunnerPlugin.kt | 62 +++++ .../test/runner/PolarisServerTestService.kt | 54 ++++ .../runner/PolarisServerTestTaskConfigurer.kt | 194 ++++++++++++++ .../test/runner/RunningPolarisServer.kt | 167 ++++++++++++ .../PolarisServerTestRunnerPluginTest.kt | 243 ++++++++++++++++++ settings.gradle.kts | 2 + 15 files changed, 1132 insertions(+), 1 deletion(-) create mode 100644 gradle/server-test-runner/build.gradle.kts create mode 100644 gradle/server-test-runner/gradle/libs.versions.toml create mode 100644 gradle/server-test-runner/settings.gradle.kts create mode 100644 gradle/server-test-runner/src/main/java/org/apache/polaris/server/test/runner/spi/PolarisServerStartupAction.java create mode 100644 gradle/server-test-runner/src/main/java/org/apache/polaris/server/test/runner/spi/PolarisServerStartupContext.java create mode 100644 gradle/server-test-runner/src/main/kotlin/PolarisServerTestRunnerExtensions.kt create mode 100644 gradle/server-test-runner/src/main/kotlin/org/apache/polaris/server/test/runner/PolarisServerTestRunnerExtension.kt create mode 100644 gradle/server-test-runner/src/main/kotlin/org/apache/polaris/server/test/runner/PolarisServerTestRunnerPlugin.kt create mode 100644 gradle/server-test-runner/src/main/kotlin/org/apache/polaris/server/test/runner/PolarisServerTestService.kt create mode 100644 gradle/server-test-runner/src/main/kotlin/org/apache/polaris/server/test/runner/PolarisServerTestTaskConfigurer.kt create mode 100644 gradle/server-test-runner/src/main/kotlin/org/apache/polaris/server/test/runner/RunningPolarisServer.kt create mode 100644 gradle/server-test-runner/src/test/kotlin/org/apache/polaris/server/test/runner/PolarisServerTestRunnerPluginTest.kt diff --git a/.gitignore b/.gitignore index 9a5ebaf9db5..477e2406a78 100644 --- a/.gitignore +++ b/.gitignore @@ -69,6 +69,8 @@ gradle/wrapper/gradle-wrapper-*.sha256 /.gradle /build-logic/.gradle /build-logic/.kotlin +/gradle/server-test-runner/.gradle +/gradle/server-test-runner/.kotlin **/build/ !src/**/build/ diff --git a/build-logic/src/main/kotlin/polaris-root.gradle.kts b/build-logic/src/main/kotlin/polaris-root.gradle.kts index 4e9b4584c03..778cfd449c0 100644 --- a/build-logic/src/main/kotlin/polaris-root.gradle.kts +++ b/build-logic/src/main/kotlin/polaris-root.gradle.kts @@ -36,7 +36,13 @@ if (!project.extra.has("duplicated-project-sources")) { kotlinGradle { ktfmt().googleStyle() // licenseHeaderFile(rootProject.file("codestyle/copyright-header-java.txt"), "$") - target("*.gradle.kts", "build-logic/*.gradle.kts", "build-logic/src/**/*.kt*") + target( + "*.gradle.kts", + "build-logic/*.gradle.kts", + "build-logic/src/**/*.kt*", + "gradle/server-test-runner/**/*.gradle.kts", + "gradle/server-test-runner/src/**/*.kt*", + ) } } } diff --git a/gradle/server-test-runner/build.gradle.kts b/gradle/server-test-runner/build.gradle.kts new file mode 100644 index 00000000000..504ebff8fce --- /dev/null +++ b/gradle/server-test-runner/build.gradle.kts @@ -0,0 +1,43 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +plugins { + `kotlin-dsl` + `java-gradle-plugin` +} + +gradlePlugin { + plugins { + create("polarisServerTestRunner") { + id = "polaris-server-test-runner" + implementationClass = "org.apache.polaris.server.test.runner.PolarisServerTestRunnerPlugin" + } + } +} + +testing { suites { named("test") { useJUnitJupiter() } } } + +dependencies { + testImplementation(gradleTestKit()) + testImplementation(platform(libs.junit.bom)) + testImplementation("org.junit.jupiter:junit-jupiter") + testImplementation(libs.assertj.core) +} + +group = "org.apache.polaris.server-test-runner" diff --git a/gradle/server-test-runner/gradle/libs.versions.toml b/gradle/server-test-runner/gradle/libs.versions.toml new file mode 100644 index 00000000000..a8e5324ce7b --- /dev/null +++ b/gradle/server-test-runner/gradle/libs.versions.toml @@ -0,0 +1,22 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# + +[libraries] +assertj-core = { module = "org.assertj:assertj-core", version = "3.27.7" } +junit-bom = { module = "org.junit:junit-bom", version = "6.1.0" } diff --git a/gradle/server-test-runner/settings.gradle.kts b/gradle/server-test-runner/settings.gradle.kts new file mode 100644 index 00000000000..0f3d5c54557 --- /dev/null +++ b/gradle/server-test-runner/settings.gradle.kts @@ -0,0 +1,24 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +pluginManagement { repositories { gradlePluginPortal() } } + +dependencyResolutionManagement { repositories { mavenCentral() } } + +rootProject.name = "polaris-server-test-runner" diff --git a/gradle/server-test-runner/src/main/java/org/apache/polaris/server/test/runner/spi/PolarisServerStartupAction.java b/gradle/server-test-runner/src/main/java/org/apache/polaris/server/test/runner/spi/PolarisServerStartupAction.java new file mode 100644 index 00000000000..4301579999c --- /dev/null +++ b/gradle/server-test-runner/src/main/java/org/apache/polaris/server/test/runner/spi/PolarisServerStartupAction.java @@ -0,0 +1,36 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.polaris.server.test.runner.spi; + +/** + * Isolated startup action that can prepare external services before the Polaris server process + * starts. + * + *

Implementations are loaded from a child classloader configured by the build and may mutate the + * provided context to pass task-specific system properties or environment variables to the Polaris + * server. Implementations are closed after the decorated test task finishes, or when startup fails. + */ +public interface PolarisServerStartupAction extends AutoCloseable { + /** Starts required services and updates the Polaris server startup context. */ + void start(PolarisServerStartupContext context) throws Exception; + + /** Stops resources owned by this action. */ + @Override + default void close() throws Exception {} +} diff --git a/gradle/server-test-runner/src/main/java/org/apache/polaris/server/test/runner/spi/PolarisServerStartupContext.java b/gradle/server-test-runner/src/main/java/org/apache/polaris/server/test/runner/spi/PolarisServerStartupContext.java new file mode 100644 index 00000000000..8c1233aee4a --- /dev/null +++ b/gradle/server-test-runner/src/main/java/org/apache/polaris/server/test/runner/spi/PolarisServerStartupContext.java @@ -0,0 +1,33 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.polaris.server.test.runner.spi; + +import java.util.Map; + +/** Context passed to an isolated {@link PolarisServerStartupAction}. */ +public interface PolarisServerStartupContext { + /** Build-configured action parameters. */ + Map getParameters(); + + /** Mutable system properties that will be passed to the Polaris server JVM. */ + Map getSystemProperties(); + + /** Mutable environment variables that will be passed to the Polaris server process. */ + Map getEnvironment(); +} diff --git a/gradle/server-test-runner/src/main/kotlin/PolarisServerTestRunnerExtensions.kt b/gradle/server-test-runner/src/main/kotlin/PolarisServerTestRunnerExtensions.kt new file mode 100644 index 00000000000..e994a4c4754 --- /dev/null +++ b/gradle/server-test-runner/src/main/kotlin/PolarisServerTestRunnerExtensions.kt @@ -0,0 +1,107 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +import org.apache.polaris.server.test.runner.PolarisServerTestRunnerExtension +import org.apache.polaris.server.test.runner.PolarisServerTestRunnerPlugin +import org.apache.polaris.server.test.runner.configurePolarisServer +import org.apache.polaris.server.test.runner.conventionFrom +import org.gradle.api.Action +import org.gradle.api.file.FileCollection +import org.gradle.api.provider.Provider +import org.gradle.api.tasks.testing.Test +import org.gradle.process.CommandLineArgumentProvider + +/** + * Runs a Polaris server from [server] for the duration of this [Test] task. + * + * The file collection must resolve to exactly one runnable jar. The server is started only when the + * test task executes, not during configuration. The discovered HTTP and management ports are passed + * to the test JVM using the default property names from [PolarisServerTestRunnerExtension]. + */ +fun Test.withPolarisServer(server: FileCollection) { + withPolarisServer(server) {} +} + +/** + * Runs a Polaris server from the file collection provided by [server] for the duration of this + * [Test] task. + * + * This overload is convenient for lazily named configurations such as + * `withPolarisServer(configurations.polarisServer)`. + */ +fun Test.withPolarisServer(server: Provider) { + withPolarisServer(server) {} +} + +/** + * Runs a Polaris server from [server] for the duration of this [Test] task and customizes its + * process settings using [configure]. + * + * Use this overload when the server artifact is task-specific, for example + * `withPolarisServer(configurations.polarisServer) { ... }`. Dynamic port values are passed via a + * [CommandLineArgumentProvider] so they do not become test task inputs. + */ +fun Test.withPolarisServer( + server: FileCollection, + configure: Action, +) { + val extension = polarisServerTestRunnerExtension() + extension.server.setFrom(server) + configure.execute(extension) + configurePolarisServer(extension) +} + +/** + * Runs a Polaris server from the file collection provided by [server] for the duration of this + * [Test] task and customizes its process settings using [configure]. + * + * Use this overload when the server artifact is exposed through a lazy Gradle provider, for example + * `withPolarisServer(configurations.polarisServer) { ... }`. + */ +fun Test.withPolarisServer( + server: Provider, + configure: Action, +) { + val extension = polarisServerTestRunnerExtension() + extension.server.setFrom(server) + configure.execute(extension) + configurePolarisServer(extension) +} + +/** + * Runs a Polaris server for the duration of this [Test] task using the plugin extension's + * configured [PolarisServerTestRunnerExtension.server] artifact. + * + * Use this overload when the `polarisServerTestRunner` extension is configured globally and the + * test task only needs to opt into the server lifecycle. + */ +fun Test.withPolarisServer(configure: Action) { + val extension = polarisServerTestRunnerExtension() + configure.execute(extension) + configurePolarisServer(extension) +} + +private fun Test.polarisServerTestRunnerExtension(): PolarisServerTestRunnerExtension { + val projectExtension = + project.extensions.getByName(PolarisServerTestRunnerPlugin.POLARIS_SERVER_EXTENSION) + as PolarisServerTestRunnerExtension + return project.objects + .newInstance(PolarisServerTestRunnerExtension::class.java, project, projectExtension.service) + .apply { conventionFrom(projectExtension) } +} diff --git a/gradle/server-test-runner/src/main/kotlin/org/apache/polaris/server/test/runner/PolarisServerTestRunnerExtension.kt b/gradle/server-test-runner/src/main/kotlin/org/apache/polaris/server/test/runner/PolarisServerTestRunnerExtension.kt new file mode 100644 index 00000000000..24452ec92a1 --- /dev/null +++ b/gradle/server-test-runner/src/main/kotlin/org/apache/polaris/server/test/runner/PolarisServerTestRunnerExtension.kt @@ -0,0 +1,136 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.polaris.server.test.runner + +import java.time.Duration +import javax.inject.Inject +import org.gradle.api.Project +import org.gradle.api.file.ConfigurableFileCollection +import org.gradle.api.file.DirectoryProperty +import org.gradle.api.provider.ListProperty +import org.gradle.api.provider.MapProperty +import org.gradle.api.provider.Property +import org.gradle.api.provider.Provider + +/** + * Configuration for running a Polaris server process for Gradle test tasks. + * + * The plugin starts the configured server artifact before a decorated + * [Test][org.gradle.api.tasks.testing.Test] task executes and stops it when the task's root test + * suite completes. The discovered Quarkus HTTP and management ports are passed to the test JVM + * using the property names configured here. + */ +abstract class PolarisServerTestRunnerExtension +@Inject +constructor(project: Project, internal val service: Provider) { + /** Server artifact to execute. This must resolve to exactly one runnable `java -jar` artifact. */ + val server: ConfigurableFileCollection = project.objects.fileCollection() + + /** + * Classpath used to load the optional startup action in an isolated child classloader. + * + * The startup action can prepare external services before the Polaris server starts and can add + * dynamic server system properties or environment variables through the startup context. + */ + val startupActionClasspath: ConfigurableFileCollection = project.objects.fileCollection() + + /** + * Fully qualified implementation class name of the optional startup action. + * + * The class must implement [org.apache.polaris.server.test.runner.spi.PolarisServerStartupAction] + * and have a public no-argument constructor. + */ + val startupActionClass: Property = project.objects.property(String::class.java) + + /** String parameters passed to the optional startup action. */ + val startupActionParameters: MapProperty = + project.objects.mapProperty(String::class.java, String::class.java) + + /** Working directory for the server process. Defaults to `build/polaris-server`. */ + val workingDirectory: DirectoryProperty = + project.objects + .directoryProperty() + .convention(project.layout.buildDirectory.dir("polaris-server")) + + /** Maximum time to wait for the server process to print its Quarkus listen ports. */ + val startupTimeout: Property = + project.objects.property(Duration::class.java).convention(Duration.ofSeconds(30)) + + /** Maximum time to wait for the server process to stop gracefully before it is killed. */ + val stopTimeout: Property = + project.objects.property(Duration::class.java).convention(Duration.ofSeconds(15)) + + /** Test JVM system property name that receives the discovered HTTP port. */ + val httpPortProperty: Property = + project.objects.property(String::class.java).convention("quarkus.http.test-port") + + /** Test JVM system property name that receives the discovered management port, when present. */ + val managementPortProperty: Property = + project.objects.property(String::class.java).convention("quarkus.management.test-port") + + /** + * System properties passed to the server JVM. + * + * These properties are also declared as inputs of the decorated test task. + */ + val systemProperties: MapProperty = + project.objects.mapProperty(String::class.java, String::class.java) + + /** + * Environment variables passed to the server process. + * + * These variables are also declared as inputs of the decorated test task. + */ + val environment: MapProperty = + project.objects.mapProperty(String::class.java, String::class.java) + + /** + * JVM arguments passed before `-jar` when starting the server. + * + * These arguments are also declared as inputs of the decorated test task. + */ + val jvmArguments: ListProperty = + project.objects.listProperty(String::class.java).convention(emptyList()) + + /** + * Application arguments passed after the server jar path. + * + * These arguments are also declared as inputs of the decorated test task. + */ + val arguments: ListProperty = + project.objects.listProperty(String::class.java).convention(emptyList()) +} + +internal fun PolarisServerTestRunnerExtension.conventionFrom( + other: PolarisServerTestRunnerExtension +) { + server.setFrom(other.server) + startupActionClasspath.setFrom(other.startupActionClasspath) + startupActionClass.convention(other.startupActionClass) + startupActionParameters.convention(other.startupActionParameters) + workingDirectory.convention(other.workingDirectory) + startupTimeout.convention(other.startupTimeout) + stopTimeout.convention(other.stopTimeout) + httpPortProperty.convention(other.httpPortProperty) + managementPortProperty.convention(other.managementPortProperty) + systemProperties.convention(other.systemProperties) + environment.convention(other.environment) + jvmArguments.convention(other.jvmArguments) + arguments.convention(other.arguments) +} diff --git a/gradle/server-test-runner/src/main/kotlin/org/apache/polaris/server/test/runner/PolarisServerTestRunnerPlugin.kt b/gradle/server-test-runner/src/main/kotlin/org/apache/polaris/server/test/runner/PolarisServerTestRunnerPlugin.kt new file mode 100644 index 00000000000..6dfeba8dbc5 --- /dev/null +++ b/gradle/server-test-runner/src/main/kotlin/org/apache/polaris/server/test/runner/PolarisServerTestRunnerPlugin.kt @@ -0,0 +1,62 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.polaris.server.test.runner + +import org.gradle.api.Plugin +import org.gradle.api.Project + +class PolarisServerTestRunnerPlugin : Plugin { + override fun apply(project: Project) { + val serverConfiguration = + project.configurations.create(POLARIS_SERVER_CONFIGURATION) { + isCanBeConsumed = false + isCanBeResolved = true + isTransitive = false + description = "Runnable Polaris server artifact used by integration test tasks." + } + + // We need one service per project to prevent classloader isolation issues. + // Build services are global to the current Gradle build invocation and shared + // across projects. As plugins are applied to projects, those use a "per project" + // class loader, which can then cause classic `ClassCastException` with a cause + // that `PolarisServerTestService$Inject_` from class loader A cannot be cast to + // `PolarisServerTestService` from class loader B. + // Using a project-scoped service name prevents this issue. + val serviceName = "polarisServerTestRunner-${project.path}" + val service = + project.gradle.sharedServices.registerIfAbsent( + serviceName, + PolarisServerTestService::class.java, + ) {} + + val extension = + project.extensions.create( + POLARIS_SERVER_EXTENSION, + PolarisServerTestRunnerExtension::class.java, + project, + service, + ) + extension.server.convention(serverConfiguration) + } + + companion object { + const val POLARIS_SERVER_CONFIGURATION = "polarisServer" + const val POLARIS_SERVER_EXTENSION = "polarisServerTestRunner" + } +} diff --git a/gradle/server-test-runner/src/main/kotlin/org/apache/polaris/server/test/runner/PolarisServerTestService.kt b/gradle/server-test-runner/src/main/kotlin/org/apache/polaris/server/test/runner/PolarisServerTestService.kt new file mode 100644 index 00000000000..cae1aae865e --- /dev/null +++ b/gradle/server-test-runner/src/main/kotlin/org/apache/polaris/server/test/runner/PolarisServerTestService.kt @@ -0,0 +1,54 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.polaris.server.test.runner + +import org.gradle.api.Task +import org.gradle.api.logging.Logging +import org.gradle.api.services.BuildService +import org.gradle.api.services.BuildServiceParameters + +abstract class PolarisServerTestService : BuildService, AutoCloseable { + private val servers = linkedMapOf() + private val logger = Logging.getLogger(PolarisServerTestService::class.java) + + internal fun register(task: Task, server: RunningPolarisServer) { + synchronized(servers) { + if (servers.containsKey(task.path)) { + throw IllegalStateException("Server already registered for task ${task.path}") + } + servers[task.path] = server + } + } + + fun finished(task: Task) { + val server = synchronized(servers) { servers.remove(task.path) } + server?.stop(task.logger) + } + + override fun close() { + val remaining = synchronized(servers) { servers.values.toList().also { servers.clear() } } + if (remaining.isNotEmpty()) { + logger.warn( + "Stopping {} Polaris server process(es) during build service cleanup", + remaining.size, + ) + } + remaining.forEach { it.stop(logger) } + } +} diff --git a/gradle/server-test-runner/src/main/kotlin/org/apache/polaris/server/test/runner/PolarisServerTestTaskConfigurer.kt b/gradle/server-test-runner/src/main/kotlin/org/apache/polaris/server/test/runner/PolarisServerTestTaskConfigurer.kt new file mode 100644 index 00000000000..2478341c015 --- /dev/null +++ b/gradle/server-test-runner/src/main/kotlin/org/apache/polaris/server/test/runner/PolarisServerTestTaskConfigurer.kt @@ -0,0 +1,194 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.polaris.server.test.runner + +import java.io.File +import java.net.URLClassLoader +import org.apache.polaris.server.test.runner.spi.PolarisServerStartupAction +import org.apache.polaris.server.test.runner.spi.PolarisServerStartupContext +import org.gradle.api.GradleException +import org.gradle.api.tasks.Internal +import org.gradle.api.tasks.PathSensitivity +import org.gradle.api.tasks.testing.Test +import org.gradle.api.tasks.testing.TestDescriptor +import org.gradle.api.tasks.testing.TestListener +import org.gradle.api.tasks.testing.TestResult +import org.gradle.process.CommandLineArgumentProvider + +internal fun Test.configurePolarisServer(extension: PolarisServerTestRunnerExtension) { + val dynamicArguments = PolarisServerArguments() + var startupAction: AutoCloseable? = null + jvmArgumentProviders.add(dynamicArguments) + inputs.files(extension.server).withPathSensitivity(PathSensitivity.RELATIVE) + inputs.files(extension.startupActionClasspath).withPathSensitivity(PathSensitivity.RELATIVE) + inputs.property("polarisServerStartupActionClass", extension.startupActionClass.orNull ?: "") + inputs.properties(extension.startupActionParameters.get()) + inputs.properties(extension.systemProperties.get()) + inputs.properties(extension.environment.get()) + inputs.property("polarisServerJvmArguments", extension.jvmArguments.get()) + inputs.property("polarisServerArguments", extension.arguments.get()) + usesService(extension.service) + addTestListener( + object : TestListener { + override fun beforeSuite(suite: TestDescriptor) {} + + override fun beforeTest(testDescriptor: TestDescriptor) {} + + override fun afterTest(testDescriptor: TestDescriptor, result: TestResult) {} + + override fun afterSuite(suite: TestDescriptor, result: TestResult) { + if (suite.parent == null) { + extension.service.get().finished(this@configurePolarisServer) + startupAction?.closeQuietly() + startupAction = null + } + } + } + ) + + doFirst { + val files = extension.server.files + if (files.size != 1) { + throw GradleException( + "Expected exactly one Polaris server artifact, but found ${files.size}: $files" + ) + } + val jar = files.single() + val javaExecutable = javaLauncher.get().executablePath.asFile.absolutePath + val systemProperties = extension.systemProperties.get().toMutableMap() + val environment = extension.environment.get().toMutableMap() + try { + startupAction = + extension.startupActionClass.orNull?.let { + IsolatedPolarisServerStartupAction.start( + implementationClass = it, + classpath = extension.startupActionClasspath.files, + parameters = extension.startupActionParameters.get(), + systemProperties = systemProperties, + environment = environment, + ) + } + val started = + RunningPolarisServer.start( + javaExecutable = javaExecutable, + jar = jar, + workingDirectory = extension.workingDirectory.get().asFile, + startupTimeout = extension.startupTimeout.get(), + stopTimeout = extension.stopTimeout.get(), + jvmArguments = extension.jvmArguments.get(), + systemProperties = systemProperties, + environment = environment, + arguments = extension.arguments.get(), + logger = logger, + ) + extension.service.get().register(this, started.server) + dynamicArguments.arguments = buildList { + add("-D${extension.httpPortProperty.get()}=${started.ports.httpPort}") + started.ports.managementPort?.let { add("-D${extension.managementPortProperty.get()}=$it") } + } + } catch (e: Throwable) { + startupAction?.closeQuietly() + startupAction = null + throw e + } + } +} + +private class PolarisServerArguments : CommandLineArgumentProvider { + @get:Internal var arguments: List = emptyList() + + override fun asArguments(): Iterable = arguments +} + +private class DefaultPolarisServerStartupContext( + private val parameters: Map, + private val systemProperties: MutableMap, + private val environment: MutableMap, +) : PolarisServerStartupContext { + override fun getParameters(): Map = parameters + + override fun getSystemProperties(): MutableMap = systemProperties + + override fun getEnvironment(): MutableMap = environment +} + +private class IsolatedPolarisServerStartupAction( + private val classLoader: URLClassLoader, + private val delegate: PolarisServerStartupAction, +) : AutoCloseable { + override fun close() { + withContextClassLoader(classLoader) { delegate.close() } + classLoader.close() + } + + companion object { + fun start( + implementationClass: String, + classpath: Set, + parameters: Map, + systemProperties: MutableMap, + environment: MutableMap, + ): IsolatedPolarisServerStartupAction { + val urls = classpath.map { it.toURI().toURL() }.toTypedArray() + val classLoader = URLClassLoader(urls, PolarisServerStartupAction::class.java.classLoader) + val action = + try { + withContextClassLoader(classLoader) { + val type = Class.forName(implementationClass, true, classLoader) + type.getDeclaredConstructor().newInstance() as PolarisServerStartupAction + } + } catch (e: Throwable) { + classLoader.close() + throw GradleException( + "Failed to load Polaris server startup action $implementationClass", + e, + ) + } + val isolated = IsolatedPolarisServerStartupAction(classLoader, action) + try { + withContextClassLoader(classLoader) { + action.start( + DefaultPolarisServerStartupContext(parameters, systemProperties, environment) + ) + } + } catch (e: Throwable) { + isolated.closeQuietly() + throw GradleException("Failed to run Polaris server startup action $implementationClass", e) + } + return isolated + } + } +} + +private fun AutoCloseable.closeQuietly() { + try { + close() + } catch (_: Exception) {} +} + +private fun withContextClassLoader(classLoader: ClassLoader, action: () -> T): T { + val thread = Thread.currentThread() + val previous = thread.contextClassLoader + thread.contextClassLoader = classLoader + try { + return action() + } finally { + thread.contextClassLoader = previous + } +} diff --git a/gradle/server-test-runner/src/main/kotlin/org/apache/polaris/server/test/runner/RunningPolarisServer.kt b/gradle/server-test-runner/src/main/kotlin/org/apache/polaris/server/test/runner/RunningPolarisServer.kt new file mode 100644 index 00000000000..2470a89be70 --- /dev/null +++ b/gradle/server-test-runner/src/main/kotlin/org/apache/polaris/server/test/runner/RunningPolarisServer.kt @@ -0,0 +1,167 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.polaris.server.test.runner + +import java.io.BufferedReader +import java.io.File +import java.io.InputStreamReader +import java.net.URI +import java.time.Duration +import java.util.concurrent.CountDownLatch +import java.util.concurrent.TimeUnit +import java.util.concurrent.atomic.AtomicReference +import org.gradle.api.GradleException +import org.gradle.api.logging.Logger + +internal class RunningPolarisServer( + private val process: Process, + private val outputThread: Thread, + private val stopTimeout: Duration, +) { + fun stop(logger: Logger) { + if (!process.isAlive) { + joinOutputThread(logger) + return + } + + process.destroy() + if (!process.waitFor(stopTimeout.toMillis(), TimeUnit.MILLISECONDS)) { + process.destroyForcibly() + process.waitFor(stopTimeout.toMillis(), TimeUnit.MILLISECONDS) + } + joinOutputThread(logger) + } + + private fun joinOutputThread(logger: Logger) { + try { + outputThread.join(stopTimeout.toMillis()) + } catch (e: InterruptedException) { + Thread.currentThread().interrupt() + logger.warn("Interrupted while waiting for Polaris server output thread to finish", e) + } + } + + companion object { + private val listenPattern = + Regex( + "^.*Listening on: (https?://\\S+?)(?:[.] Management interface listening on (https?://\\S+?)[.])?\\s*$" + ) + + fun start( + javaExecutable: String, + jar: File, + workingDirectory: File, + startupTimeout: Duration, + stopTimeout: Duration, + jvmArguments: List, + systemProperties: Map, + environment: Map, + arguments: List, + logger: Logger, + ): StartedServer { + workingDirectory.mkdirs() + val command = buildList { + add(javaExecutable) + addAll(jvmArguments) + add("-Dquarkus.http.port=0") + add("-Dquarkus.management.port=0") + systemProperties.forEach { (name, value) -> add("-D$name=$value") } + add("-jar") + add(jar.absolutePath) + addAll(arguments) + } + + logger.info("Starting Polaris server process: {}", command) + val processBuilder = + ProcessBuilder(command).directory(workingDirectory).redirectErrorStream(true) + processBuilder.environment().putAll(environment) + val process = processBuilder.start() + val detected = AtomicReference() + val failed = AtomicReference() + val ready = CountDownLatch(1) + val capturedOutput = mutableListOf() + + val outputThread = + Thread( + { + try { + BufferedReader(InputStreamReader(process.inputStream)).useLines { lines -> + lines.forEach { line -> + logger.info("[polaris-server] {}", line) + synchronized(capturedOutput) { capturedOutput.add(line) } + if (detected.get() == null) { + val match = listenPattern.matchEntire(line) + if (match != null) { + detected.set( + ListenUrls( + match.groupValues[1], + match.groupValues.getOrNull(2).orEmpty().ifEmpty { null }, + ) + ) + ready.countDown() + } + } + } + } + } catch (e: Throwable) { + failed.set(e) + ready.countDown() + } finally { + ready.countDown() + } + }, + "polaris-server-output", + ) + outputThread.isDaemon = true + outputThread.start() + + if (!ready.await(startupTimeout.toMillis(), TimeUnit.MILLISECONDS)) { + process.destroyForcibly() + throw GradleException( + "Polaris server did not emit a listen URL within $startupTimeout. Captured output:\n${capturedOutput(capturedOutput)}" + ) + } + + failed.get()?.let { throw GradleException("Failed while reading Polaris server output", it) } + val urls = detected.get() + if (urls == null) { + val exit = + if (process.isAlive) "still running" else "exited with code ${process.exitValue()}" + process.destroyForcibly() + throw GradleException( + "Polaris server $exit before emitting a listen URL. Captured output:\n${capturedOutput(capturedOutput)}" + ) + } + + return StartedServer(RunningPolarisServer(process, outputThread, stopTimeout), urls.toPorts()) + } + + private fun capturedOutput(lines: MutableList): String = + synchronized(lines) { lines.takeLast(200).joinToString("\n") }.ifBlank { "" } + } +} + +internal data class StartedServer(val server: RunningPolarisServer, val ports: ServerPorts) + +private data class ListenUrls(val httpUrl: String, val managementUrl: String?) { + fun toPorts(): ServerPorts = + ServerPorts(URI.create(httpUrl).port, managementUrl?.let { URI.create(it).port }) +} + +internal data class ServerPorts(val httpPort: Int, val managementPort: Int?) diff --git a/gradle/server-test-runner/src/test/kotlin/org/apache/polaris/server/test/runner/PolarisServerTestRunnerPluginTest.kt b/gradle/server-test-runner/src/test/kotlin/org/apache/polaris/server/test/runner/PolarisServerTestRunnerPluginTest.kt new file mode 100644 index 00000000000..c9781bbf8bd --- /dev/null +++ b/gradle/server-test-runner/src/test/kotlin/org/apache/polaris/server/test/runner/PolarisServerTestRunnerPluginTest.kt @@ -0,0 +1,243 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.polaris.server.test.runner + +import java.nio.file.Path +import kotlin.io.path.createDirectories +import kotlin.io.path.writeText +import org.assertj.core.api.Assertions.assertThat +import org.gradle.testkit.runner.GradleRunner +import org.gradle.testkit.runner.TaskOutcome +import org.junit.jupiter.api.Test +import org.junit.jupiter.api.io.TempDir + +class PolarisServerTestRunnerPluginTest { + @TempDir lateinit var projectDir: Path + + @Test + fun `starts server for test task and passes discovered ports`() { + writeSettings() + writeBuild( + """ + import org.gradle.api.tasks.compile.JavaCompile + import org.gradle.jvm.tasks.Jar + + plugins { + java + id("polaris-server-test-runner") + } + + repositories { mavenCentral() } + + dependencies { + testImplementation(platform("org.junit:junit-bom:6.1.0")) + testImplementation("org.junit.jupiter:junit-jupiter") + testRuntimeOnly("org.junit.platform:junit-platform-launcher") + } + + val serverRuntime by configurations.creating + + dependencies { + serverRuntime(files(tasks.named("jar"))) + } + + val startupActionSourceDir = layout.buildDirectory.dir("startup-action-src") + val startupActionClassesDir = layout.buildDirectory.dir("startup-action-classes") + + val writeStartupAction by tasks.registering { + val sourceFile = startupActionSourceDir.map { it.file("test/StartupAction.java") } + outputs.file(sourceFile) + doLast { + sourceFile.get().asFile.apply { + parentFile.mkdirs() + writeText( + ${"\"\"\""} + package test; + + import org.apache.polaris.server.test.runner.spi.PolarisServerStartupAction; + import org.apache.polaris.server.test.runner.spi.PolarisServerStartupContext; + + public class StartupAction implements PolarisServerStartupAction { + @Override + public void start(PolarisServerStartupContext context) { + context.getSystemProperties().put( + "startup.value", context.getParameters().get("value")); + } + } + ${"\"\"\""}.trimIndent() + ) + } + } + } + + val compileStartupAction by tasks.registering(JavaCompile::class) { + dependsOn(writeStartupAction) + source(startupActionSourceDir) + destinationDirectory.set(startupActionClassesDir) + classpath = + files( + org.apache.polaris.server.test.runner.spi.PolarisServerStartupAction::class.java + .protectionDomain + .codeSource + .location + ) + } + + val startupActionJar by tasks.registering(Jar::class) { + dependsOn(compileStartupAction) + from(startupActionClassesDir) + archiveBaseName.set("startup-action") + } + + tasks.named("jar") { + manifest { attributes("Main-Class" to "test.FakeServer") } + } + + tasks.test { + useJUnitPlatform() + withPolarisServer(configurations.named("serverRuntime")) { + arguments.add(layout.buildDirectory.file("server-stopped.txt").get().asFile.absolutePath) + arguments.add(layout.buildDirectory.file("server-started.txt").get().asFile.absolutePath) + startupActionClasspath.from(startupActionJar) + startupActionClass.set("test.StartupAction") + startupActionParameters.put("value", "started") + } + } + """ + .trimIndent() + ) + writeFakeServer() + writePropertyTest() + + val result = gradleRunner("test").build() + + assertThat(result.task(":test")?.outcome).isEqualTo(TaskOutcome.SUCCESS) + assertThat(projectDir.resolve("build/server-stopped.txt")).exists() + assertThat(projectDir.resolve("build/server-started.txt")).hasContent("started") + } + + @Test + fun `fails when server artifact is missing`() { + writeSettings() + writeBuild( + """ + plugins { + java + id("polaris-server-test-runner") + } + + repositories { mavenCentral() } + + dependencies { + testImplementation(platform("org.junit:junit-bom:6.1.0")) + testImplementation("org.junit.jupiter:junit-jupiter") + testRuntimeOnly("org.junit.platform:junit-platform-launcher") + } + + tasks.test { + useJUnitPlatform() + withPolarisServer(files()) + } + """ + .trimIndent() + ) + writePropertyTest() + + val result = gradleRunner("test").buildAndFail() + + assertThat(result.output).contains("Expected exactly one Polaris server artifact") + } + + private fun writeSettings() { + projectDir.resolve("settings.gradle.kts").writeText("rootProject.name = \"fixture\"\n") + } + + private fun writeBuild(content: String) { + projectDir.resolve("build.gradle.kts").writeText(content) + } + + private fun writeFakeServer() { + val sourceDir = projectDir.resolve("src/main/java/test").createDirectories() + sourceDir + .resolve("FakeServer.java") + .writeText( + """ + package test; + + import java.nio.file.Files; + import java.nio.file.Path; + import java.util.concurrent.CountDownLatch; + + public final class FakeServer { + public static void main(String[] args) throws Exception { + Path stopped = Path.of(args[0]); + if (args.length > 1) { + String startupValue = System.getProperty("startup.value"); + if (!"started".equals(startupValue)) { + throw new IllegalStateException("Unexpected startup value: " + startupValue); + } + Files.writeString(Path.of(args[1]), startupValue); + } + Runtime.getRuntime().addShutdownHook(new Thread(() -> { + try { + Files.writeString(stopped, "stopped"); + } catch (Exception e) { + throw new RuntimeException(e); + } + })); + System.out.println("Listening on: http://0.0.0.0:12345. Management interface listening on http://0.0.0.0:12346."); + new CountDownLatch(1).await(); + } + } + """ + .trimIndent() + ) + } + + private fun writePropertyTest() { + val testDir = projectDir.resolve("src/test/java/test").createDirectories() + testDir + .resolve("ServerPropertiesTest.java") + .writeText( + """ + package test; + + import static org.junit.jupiter.api.Assertions.assertEquals; + + import org.junit.jupiter.api.Test; + + public class ServerPropertiesTest { + @Test + void receivesServerProperties() { + assertEquals("12345", System.getProperty("quarkus.http.test-port")); + assertEquals("12346", System.getProperty("quarkus.management.test-port")); + } + } + """ + .trimIndent() + ) + } + + private fun gradleRunner(vararg arguments: String): GradleRunner = + GradleRunner.create() + .withProjectDir(projectDir.toFile()) + .withPluginClasspath() + .withArguments(*arguments, "--stacktrace") + .forwardOutput() +} diff --git a/settings.gradle.kts b/settings.gradle.kts index 8e4db0cb62e..6eefe52a29f 100644 --- a/settings.gradle.kts +++ b/settings.gradle.kts @@ -35,6 +35,8 @@ if ( includeBuild("build-logic") { name = "polaris-build-logic" } +includeBuild("gradle/server-test-runner") { name = "polaris-server-test-runner" } + if (!JavaVersion.current().isCompatibleWith(JavaVersion.VERSION_21)) { throw GradleException( """ From 9d4b6f06abb3dd82ff3d4a3de162f5159fb7b6e5 Mon Sep 17 00:00:00 2001 From: Robert Stupp Date: Tue, 9 Jun 2026 09:38:14 +0200 Subject: [PATCH 03/22] Add external Polaris server test manager Adds a `PolarisServerManager` implementation for the polaris-server-test-runner plugin. --- .../it/ext/ExternalPolarisServerManager.java | 81 ++++++++++++++ .../ext/ExternalPolarisServerManagerTest.java | 102 ++++++++++++++++++ 2 files changed, 183 insertions(+) create mode 100644 integration-tests/src/main/java/org/apache/polaris/service/it/ext/ExternalPolarisServerManager.java create mode 100644 integration-tests/src/test/java/org/apache/polaris/service/it/ext/ExternalPolarisServerManagerTest.java diff --git a/integration-tests/src/main/java/org/apache/polaris/service/it/ext/ExternalPolarisServerManager.java b/integration-tests/src/main/java/org/apache/polaris/service/it/ext/ExternalPolarisServerManager.java new file mode 100644 index 00000000000..276ef797ce5 --- /dev/null +++ b/integration-tests/src/main/java/org/apache/polaris/service/it/ext/ExternalPolarisServerManager.java @@ -0,0 +1,81 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.polaris.service.it.ext; + +import java.net.URI; +import org.apache.polaris.service.it.env.ClientCredentials; +import org.apache.polaris.service.it.env.ClientPrincipal; +import org.apache.polaris.service.it.env.Server; +import org.junit.jupiter.api.extension.ExtensionContext; + +/** + * {@link PolarisServerManager} implementation for tests whose Polaris server lifecycle is managed + * outside JUnit, for example by Gradle. + */ +public class ExternalPolarisServerManager implements PolarisServerManager { + + public static final String HTTP_PORT_PROPERTY = "quarkus.http.test-port"; + public static final String HOST_PROPERTY = "polaris.test.server.host"; + + @Override + public Server serverForContext(ExtensionContext context) { + URI baseUri = URI.create(String.format("http://%s:%d", host(), httpPort())); + return new Server() { + @Override + public URI baseUri() { + return baseUri; + } + + @Override + public ClientPrincipal adminCredentials() { + return new ClientPrincipal("root", new ClientCredentials("test-admin", "test-secret")); + } + + @Override + public void close() { + // The external process owner is responsible for server lifecycle. + } + }; + } + + private static String host() { + String host = System.getProperty(HOST_PROPERTY, "localhost"); + return host.isBlank() ? "localhost" : host; + } + + private static int httpPort() { + String port = System.getProperty(HTTP_PORT_PROPERTY); + if (port == null || port.isBlank()) { + throw new IllegalStateException( + String.format("System property '%s' must be set", HTTP_PORT_PROPERTY)); + } + try { + int parsed = Integer.parseInt(port); + if (parsed <= 0 || parsed > 65535) { + throw new IllegalArgumentException("Port out of range: " + parsed); + } + return parsed; + } catch (IllegalArgumentException e) { + throw new IllegalStateException( + String.format( + "System property '%s' must be a valid TCP port: %s", HTTP_PORT_PROPERTY, port), + e); + } + } +} diff --git a/integration-tests/src/test/java/org/apache/polaris/service/it/ext/ExternalPolarisServerManagerTest.java b/integration-tests/src/test/java/org/apache/polaris/service/it/ext/ExternalPolarisServerManagerTest.java new file mode 100644 index 00000000000..a6a05bf5b23 --- /dev/null +++ b/integration-tests/src/test/java/org/apache/polaris/service/it/ext/ExternalPolarisServerManagerTest.java @@ -0,0 +1,102 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.polaris.service.it.ext; + +import static org.assertj.core.api.Assertions.assertThat; +import static org.assertj.core.api.Assertions.assertThatThrownBy; + +import org.apache.polaris.service.it.env.Server; +import org.junit.jupiter.api.AfterEach; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; + +class ExternalPolarisServerManagerTest { + + private String originalHost; + private String originalPort; + + @BeforeEach + void captureProperties() { + originalHost = System.getProperty(ExternalPolarisServerManager.HOST_PROPERTY); + originalPort = System.getProperty(ExternalPolarisServerManager.HTTP_PORT_PROPERTY); + } + + @AfterEach + void restoreProperties() { + restore(ExternalPolarisServerManager.HOST_PROPERTY, originalHost); + restore(ExternalPolarisServerManager.HTTP_PORT_PROPERTY, originalPort); + } + + @Test + void serverForContextUsesConfiguredHttpPort() { + System.setProperty(ExternalPolarisServerManager.HTTP_PORT_PROPERTY, "12345"); + + Server server = new ExternalPolarisServerManager().serverForContext(null); + + assertThat(server.baseUri()).hasToString("http://localhost:12345"); + assertThat(server.adminCredentials().principalName()).isEqualTo("root"); + assertThat(server.adminCredentials().credentials().clientId()).isEqualTo("test-admin"); + assertThat(server.adminCredentials().credentials().clientSecret()).isEqualTo("test-secret"); + } + + @Test + void serverForContextUsesConfiguredHost() { + System.setProperty(ExternalPolarisServerManager.HTTP_PORT_PROPERTY, "12345"); + System.setProperty(ExternalPolarisServerManager.HOST_PROPERTY, "127.0.0.1"); + + Server server = new ExternalPolarisServerManager().serverForContext(null); + + assertThat(server.baseUri()).hasToString("http://127.0.0.1:12345"); + } + + @Test + void serverForContextRejectsMissingHttpPort() { + System.clearProperty(ExternalPolarisServerManager.HTTP_PORT_PROPERTY); + + assertThatThrownBy(() -> new ExternalPolarisServerManager().serverForContext(null)) + .isInstanceOf(IllegalStateException.class) + .hasMessageContaining(ExternalPolarisServerManager.HTTP_PORT_PROPERTY); + } + + @Test + void serverForContextRejectsInvalidHttpPort() { + System.setProperty(ExternalPolarisServerManager.HTTP_PORT_PROPERTY, "not-a-port"); + + assertThatThrownBy(() -> new ExternalPolarisServerManager().serverForContext(null)) + .isInstanceOf(IllegalStateException.class) + .hasMessageContaining(ExternalPolarisServerManager.HTTP_PORT_PROPERTY); + } + + @Test + void serverForContextRejectsOutOfRangeHttpPort() { + System.setProperty(ExternalPolarisServerManager.HTTP_PORT_PROPERTY, "65536"); + + assertThatThrownBy(() -> new ExternalPolarisServerManager().serverForContext(null)) + .isInstanceOf(IllegalStateException.class) + .hasMessageContaining(ExternalPolarisServerManager.HTTP_PORT_PROPERTY); + } + + private static void restore(String propertyName, String value) { + if (value == null) { + System.clearProperty(propertyName); + } else { + System.setProperty(propertyName, value); + } + } +} From 35ea4e060168d676fb8896c5da58f35eace1abd9 Mon Sep 17 00:00:00 2001 From: Robert Stupp Date: Tue, 9 Jun 2026 09:55:40 +0200 Subject: [PATCH 04/22] Migrate Spark integration tests to server runner --- .../quarkus/it/SparkCatalogIcebergIT.java | 2 - .../quarkus/it/SparkCatalogPolarisIT.java | 2 - .../spark/quarkus/it/SparkDeltaIT.java | 2 - .../polaris/spark/quarkus/it/SparkHudiIT.java | 2 - .../polaris/spark/quarkus/it/SparkIT.java | 2 - ...olaris.service.it.ext.PolarisServerManager | 2 +- .../spark/v3.5/integration/build.gradle.kts | 273 +++++++++-------- .../spark/quarkus/it/SparkCatalogBaseIT.java | 2 - .../spark/v4.0/integration/build.gradle.kts | 279 +++++++++--------- .../spark/quarkus/it/SparkCatalogBaseIT.java | 2 - 10 files changed, 285 insertions(+), 283 deletions(-) diff --git a/plugins/spark/common/src/intTest/java/org/apache/polaris/spark/quarkus/it/SparkCatalogIcebergIT.java b/plugins/spark/common/src/intTest/java/org/apache/polaris/spark/quarkus/it/SparkCatalogIcebergIT.java index 812d8f19d53..36161a7b95a 100644 --- a/plugins/spark/common/src/intTest/java/org/apache/polaris/spark/quarkus/it/SparkCatalogIcebergIT.java +++ b/plugins/spark/common/src/intTest/java/org/apache/polaris/spark/quarkus/it/SparkCatalogIcebergIT.java @@ -18,11 +18,9 @@ */ package org.apache.polaris.spark.quarkus.it; -import io.quarkus.test.junit.QuarkusIntegrationTest; import org.apache.polaris.service.it.ext.SparkSessionBuilder; import org.apache.spark.sql.SparkSession; -@QuarkusIntegrationTest public class SparkCatalogIcebergIT extends SparkCatalogBaseIT { /** Initialize the spark catalog to use the iceberg spark catalog. */ @Override diff --git a/plugins/spark/common/src/intTest/java/org/apache/polaris/spark/quarkus/it/SparkCatalogPolarisIT.java b/plugins/spark/common/src/intTest/java/org/apache/polaris/spark/quarkus/it/SparkCatalogPolarisIT.java index 97a4c222db1..5e823524f1c 100644 --- a/plugins/spark/common/src/intTest/java/org/apache/polaris/spark/quarkus/it/SparkCatalogPolarisIT.java +++ b/plugins/spark/common/src/intTest/java/org/apache/polaris/spark/quarkus/it/SparkCatalogPolarisIT.java @@ -18,7 +18,5 @@ */ package org.apache.polaris.spark.quarkus.it; -import io.quarkus.test.junit.QuarkusIntegrationTest; -@QuarkusIntegrationTest public class SparkCatalogPolarisIT extends SparkCatalogBaseIT {} diff --git a/plugins/spark/common/src/intTest/java/org/apache/polaris/spark/quarkus/it/SparkDeltaIT.java b/plugins/spark/common/src/intTest/java/org/apache/polaris/spark/quarkus/it/SparkDeltaIT.java index 7beacb11415..cef29c36fb6 100644 --- a/plugins/spark/common/src/intTest/java/org/apache/polaris/spark/quarkus/it/SparkDeltaIT.java +++ b/plugins/spark/common/src/intTest/java/org/apache/polaris/spark/quarkus/it/SparkDeltaIT.java @@ -21,7 +21,6 @@ import static org.assertj.core.api.Assertions.assertThat; import static org.assertj.core.api.Assertions.assertThatThrownBy; -import io.quarkus.test.junit.QuarkusIntegrationTest; import java.io.File; import java.nio.file.Path; import java.util.Arrays; @@ -41,7 +40,6 @@ import org.junit.jupiter.api.Test; import org.junit.jupiter.api.io.TempDir; -@QuarkusIntegrationTest public class SparkDeltaIT extends SparkIntegrationBase { private String defaultNs; private String tableRootDir; diff --git a/plugins/spark/common/src/intTest/java/org/apache/polaris/spark/quarkus/it/SparkHudiIT.java b/plugins/spark/common/src/intTest/java/org/apache/polaris/spark/quarkus/it/SparkHudiIT.java index aec85f5d46f..30d5fa71a4a 100644 --- a/plugins/spark/common/src/intTest/java/org/apache/polaris/spark/quarkus/it/SparkHudiIT.java +++ b/plugins/spark/common/src/intTest/java/org/apache/polaris/spark/quarkus/it/SparkHudiIT.java @@ -21,7 +21,6 @@ import static org.assertj.core.api.Assertions.assertThat; import static org.assertj.core.api.Assertions.assertThatThrownBy; -import io.quarkus.test.junit.QuarkusIntegrationTest; import java.io.File; import java.nio.file.Path; import java.util.List; @@ -33,7 +32,6 @@ import org.junit.jupiter.api.Test; import org.junit.jupiter.api.io.TempDir; -@QuarkusIntegrationTest public class SparkHudiIT extends SparkIntegrationBase { @Override diff --git a/plugins/spark/common/src/intTest/java/org/apache/polaris/spark/quarkus/it/SparkIT.java b/plugins/spark/common/src/intTest/java/org/apache/polaris/spark/quarkus/it/SparkIT.java index a4e060a52fc..4850512728a 100644 --- a/plugins/spark/common/src/intTest/java/org/apache/polaris/spark/quarkus/it/SparkIT.java +++ b/plugins/spark/common/src/intTest/java/org/apache/polaris/spark/quarkus/it/SparkIT.java @@ -21,7 +21,6 @@ import static org.assertj.core.api.Assertions.assertThat; import static org.assertj.core.api.Assertions.assertThatThrownBy; -import io.quarkus.test.junit.QuarkusIntegrationTest; import java.io.File; import java.nio.file.Path; import java.util.List; @@ -30,7 +29,6 @@ import org.junit.jupiter.api.Test; import org.junit.jupiter.api.io.TempDir; -@QuarkusIntegrationTest public class SparkIT extends SparkIntegrationBase { @Test public void testNamespaces() { diff --git a/plugins/spark/common/src/intTest/resources/META-INF/services/org.apache.polaris.service.it.ext.PolarisServerManager b/plugins/spark/common/src/intTest/resources/META-INF/services/org.apache.polaris.service.it.ext.PolarisServerManager index b3dd7d7c069..6aa2dac9e0c 100644 --- a/plugins/spark/common/src/intTest/resources/META-INF/services/org.apache.polaris.service.it.ext.PolarisServerManager +++ b/plugins/spark/common/src/intTest/resources/META-INF/services/org.apache.polaris.service.it.ext.PolarisServerManager @@ -17,4 +17,4 @@ # under the License. # -org.apache.polaris.service.it.ServerManager +org.apache.polaris.service.it.ext.ExternalPolarisServerManager diff --git a/plugins/spark/v3.5/integration/build.gradle.kts b/plugins/spark/v3.5/integration/build.gradle.kts index 476c675fc31..645db456507 100644 --- a/plugins/spark/v3.5/integration/build.gradle.kts +++ b/plugins/spark/v3.5/integration/build.gradle.kts @@ -17,10 +17,11 @@ * under the License. */ +import org.gradle.api.plugins.jvm.JvmTestSuite + plugins { - alias(libs.plugins.quarkus) - id("org.kordamp.gradle.jandex") - id("polaris-runtime") + id("polaris-java") + id("polaris-server-test-runner") } // get version information @@ -34,6 +35,144 @@ val scalaLibraryVersion = } else { libs.versions.scala213.get() } +val errorProneAnnotationsVersion = libs.errorprone.get().versionConstraint.requiredVersion + +dependencies { polarisServer(project(path = ":polaris-server", configuration = "quarkusRunner")) } + +testing { + suites { + register("intTest") { + dependencies { + implementation( + "org.apache.iceberg:iceberg-spark-runtime-${sparkMajorVersion}_${scalaVersion}:${icebergVersion}" + ) + implementation(project(":polaris-spark-${sparkMajorVersion}_${scalaVersion}")) + + implementation(project(":polaris-api-management-model")) + + implementation(project(":polaris-runtime-test-common")) + + implementation("org.apache.spark:spark-sql_${scalaVersion}:${spark35Version}") { + // exclude log4j dependencies. Explicit dependencies for the log4j libraries are + // enforced below to ensure the version compatibility + exclude("org.apache.logging.log4j", "log4j-slf4j2-impl") + exclude("org.apache.logging.log4j", "log4j-1.2-api") + exclude("org.apache.logging.log4j", "log4j-core") + exclude("org.slf4j", "jul-to-slf4j") + } + + // Add spark-hive for Hudi integration - provides HiveExternalCatalog that Hudi needs + runtimeOnly("org.apache.spark:spark-hive_${scalaVersion}:${spark35Version}") { + // exclude log4j dependencies to match spark-sql exclusions + exclude("org.apache.logging.log4j", "log4j-slf4j2-impl") + exclude("org.apache.logging.log4j", "log4j-1.2-api") + exclude("org.apache.logging.log4j", "log4j-core") + exclude("org.slf4j", "jul-to-slf4j") + // exclude old slf4j 1.x to log4j 2.x bridge that conflicts with slf4j 2.x bridge + exclude("org.apache.logging.log4j", "log4j-slf4j-impl") + } + // enforce the usage of log4j 2.24.3. This is for the log4j-api compatibility + // of spark-sql dependency + runtimeOnly("org.apache.logging.log4j:log4j-core:2.26.0") + + implementation("io.delta:delta-spark_${scalaVersion}:3.3.1") + implementation("org.apache.hudi:hudi-spark3.5-bundle_${scalaVersion}:1.1.1") { + // exclude log4j dependencies to match spark-sql exclusions + // exclude log4j dependencies to match spark-sql exclusions and prevent version conflicts + exclude("org.apache.logging.log4j", "log4j-slf4j2-impl") + exclude("org.apache.logging.log4j", "log4j-1.2-api") + exclude("org.apache.logging.log4j", "log4j-core") + exclude("org.slf4j", "jul-to-slf4j") + exclude("org.slf4j", "slf4j-log4j12") + exclude("org.slf4j", "slf4j-reload4j") + exclude("ch.qos.reload4j", "reload4j") + exclude("log4j", "log4j") + // exclude old slf4j 1.x to log4j 2.x bridge that conflicts with slf4j 2.x bridge + exclude("org.apache.logging.log4j", "log4j-slf4j-impl") + } + + // The hudi-spark-bundle includes most Hive libraries but excludes hive-exec to keep size + // manageable + // This matches what Spark 3.5 distribution provides (hive-exec-2.3.10-core.jar) + implementation("org.apache.hive:hive-exec:2.3.10:core") { + // Exclude conflicting dependencies to use Spark's versions + exclude("org.apache.hadoop", "*") + exclude("org.apache.commons", "*") + exclude("org.slf4j", "*") + exclude("log4j", "*") + exclude("org.apache.logging.log4j", "*") + exclude("org.pentaho", "*") + exclude("org.apache.calcite", "*") + exclude("org.apache.tez", "*") + } + + implementation(platform(libs.jackson.bom)) + implementation("com.fasterxml.jackson.jakarta.rs:jackson-jakarta-rs-json-provider") + implementation(libs.jakarta.ws.rs.api) + compileOnly("com.google.errorprone:error_prone_annotations:${errorProneAnnotationsVersion}") + + implementation(testFixtures(project(":polaris-runtime-service"))) + + implementation(platform(libs.awssdk.bom)) + implementation("software.amazon.awssdk:glue") + implementation("software.amazon.awssdk:kms") + implementation("software.amazon.awssdk:dynamodb") + + implementation(platform(libs.testcontainers.bom)) + implementation("org.testcontainers:testcontainers") + implementation(libs.s3mock.testcontainers) + + // Required for Spark integration tests + implementation(enforcedPlatform("org.scala-lang:scala-library:${scalaLibraryVersion}")) + implementation(enforcedPlatform("org.scala-lang:scala-reflect:${scalaLibraryVersion}")) + implementation(libs.javax.servlet.api) + implementation(libs.antlr4.runtime.spark35) + } + + targets.all { + testTask.configure { + environment( + "AWS_REGION", + providers.environmentVariable("AWS_REGION").getOrElse("us-west-2"), + ) + jvmArgs("--add-exports", "java.base/sun.nio.ch=ALL-UNNAMED") + // Need to allow a java security manager after Java 21, for Subject.getSubject to work + // "getSubject is supported only if a security manager is allowed". + systemProperty("java.security.manager", "allow") + val logsDir = project.layout.buildDirectory.get().asFile.resolve("logs") + // delete files from previous runs + doFirst { + // delete log files written by Polaris + logsDir.deleteRecursively() + } + withPolarisServer(configurations.polarisServer) { + environment.put( + "AWS_REGION", + providers.environmentVariable("AWS_REGION").orElse("us-west-2"), + ) + environment.put( + "AWS_ACCESS_KEY_ID", + providers.environmentVariable("AWS_ACCESS_KEY_ID").orElse("ap1"), + ) + environment.put( + "AWS_SECRET_ACCESS_KEY", + providers.environmentVariable("AWS_SECRET_ACCESS_KEY").orElse("s3cr3t"), + ) + environment.put("AWS_EC2_METADATA_DISABLED", "true") + environment.put("POLARIS_BOOTSTRAP_CREDENTIALS", "POLARIS,test-admin,test-secret") + systemProperties.put("quarkus.profile", "it") + systemProperties.put( + "quarkus.log.file.path", + logsDir.resolve("polaris.log").absolutePath, + ) + } + // For Spark integration tests + addSparkJvmOptions() + } + } + } + } +} sourceSets { named("intTest") { @@ -41,131 +180,3 @@ sourceSets { resources { srcDir("../../common/src/intTest/resources") } } } - -dependencies { - // must be enforced to get a consistent and validated set of dependencies - implementation(enforcedPlatform(libs.quarkus.bom)) { - exclude(group = "org.antlr", module = "antlr4-runtime") - exclude(group = "org.scala-lang", module = "scala-library") - exclude(group = "org.scala-lang", module = "scala-reflect") - } - - implementation(project(":polaris-runtime-service")) - - testImplementation( - "org.apache.iceberg:iceberg-spark-runtime-${sparkMajorVersion}_${scalaVersion}:${icebergVersion}" - ) - testImplementation(project(":polaris-spark-${sparkMajorVersion}_${scalaVersion}")) - - testImplementation(project(":polaris-api-management-model")) - - testImplementation(project(":polaris-runtime-test-common")) - - testImplementation("org.apache.spark:spark-sql_${scalaVersion}:${spark35Version}") { - // exclude log4j dependencies. Explicit dependencies for the log4j libraries are - // enforced below to ensure the version compatibility - exclude("org.apache.logging.log4j", "log4j-slf4j2-impl") - exclude("org.apache.logging.log4j", "log4j-1.2-api") - exclude("org.apache.logging.log4j", "log4j-core") - exclude("org.slf4j", "jul-to-slf4j") - } - - // Add spark-hive for Hudi integration - provides HiveExternalCatalog that Hudi needs - testRuntimeOnly("org.apache.spark:spark-hive_${scalaVersion}:${spark35Version}") { - // exclude log4j dependencies to match spark-sql exclusions - exclude("org.apache.logging.log4j", "log4j-slf4j2-impl") - exclude("org.apache.logging.log4j", "log4j-1.2-api") - exclude("org.apache.logging.log4j", "log4j-core") - exclude("org.slf4j", "jul-to-slf4j") - // exclude old slf4j 1.x to log4j 2.x bridge that conflicts with slf4j 2.x bridge - exclude("org.apache.logging.log4j", "log4j-slf4j-impl") - } - // enforce the usage of log4j 2.24.3. This is for the log4j-api compatibility - // of spark-sql dependency - testRuntimeOnly("org.apache.logging.log4j:log4j-core:2.26.0") - - testImplementation("io.delta:delta-spark_${scalaVersion}:3.3.1") - testImplementation("org.apache.hudi:hudi-spark3.5-bundle_${scalaVersion}:1.1.1") { - // exclude log4j dependencies to match spark-sql exclusions - // exclude log4j dependencies to match spark-sql exclusions and prevent version conflicts - exclude("org.apache.logging.log4j", "log4j-slf4j2-impl") - exclude("org.apache.logging.log4j", "log4j-1.2-api") - exclude("org.apache.logging.log4j", "log4j-core") - exclude("org.slf4j", "jul-to-slf4j") - exclude("org.slf4j", "slf4j-log4j12") - exclude("org.slf4j", "slf4j-reload4j") - exclude("ch.qos.reload4j", "reload4j") - exclude("log4j", "log4j") - // exclude old slf4j 1.x to log4j 2.x bridge that conflicts with slf4j 2.x bridge - exclude("org.apache.logging.log4j", "log4j-slf4j-impl") - } - - // The hudi-spark-bundle includes most Hive libraries but excludes hive-exec to keep size - // manageable - // This matches what Spark 3.5 distribution provides (hive-exec-2.3.10-core.jar) - testImplementation("org.apache.hive:hive-exec:2.3.10:core") { - // Exclude conflicting dependencies to use Spark's versions - exclude("org.apache.hadoop", "*") - exclude("org.apache.commons", "*") - exclude("org.slf4j", "*") - exclude("log4j", "*") - exclude("org.apache.logging.log4j", "*") - exclude("org.pentaho", "*") - exclude("org.apache.calcite", "*") - exclude("org.apache.tez", "*") - } - - testImplementation(platform(libs.jackson.bom)) - testImplementation("com.fasterxml.jackson.jakarta.rs:jackson-jakarta-rs-json-provider") - - testImplementation(testFixtures(project(":polaris-runtime-service"))) - - testImplementation(platform(libs.quarkus.bom)) - testImplementation("io.quarkus:quarkus-junit") - testImplementation("io.quarkus:quarkus-rest-client") - testImplementation("io.quarkus:quarkus-rest-client-jackson") - - testImplementation(platform(libs.awssdk.bom)) - testImplementation("software.amazon.awssdk:glue") - testImplementation("software.amazon.awssdk:kms") - testImplementation("software.amazon.awssdk:dynamodb") - - testImplementation(platform(libs.testcontainers.bom)) - testImplementation("org.testcontainers:testcontainers") - testImplementation(libs.s3mock.testcontainers) - - // Required for Spark integration tests - testImplementation(enforcedPlatform("org.scala-lang:scala-library:${scalaLibraryVersion}")) - testImplementation(enforcedPlatform("org.scala-lang:scala-reflect:${scalaLibraryVersion}")) - testImplementation(libs.javax.servlet.api) - testImplementation(libs.antlr4.runtime.spark35) -} - -tasks.named("intTest").configure { - if (System.getenv("AWS_REGION") == null) { - environment("AWS_REGION", "us-west-2") - } - // Note: the test secrets are referenced in - // org.apache.polaris.service.it.ServerManager - environment("POLARIS_BOOTSTRAP_CREDENTIALS", "POLARIS,test-admin,test-secret") - jvmArgs("--add-exports", "java.base/sun.nio.ch=ALL-UNNAMED") - // Need to allow a java security manager after Java 21, for Subject.getSubject to work - // "getSubject is supported only if a security manager is allowed". - systemProperty("java.security.manager", "allow") - // Same issue as above: allow a java security manager after Java 21 - // (this setting is for the application under test, while the setting above is for test code). - systemProperty("quarkus.test.arg-line", "-Djava.security.manager=allow") - val logsDir = project.layout.buildDirectory.get().asFile.resolve("logs") - // delete files from previous runs - doFirst { - // delete log files written by Polaris - logsDir.deleteRecursively() - // delete quarkus.log file (captured Polaris stdout/stderr) - project.layout.buildDirectory.get().asFile.resolve("quarkus.log").delete() - } - // This property is not honored in a per-profile application.properties file, - // so we need to set it here. - systemProperty("quarkus.log.file.path", logsDir.resolve("polaris.log").absolutePath) - // For Spark integration tests - addSparkJvmOptions() -} diff --git a/plugins/spark/v3.5/integration/src/intTest/java/org/apache/polaris/spark/quarkus/it/SparkCatalogBaseIT.java b/plugins/spark/v3.5/integration/src/intTest/java/org/apache/polaris/spark/quarkus/it/SparkCatalogBaseIT.java index 23f2f1f5fd1..b64f1e6766e 100644 --- a/plugins/spark/v3.5/integration/src/intTest/java/org/apache/polaris/spark/quarkus/it/SparkCatalogBaseIT.java +++ b/plugins/spark/v3.5/integration/src/intTest/java/org/apache/polaris/spark/quarkus/it/SparkCatalogBaseIT.java @@ -23,7 +23,6 @@ import com.google.common.base.Preconditions; import com.google.common.collect.Maps; -import io.quarkus.test.junit.QuarkusIntegrationTest; import java.util.Arrays; import java.util.Map; import org.apache.iceberg.exceptions.BadRequestException; @@ -49,7 +48,6 @@ * SparkCatalog operations, some operations like listNamespaces under a namespace can not be * triggered through a SQL interface directly with Spark. */ -@QuarkusIntegrationTest public abstract class SparkCatalogBaseIT extends SparkIntegrationBase { private static StructType schema = new StructType().add("id", "long").add("name", "string"); protected StagingTableCatalog tableCatalog = null; diff --git a/plugins/spark/v4.0/integration/build.gradle.kts b/plugins/spark/v4.0/integration/build.gradle.kts index ccc140fad3f..fe61e0f42d1 100644 --- a/plugins/spark/v4.0/integration/build.gradle.kts +++ b/plugins/spark/v4.0/integration/build.gradle.kts @@ -17,10 +17,11 @@ * under the License. */ +import org.gradle.api.plugins.jvm.JvmTestSuite + plugins { - alias(libs.plugins.quarkus) - id("org.kordamp.gradle.jandex") - id("polaris-runtime") + id("polaris-java") + id("polaris-server-test-runner") } // get version information @@ -29,6 +30,144 @@ val scalaVersion = getAndUseScalaVersionForProject() val icebergVersion = libs.versions.iceberg.get() val spark40Version = libs.versions.spark40.get() val scalaLibraryVersion = libs.versions.scala213.get() +val errorProneAnnotationsVersion = libs.errorprone.get().versionConstraint.requiredVersion + +dependencies { polarisServer(project(path = ":polaris-server", configuration = "quarkusRunner")) } + +testing { + suites { + register("intTest") { + dependencies { + implementation( + "org.apache.iceberg:iceberg-spark-runtime-${sparkMajorVersion}_${scalaVersion}:${icebergVersion}" + ) + implementation(project(":polaris-spark-${sparkMajorVersion}_${scalaVersion}")) + + implementation(project(":polaris-api-management-model")) + + implementation(project(":polaris-runtime-test-common")) + + implementation("org.apache.spark:spark-sql_${scalaVersion}:${spark40Version}") { + // exclude log4j dependencies. Explicit dependencies for the log4j libraries are + // enforced below to ensure the version compatibility + exclude("org.apache.logging.log4j", "log4j-slf4j2-impl") + exclude("org.apache.logging.log4j", "log4j-1.2-api") + exclude("org.apache.logging.log4j", "log4j-core") + exclude("org.slf4j", "jul-to-slf4j") + } + + // Add spark-hive for Hudi integration - provides HiveExternalCatalog that Hudi needs + runtimeOnly("org.apache.spark:spark-hive_${scalaVersion}:${spark40Version}") { + // exclude log4j dependencies to match spark-sql exclusions + exclude("org.apache.logging.log4j", "log4j-slf4j2-impl") + exclude("org.apache.logging.log4j", "log4j-1.2-api") + exclude("org.apache.logging.log4j", "log4j-core") + exclude("org.slf4j", "jul-to-slf4j") + // exclude old slf4j 1.x to log4j 2.x bridge that conflicts with slf4j 2.x bridge + exclude("org.apache.logging.log4j", "log4j-slf4j-impl") + } + // enforce the usage of log4j 2.24.3. This is for the log4j-api compatibility + // of spark-sql dependency + runtimeOnly("org.apache.logging.log4j:log4j-core:2.26.0") + + implementation("io.delta:delta-spark_${scalaVersion}:4.0.1") + implementation("org.apache.hudi:hudi-spark4.0-bundle_${scalaVersion}:1.1.1") { + // exclude log4j dependencies to match spark-sql exclusions + // exclude log4j dependencies to match spark-sql exclusions and prevent version conflicts + exclude("org.apache.logging.log4j", "log4j-slf4j2-impl") + exclude("org.apache.logging.log4j", "log4j-1.2-api") + exclude("org.apache.logging.log4j", "log4j-core") + exclude("org.slf4j", "jul-to-slf4j") + exclude("org.slf4j", "slf4j-log4j12") + exclude("org.slf4j", "slf4j-reload4j") + exclude("ch.qos.reload4j", "reload4j") + exclude("log4j", "log4j") + // exclude old slf4j 1.x to log4j 2.x bridge that conflicts with slf4j 2.x bridge + exclude("org.apache.logging.log4j", "log4j-slf4j-impl") + } + + // The hudi-spark-bundle includes most Hive libraries but excludes hive-exec to keep size + // manageable + // This matches what Spark 4.0 distribution provides (hive-exec-2.3.10-core.jar) + implementation("org.apache.hive:hive-exec:2.3.10:core") { + // Exclude conflicting dependencies to use Spark's versions + exclude("org.apache.hadoop", "*") + exclude("org.apache.commons", "*") + exclude("org.slf4j", "*") + exclude("log4j", "*") + exclude("org.apache.logging.log4j", "*") + exclude("org.pentaho", "*") + exclude("org.apache.calcite", "*") + exclude("org.apache.tez", "*") + } + + implementation(platform(libs.jackson.bom)) + implementation("com.fasterxml.jackson.jakarta.rs:jackson-jakarta-rs-json-provider") + implementation(libs.jakarta.ws.rs.api) + compileOnly("com.google.errorprone:error_prone_annotations:${errorProneAnnotationsVersion}") + + implementation(testFixtures(project(":polaris-runtime-service"))) + + implementation(platform(libs.awssdk.bom)) + implementation("software.amazon.awssdk:glue") + implementation("software.amazon.awssdk:kms") + implementation("software.amazon.awssdk:dynamodb") + + implementation(platform(libs.testcontainers.bom)) + implementation("org.testcontainers:testcontainers") + implementation(libs.s3mock.testcontainers) + + // Required for Spark integration tests + implementation(enforcedPlatform("org.scala-lang:scala-library:${scalaLibraryVersion}")) + implementation(enforcedPlatform("org.scala-lang:scala-reflect:${scalaLibraryVersion}")) + implementation(libs.javax.servlet.api) + implementation(libs.antlr4.runtime.spark40) + } + + targets.all { + testTask.configure { + environment( + "AWS_REGION", + providers.environmentVariable("AWS_REGION").getOrElse("us-west-2"), + ) + jvmArgs("--add-exports", "java.base/sun.nio.ch=ALL-UNNAMED") + // Need to allow a java security manager after Java 21, for Subject.getSubject to work + // "getSubject is supported only if a security manager is allowed". + systemProperty("java.security.manager", "allow") + val logsDir = project.layout.buildDirectory.get().asFile.resolve("logs") + // delete files from previous runs + doFirst { + // delete log files written by Polaris + logsDir.deleteRecursively() + } + withPolarisServer(configurations.polarisServer) { + environment.put( + "AWS_REGION", + providers.environmentVariable("AWS_REGION").orElse("us-west-2"), + ) + environment.put( + "AWS_ACCESS_KEY_ID", + providers.environmentVariable("AWS_ACCESS_KEY_ID").orElse("ap1"), + ) + environment.put( + "AWS_SECRET_ACCESS_KEY", + providers.environmentVariable("AWS_SECRET_ACCESS_KEY").orElse("s3cr3t"), + ) + environment.put("AWS_EC2_METADATA_DISABLED", "true") + environment.put("POLARIS_BOOTSTRAP_CREDENTIALS", "POLARIS,test-admin,test-secret") + systemProperties.put("quarkus.profile", "it") + systemProperties.put( + "quarkus.log.file.path", + logsDir.resolve("polaris.log").absolutePath, + ) + } + // For Spark integration tests + addSparkJvmOptions() + } + } + } + } +} sourceSets { named("intTest") { @@ -37,143 +176,9 @@ sourceSets { } } -dependencies { - // must be enforced to get a consistent and validated set of dependencies - implementation(enforcedPlatform(libs.quarkus.bom)) { - exclude(group = "org.antlr", module = "antlr4-runtime") - exclude(group = "org.scala-lang", module = "scala-library") - exclude(group = "org.scala-lang", module = "scala-reflect") - } - - // For test configurations, exclude jakarta.servlet-api from Quarkus BOM - // to allow Spark 4.0's version (5.0.0) which includes SingleThreadModel - testImplementation(platform(libs.quarkus.bom)) { - exclude(group = "jakarta.servlet", module = "jakarta.servlet-api") - } - - implementation(project(":polaris-runtime-service")) - - testImplementation( - "org.apache.iceberg:iceberg-spark-runtime-${sparkMajorVersion}_${scalaVersion}:${icebergVersion}" - ) - testImplementation(project(":polaris-spark-${sparkMajorVersion}_${scalaVersion}")) - - testImplementation(project(":polaris-api-management-model")) - - testImplementation(project(":polaris-runtime-test-common")) - - testImplementation("org.apache.spark:spark-sql_${scalaVersion}:${spark40Version}") { - // exclude log4j dependencies. Explicit dependencies for the log4j libraries are - // enforced below to ensure the version compatibility - exclude("org.apache.logging.log4j", "log4j-slf4j2-impl") - exclude("org.apache.logging.log4j", "log4j-1.2-api") - exclude("org.apache.logging.log4j", "log4j-core") - exclude("org.slf4j", "jul-to-slf4j") - } - - // Add spark-hive for Hudi integration - provides HiveExternalCatalog that Hudi needs - testRuntimeOnly("org.apache.spark:spark-hive_${scalaVersion}:${spark40Version}") { - // exclude log4j dependencies to match spark-sql exclusions - exclude("org.apache.logging.log4j", "log4j-slf4j2-impl") - exclude("org.apache.logging.log4j", "log4j-1.2-api") - exclude("org.apache.logging.log4j", "log4j-core") - exclude("org.slf4j", "jul-to-slf4j") - // exclude old slf4j 1.x to log4j 2.x bridge that conflicts with slf4j 2.x bridge - exclude("org.apache.logging.log4j", "log4j-slf4j-impl") - } - // enforce the usage of log4j 2.24.3. This is for the log4j-api compatibility - // of spark-sql dependency - testRuntimeOnly("org.apache.logging.log4j:log4j-core:2.26.0") - - testImplementation("io.delta:delta-spark_${scalaVersion}:4.0.1") - testImplementation("org.apache.hudi:hudi-spark4.0-bundle_${scalaVersion}:1.1.1") { - // exclude log4j dependencies to match spark-sql exclusions - // exclude log4j dependencies to match spark-sql exclusions and prevent version conflicts - exclude("org.apache.logging.log4j", "log4j-slf4j2-impl") - exclude("org.apache.logging.log4j", "log4j-1.2-api") - exclude("org.apache.logging.log4j", "log4j-core") - exclude("org.slf4j", "jul-to-slf4j") - exclude("org.slf4j", "slf4j-log4j12") - exclude("org.slf4j", "slf4j-reload4j") - exclude("ch.qos.reload4j", "reload4j") - exclude("log4j", "log4j") - // exclude old slf4j 1.x to log4j 2.x bridge that conflicts with slf4j 2.x bridge - exclude("org.apache.logging.log4j", "log4j-slf4j-impl") - } - - // The hudi-spark-bundle includes most Hive libraries but excludes hive-exec to keep size - // manageable - // This matches what Spark 4.0 distribution provides (hive-exec-2.3.10-core.jar) - testImplementation("org.apache.hive:hive-exec:2.3.10:core") { - // Exclude conflicting dependencies to use Spark's versions - exclude("org.apache.hadoop", "*") - exclude("org.apache.commons", "*") - exclude("org.slf4j", "*") - exclude("log4j", "*") - exclude("org.apache.logging.log4j", "*") - exclude("org.pentaho", "*") - exclude("org.apache.calcite", "*") - exclude("org.apache.tez", "*") - } - - testImplementation(platform(libs.jackson.bom)) - testImplementation("com.fasterxml.jackson.jakarta.rs:jackson-jakarta-rs-json-provider") - - testImplementation(testFixtures(project(":polaris-runtime-service"))) - - testImplementation(platform(libs.quarkus.bom)) - testImplementation("io.quarkus:quarkus-junit") - testImplementation("io.quarkus:quarkus-rest-client") - testImplementation("io.quarkus:quarkus-rest-client-jackson") - - testImplementation(platform(libs.awssdk.bom)) - testImplementation("software.amazon.awssdk:glue") - testImplementation("software.amazon.awssdk:kms") - testImplementation("software.amazon.awssdk:dynamodb") - - testImplementation(platform(libs.testcontainers.bom)) - testImplementation("org.testcontainers:testcontainers") - testImplementation(libs.s3mock.testcontainers) - - // Required for Spark integration tests - testImplementation(enforcedPlatform("org.scala-lang:scala-library:${scalaLibraryVersion}")) - testImplementation(enforcedPlatform("org.scala-lang:scala-reflect:${scalaLibraryVersion}")) - testImplementation(libs.javax.servlet.api) - testImplementation(libs.antlr4.runtime.spark40) -} - // Force jakarta.servlet-api to 5.0.0 for Spark 4.0 compatibility // Spark 4.0 requires version 5.0.0 which includes SingleThreadModel // Quarkus BOM forces it to 6.x which removed SingleThreadModel configurations.named("intTestRuntimeClasspath") { resolutionStrategy { force("jakarta.servlet:jakarta.servlet-api:5.0.0") } } - -tasks.named("intTest").configure { - if (System.getenv("AWS_REGION") == null) { - environment("AWS_REGION", "us-west-2") - } - // Note: the test secrets are referenced in - // org.apache.polaris.service.it.ServerManager - environment("POLARIS_BOOTSTRAP_CREDENTIALS", "POLARIS,test-admin,test-secret") - jvmArgs("--add-exports", "java.base/sun.nio.ch=ALL-UNNAMED") - // Need to allow a java security manager after Java 21, for Subject.getSubject to work - // "getSubject is supported only if a security manager is allowed". - systemProperty("java.security.manager", "allow") - // Same issue as above: allow a java security manager after Java 21 - // (this setting is for the application under test, while the setting above is for test code). - systemProperty("quarkus.test.arg-line", "-Djava.security.manager=allow") - val logsDir = project.layout.buildDirectory.get().asFile.resolve("logs") - // delete files from previous runs - doFirst { - // delete log files written by Polaris - logsDir.deleteRecursively() - // delete quarkus.log file (captured Polaris stdout/stderr) - project.layout.buildDirectory.get().asFile.resolve("quarkus.log").delete() - } - // This property is not honored in a per-profile application.properties file, - // so we need to set it here. - systemProperty("quarkus.log.file.path", logsDir.resolve("polaris.log").absolutePath) - // For Spark integration tests - addSparkJvmOptions() -} diff --git a/plugins/spark/v4.0/integration/src/intTest/java/org/apache/polaris/spark/quarkus/it/SparkCatalogBaseIT.java b/plugins/spark/v4.0/integration/src/intTest/java/org/apache/polaris/spark/quarkus/it/SparkCatalogBaseIT.java index 76b99d1c144..d328484cc1a 100644 --- a/plugins/spark/v4.0/integration/src/intTest/java/org/apache/polaris/spark/quarkus/it/SparkCatalogBaseIT.java +++ b/plugins/spark/v4.0/integration/src/intTest/java/org/apache/polaris/spark/quarkus/it/SparkCatalogBaseIT.java @@ -23,7 +23,6 @@ import com.google.common.base.Preconditions; import com.google.common.collect.Maps; -import io.quarkus.test.junit.QuarkusIntegrationTest; import java.util.Arrays; import java.util.Map; import org.apache.iceberg.exceptions.BadRequestException; @@ -50,7 +49,6 @@ * SparkCatalog operations, some operations like listNamespaces under a namespace can not be * triggered through a SQL interface directly with Spark. */ -@QuarkusIntegrationTest public abstract class SparkCatalogBaseIT extends SparkIntegrationBase { private static StructType schema = new StructType().add("id", "long").add("name", "string"); protected StagingTableCatalog tableCatalog = null; From 8bcaccce248a9ba73cb228b571d5af4a3a669088 Mon Sep 17 00:00:00 2001 From: Robert Stupp Date: Tue, 9 Jun 2026 11:47:24 +0200 Subject: [PATCH 05/22] Move Spark integration tests into Spark projects --- bom/build.gradle.kts | 3 - .../spark/v3.5/integration/build.gradle.kts | 182 ----------------- plugins/spark/v3.5/spark/build.gradle.kts | 167 +++++++++++++++- .../spark/quarkus/it/SparkCatalogBaseIT.java | 0 .../spark/v4.0/integration/build.gradle.kts | 184 ------------------ plugins/spark/v4.0/spark/build.gradle.kts | 174 ++++++++++++++++- .../spark/quarkus/it/SparkCatalogBaseIT.java | 0 settings.gradle.kts | 6 - 8 files changed, 329 insertions(+), 387 deletions(-) delete mode 100644 plugins/spark/v3.5/integration/build.gradle.kts rename plugins/spark/v3.5/{integration => spark}/src/intTest/java/org/apache/polaris/spark/quarkus/it/SparkCatalogBaseIT.java (100%) delete mode 100644 plugins/spark/v4.0/integration/build.gradle.kts rename plugins/spark/v4.0/{integration => spark}/src/intTest/java/org/apache/polaris/spark/quarkus/it/SparkCatalogBaseIT.java (100%) diff --git a/bom/build.gradle.kts b/bom/build.gradle.kts index 6a67f1e9488..b84a9ce0742 100644 --- a/bom/build.gradle.kts +++ b/bom/build.gradle.kts @@ -104,13 +104,10 @@ dependencies { api(project(":polaris-hms-testcontainer")) api(project(":polaris-spark-3.5_2.12")) - api(project(":polaris-spark-integration-3.5_2.12")) val ideaActive = providers.systemProperty("idea.active").getOrElse("false").toBoolean() if (!ideaActive) { api(project(":polaris-spark-3.5_2.13")) - api(project(":polaris-spark-integration-3.5_2.13")) api(project(":polaris-spark-4.0_2.13")) - api(project(":polaris-spark-integration-4.0_2.13")) } api(project(":polaris-admin")) diff --git a/plugins/spark/v3.5/integration/build.gradle.kts b/plugins/spark/v3.5/integration/build.gradle.kts deleted file mode 100644 index 645db456507..00000000000 --- a/plugins/spark/v3.5/integration/build.gradle.kts +++ /dev/null @@ -1,182 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -import org.gradle.api.plugins.jvm.JvmTestSuite - -plugins { - id("polaris-java") - id("polaris-server-test-runner") -} - -// get version information -val sparkMajorVersion = "3.5" -val scalaVersion = getAndUseScalaVersionForProject() -val icebergVersion = libs.versions.iceberg.get() -val spark35Version = libs.versions.spark35.get() -val scalaLibraryVersion = - if (scalaVersion == "2.12") { - libs.versions.scala212.get() - } else { - libs.versions.scala213.get() - } -val errorProneAnnotationsVersion = libs.errorprone.get().versionConstraint.requiredVersion - -dependencies { polarisServer(project(path = ":polaris-server", configuration = "quarkusRunner")) } - -testing { - suites { - register("intTest") { - dependencies { - implementation( - "org.apache.iceberg:iceberg-spark-runtime-${sparkMajorVersion}_${scalaVersion}:${icebergVersion}" - ) - implementation(project(":polaris-spark-${sparkMajorVersion}_${scalaVersion}")) - - implementation(project(":polaris-api-management-model")) - - implementation(project(":polaris-runtime-test-common")) - - implementation("org.apache.spark:spark-sql_${scalaVersion}:${spark35Version}") { - // exclude log4j dependencies. Explicit dependencies for the log4j libraries are - // enforced below to ensure the version compatibility - exclude("org.apache.logging.log4j", "log4j-slf4j2-impl") - exclude("org.apache.logging.log4j", "log4j-1.2-api") - exclude("org.apache.logging.log4j", "log4j-core") - exclude("org.slf4j", "jul-to-slf4j") - } - - // Add spark-hive for Hudi integration - provides HiveExternalCatalog that Hudi needs - runtimeOnly("org.apache.spark:spark-hive_${scalaVersion}:${spark35Version}") { - // exclude log4j dependencies to match spark-sql exclusions - exclude("org.apache.logging.log4j", "log4j-slf4j2-impl") - exclude("org.apache.logging.log4j", "log4j-1.2-api") - exclude("org.apache.logging.log4j", "log4j-core") - exclude("org.slf4j", "jul-to-slf4j") - // exclude old slf4j 1.x to log4j 2.x bridge that conflicts with slf4j 2.x bridge - exclude("org.apache.logging.log4j", "log4j-slf4j-impl") - } - // enforce the usage of log4j 2.24.3. This is for the log4j-api compatibility - // of spark-sql dependency - runtimeOnly("org.apache.logging.log4j:log4j-core:2.26.0") - - implementation("io.delta:delta-spark_${scalaVersion}:3.3.1") - implementation("org.apache.hudi:hudi-spark3.5-bundle_${scalaVersion}:1.1.1") { - // exclude log4j dependencies to match spark-sql exclusions - // exclude log4j dependencies to match spark-sql exclusions and prevent version conflicts - exclude("org.apache.logging.log4j", "log4j-slf4j2-impl") - exclude("org.apache.logging.log4j", "log4j-1.2-api") - exclude("org.apache.logging.log4j", "log4j-core") - exclude("org.slf4j", "jul-to-slf4j") - exclude("org.slf4j", "slf4j-log4j12") - exclude("org.slf4j", "slf4j-reload4j") - exclude("ch.qos.reload4j", "reload4j") - exclude("log4j", "log4j") - // exclude old slf4j 1.x to log4j 2.x bridge that conflicts with slf4j 2.x bridge - exclude("org.apache.logging.log4j", "log4j-slf4j-impl") - } - - // The hudi-spark-bundle includes most Hive libraries but excludes hive-exec to keep size - // manageable - // This matches what Spark 3.5 distribution provides (hive-exec-2.3.10-core.jar) - implementation("org.apache.hive:hive-exec:2.3.10:core") { - // Exclude conflicting dependencies to use Spark's versions - exclude("org.apache.hadoop", "*") - exclude("org.apache.commons", "*") - exclude("org.slf4j", "*") - exclude("log4j", "*") - exclude("org.apache.logging.log4j", "*") - exclude("org.pentaho", "*") - exclude("org.apache.calcite", "*") - exclude("org.apache.tez", "*") - } - - implementation(platform(libs.jackson.bom)) - implementation("com.fasterxml.jackson.jakarta.rs:jackson-jakarta-rs-json-provider") - implementation(libs.jakarta.ws.rs.api) - compileOnly("com.google.errorprone:error_prone_annotations:${errorProneAnnotationsVersion}") - - implementation(testFixtures(project(":polaris-runtime-service"))) - - implementation(platform(libs.awssdk.bom)) - implementation("software.amazon.awssdk:glue") - implementation("software.amazon.awssdk:kms") - implementation("software.amazon.awssdk:dynamodb") - - implementation(platform(libs.testcontainers.bom)) - implementation("org.testcontainers:testcontainers") - implementation(libs.s3mock.testcontainers) - - // Required for Spark integration tests - implementation(enforcedPlatform("org.scala-lang:scala-library:${scalaLibraryVersion}")) - implementation(enforcedPlatform("org.scala-lang:scala-reflect:${scalaLibraryVersion}")) - implementation(libs.javax.servlet.api) - implementation(libs.antlr4.runtime.spark35) - } - - targets.all { - testTask.configure { - environment( - "AWS_REGION", - providers.environmentVariable("AWS_REGION").getOrElse("us-west-2"), - ) - jvmArgs("--add-exports", "java.base/sun.nio.ch=ALL-UNNAMED") - // Need to allow a java security manager after Java 21, for Subject.getSubject to work - // "getSubject is supported only if a security manager is allowed". - systemProperty("java.security.manager", "allow") - val logsDir = project.layout.buildDirectory.get().asFile.resolve("logs") - // delete files from previous runs - doFirst { - // delete log files written by Polaris - logsDir.deleteRecursively() - } - withPolarisServer(configurations.polarisServer) { - environment.put( - "AWS_REGION", - providers.environmentVariable("AWS_REGION").orElse("us-west-2"), - ) - environment.put( - "AWS_ACCESS_KEY_ID", - providers.environmentVariable("AWS_ACCESS_KEY_ID").orElse("ap1"), - ) - environment.put( - "AWS_SECRET_ACCESS_KEY", - providers.environmentVariable("AWS_SECRET_ACCESS_KEY").orElse("s3cr3t"), - ) - environment.put("AWS_EC2_METADATA_DISABLED", "true") - environment.put("POLARIS_BOOTSTRAP_CREDENTIALS", "POLARIS,test-admin,test-secret") - systemProperties.put("quarkus.profile", "it") - systemProperties.put( - "quarkus.log.file.path", - logsDir.resolve("polaris.log").absolutePath, - ) - } - // For Spark integration tests - addSparkJvmOptions() - } - } - } - } -} - -sourceSets { - named("intTest") { - java { srcDir("../../common/src/intTest/java") } - resources { srcDir("../../common/src/intTest/resources") } - } -} diff --git a/plugins/spark/v3.5/spark/build.gradle.kts b/plugins/spark/v3.5/spark/build.gradle.kts index e0397dbaf3d..30ec54ded42 100644 --- a/plugins/spark/v3.5/spark/build.gradle.kts +++ b/plugins/spark/v3.5/spark/build.gradle.kts @@ -18,8 +18,13 @@ */ import com.github.jengelman.gradle.plugins.shadow.tasks.ShadowJar +import org.gradle.api.attributes.java.TargetJvmVersion +import org.gradle.api.plugins.jvm.JvmTestSuite -plugins { id("polaris-client") } +plugins { + id("polaris-client") + id("polaris-server-test-runner") +} checkstyle { configProperties = @@ -29,11 +34,6 @@ checkstyle { ) } -sourceSets { - main { java { srcDir("../../common/src/main/java") } } - test { java { srcDir("../../common/src/test/java") } } -} - // get version information val sparkMajorVersion = "3.5" val scalaVersion = getAndUseScalaVersionForProject() @@ -46,8 +46,12 @@ val scalaLibraryVersion = } else { libs.versions.scala213.get() } +val errorProneAnnotationsVersion = libs.errorprone.get().versionConstraint.requiredVersion +val intTestJvmVersion = 21 dependencies { + polarisServer(project(path = ":polaris-server", configuration = "quarkusRunner")) + // TODO: extract a polaris-rest module as a thin layer for // client to depends on. implementation(project(":polaris-core")) { isTransitive = false } @@ -89,6 +93,157 @@ dependencies { } } +testing { + suites { + register("intTest") { + dependencies { + implementation( + "org.apache.iceberg:iceberg-spark-runtime-${sparkMajorVersion}_${scalaVersion}:${icebergVersion}" + ) + + implementation(project(":polaris-api-management-model")) + + implementation(project(":polaris-runtime-test-common")) + + implementation("org.apache.spark:spark-sql_${scalaVersion}:${spark35Version}") { + // exclude log4j dependencies. Explicit dependencies for the log4j libraries are + // enforced below to ensure the version compatibility + exclude("org.apache.logging.log4j", "log4j-slf4j2-impl") + exclude("org.apache.logging.log4j", "log4j-1.2-api") + exclude("org.apache.logging.log4j", "log4j-core") + exclude("org.slf4j", "jul-to-slf4j") + } + + // Add spark-hive for Hudi integration - provides HiveExternalCatalog that Hudi needs + runtimeOnly("org.apache.spark:spark-hive_${scalaVersion}:${spark35Version}") { + // exclude log4j dependencies to match spark-sql exclusions + exclude("org.apache.logging.log4j", "log4j-slf4j2-impl") + exclude("org.apache.logging.log4j", "log4j-1.2-api") + exclude("org.apache.logging.log4j", "log4j-core") + exclude("org.slf4j", "jul-to-slf4j") + // exclude old slf4j 1.x to log4j 2.x bridge that conflicts with slf4j 2.x bridge + exclude("org.apache.logging.log4j", "log4j-slf4j-impl") + } + // enforce the usage of log4j 2.24.3. This is for the log4j-api compatibility + // of spark-sql dependency + runtimeOnly("org.apache.logging.log4j:log4j-core:2.26.0") + + implementation("io.delta:delta-spark_${scalaVersion}:3.3.1") + implementation("org.apache.hudi:hudi-spark3.5-bundle_${scalaVersion}:1.1.1") { + // exclude log4j dependencies to match spark-sql exclusions + // exclude log4j dependencies to match spark-sql exclusions and prevent version conflicts + exclude("org.apache.logging.log4j", "log4j-slf4j2-impl") + exclude("org.apache.logging.log4j", "log4j-1.2-api") + exclude("org.apache.logging.log4j", "log4j-core") + exclude("org.slf4j", "jul-to-slf4j") + exclude("org.slf4j", "slf4j-log4j12") + exclude("org.slf4j", "slf4j-reload4j") + exclude("ch.qos.reload4j", "reload4j") + exclude("log4j", "log4j") + // exclude old slf4j 1.x to log4j 2.x bridge that conflicts with slf4j 2.x bridge + exclude("org.apache.logging.log4j", "log4j-slf4j-impl") + } + + // The hudi-spark-bundle includes most Hive libraries but excludes hive-exec to keep size + // manageable + // This matches what Spark 3.5 distribution provides (hive-exec-2.3.10-core.jar) + implementation("org.apache.hive:hive-exec:2.3.10:core") { + // Exclude conflicting dependencies to use Spark's versions + exclude("org.apache.hadoop", "*") + exclude("org.apache.commons", "*") + exclude("org.slf4j", "*") + exclude("log4j", "*") + exclude("org.apache.logging.log4j", "*") + exclude("org.pentaho", "*") + exclude("org.apache.calcite", "*") + exclude("org.apache.tez", "*") + } + + implementation(platform(libs.jackson.bom)) + implementation("com.fasterxml.jackson.jakarta.rs:jackson-jakarta-rs-json-provider") + implementation(libs.jakarta.ws.rs.api) + compileOnly("com.google.errorprone:error_prone_annotations:${errorProneAnnotationsVersion}") + + implementation(testFixtures(project(":polaris-runtime-service"))) + + implementation(platform(libs.awssdk.bom)) + implementation("software.amazon.awssdk:glue") + implementation("software.amazon.awssdk:kms") + implementation("software.amazon.awssdk:dynamodb") + + implementation(platform(libs.testcontainers.bom)) + implementation("org.testcontainers:testcontainers") + implementation(libs.s3mock.testcontainers) + + // Required for Spark integration tests + implementation(enforcedPlatform("org.scala-lang:scala-library:${scalaLibraryVersion}")) + implementation(enforcedPlatform("org.scala-lang:scala-reflect:${scalaLibraryVersion}")) + implementation(libs.javax.servlet.api) + implementation(libs.antlr4.runtime.spark35) + } + + targets.all { + testTask.configure { + environment( + "AWS_REGION", + providers.environmentVariable("AWS_REGION").getOrElse("us-west-2"), + ) + jvmArgs("--add-exports", "java.base/sun.nio.ch=ALL-UNNAMED") + // Need to allow a java security manager after Java 21, for Subject.getSubject to work + // "getSubject is supported only if a security manager is allowed". + systemProperty("java.security.manager", "allow") + val logsDir = project.layout.buildDirectory.get().asFile.resolve("logs") + // delete files from previous runs + doFirst { + // delete log files written by Polaris + logsDir.deleteRecursively() + } + withPolarisServer(configurations.polarisServer) { + environment.put( + "AWS_REGION", + providers.environmentVariable("AWS_REGION").orElse("us-west-2"), + ) + environment.put( + "AWS_ACCESS_KEY_ID", + providers.environmentVariable("AWS_ACCESS_KEY_ID").orElse("ap1"), + ) + environment.put( + "AWS_SECRET_ACCESS_KEY", + providers.environmentVariable("AWS_SECRET_ACCESS_KEY").orElse("s3cr3t"), + ) + environment.put("AWS_EC2_METADATA_DISABLED", "true") + environment.put("POLARIS_BOOTSTRAP_CREDENTIALS", "POLARIS,test-admin,test-secret") + systemProperties.put("quarkus.profile", "it") + systemProperties.put( + "quarkus.log.file.path", + logsDir.resolve("polaris.log").absolutePath, + ) + } + // For Spark integration tests + addSparkJvmOptions() + } + } + } + } +} + +sourceSets { + main { java { srcDir("../../common/src/main/java") } } + test { java { srcDir("../../common/src/test/java") } } + named("intTest") { + java { srcDir("../../common/src/intTest/java") } + resources { srcDir("../../common/src/intTest/resources") } + } +} + +listOf("intTestCompileClasspath", "intTestRuntimeClasspath").forEach { + // :polaris-runtime-test-common and :polaris-runtime-service (testFixtures) require JVM 21 + // compatibility. + configurations.named(it).configure { + attributes.attribute(TargetJvmVersion.TARGET_JVM_VERSION_ATTRIBUTE, intTestJvmVersion) + } +} + tasks.register("createPolarisSparkJar") { archiveClassifier = "bundle" isZip64 = true diff --git a/plugins/spark/v3.5/integration/src/intTest/java/org/apache/polaris/spark/quarkus/it/SparkCatalogBaseIT.java b/plugins/spark/v3.5/spark/src/intTest/java/org/apache/polaris/spark/quarkus/it/SparkCatalogBaseIT.java similarity index 100% rename from plugins/spark/v3.5/integration/src/intTest/java/org/apache/polaris/spark/quarkus/it/SparkCatalogBaseIT.java rename to plugins/spark/v3.5/spark/src/intTest/java/org/apache/polaris/spark/quarkus/it/SparkCatalogBaseIT.java diff --git a/plugins/spark/v4.0/integration/build.gradle.kts b/plugins/spark/v4.0/integration/build.gradle.kts deleted file mode 100644 index fe61e0f42d1..00000000000 --- a/plugins/spark/v4.0/integration/build.gradle.kts +++ /dev/null @@ -1,184 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -import org.gradle.api.plugins.jvm.JvmTestSuite - -plugins { - id("polaris-java") - id("polaris-server-test-runner") -} - -// get version information -val sparkMajorVersion = "4.0" -val scalaVersion = getAndUseScalaVersionForProject() -val icebergVersion = libs.versions.iceberg.get() -val spark40Version = libs.versions.spark40.get() -val scalaLibraryVersion = libs.versions.scala213.get() -val errorProneAnnotationsVersion = libs.errorprone.get().versionConstraint.requiredVersion - -dependencies { polarisServer(project(path = ":polaris-server", configuration = "quarkusRunner")) } - -testing { - suites { - register("intTest") { - dependencies { - implementation( - "org.apache.iceberg:iceberg-spark-runtime-${sparkMajorVersion}_${scalaVersion}:${icebergVersion}" - ) - implementation(project(":polaris-spark-${sparkMajorVersion}_${scalaVersion}")) - - implementation(project(":polaris-api-management-model")) - - implementation(project(":polaris-runtime-test-common")) - - implementation("org.apache.spark:spark-sql_${scalaVersion}:${spark40Version}") { - // exclude log4j dependencies. Explicit dependencies for the log4j libraries are - // enforced below to ensure the version compatibility - exclude("org.apache.logging.log4j", "log4j-slf4j2-impl") - exclude("org.apache.logging.log4j", "log4j-1.2-api") - exclude("org.apache.logging.log4j", "log4j-core") - exclude("org.slf4j", "jul-to-slf4j") - } - - // Add spark-hive for Hudi integration - provides HiveExternalCatalog that Hudi needs - runtimeOnly("org.apache.spark:spark-hive_${scalaVersion}:${spark40Version}") { - // exclude log4j dependencies to match spark-sql exclusions - exclude("org.apache.logging.log4j", "log4j-slf4j2-impl") - exclude("org.apache.logging.log4j", "log4j-1.2-api") - exclude("org.apache.logging.log4j", "log4j-core") - exclude("org.slf4j", "jul-to-slf4j") - // exclude old slf4j 1.x to log4j 2.x bridge that conflicts with slf4j 2.x bridge - exclude("org.apache.logging.log4j", "log4j-slf4j-impl") - } - // enforce the usage of log4j 2.24.3. This is for the log4j-api compatibility - // of spark-sql dependency - runtimeOnly("org.apache.logging.log4j:log4j-core:2.26.0") - - implementation("io.delta:delta-spark_${scalaVersion}:4.0.1") - implementation("org.apache.hudi:hudi-spark4.0-bundle_${scalaVersion}:1.1.1") { - // exclude log4j dependencies to match spark-sql exclusions - // exclude log4j dependencies to match spark-sql exclusions and prevent version conflicts - exclude("org.apache.logging.log4j", "log4j-slf4j2-impl") - exclude("org.apache.logging.log4j", "log4j-1.2-api") - exclude("org.apache.logging.log4j", "log4j-core") - exclude("org.slf4j", "jul-to-slf4j") - exclude("org.slf4j", "slf4j-log4j12") - exclude("org.slf4j", "slf4j-reload4j") - exclude("ch.qos.reload4j", "reload4j") - exclude("log4j", "log4j") - // exclude old slf4j 1.x to log4j 2.x bridge that conflicts with slf4j 2.x bridge - exclude("org.apache.logging.log4j", "log4j-slf4j-impl") - } - - // The hudi-spark-bundle includes most Hive libraries but excludes hive-exec to keep size - // manageable - // This matches what Spark 4.0 distribution provides (hive-exec-2.3.10-core.jar) - implementation("org.apache.hive:hive-exec:2.3.10:core") { - // Exclude conflicting dependencies to use Spark's versions - exclude("org.apache.hadoop", "*") - exclude("org.apache.commons", "*") - exclude("org.slf4j", "*") - exclude("log4j", "*") - exclude("org.apache.logging.log4j", "*") - exclude("org.pentaho", "*") - exclude("org.apache.calcite", "*") - exclude("org.apache.tez", "*") - } - - implementation(platform(libs.jackson.bom)) - implementation("com.fasterxml.jackson.jakarta.rs:jackson-jakarta-rs-json-provider") - implementation(libs.jakarta.ws.rs.api) - compileOnly("com.google.errorprone:error_prone_annotations:${errorProneAnnotationsVersion}") - - implementation(testFixtures(project(":polaris-runtime-service"))) - - implementation(platform(libs.awssdk.bom)) - implementation("software.amazon.awssdk:glue") - implementation("software.amazon.awssdk:kms") - implementation("software.amazon.awssdk:dynamodb") - - implementation(platform(libs.testcontainers.bom)) - implementation("org.testcontainers:testcontainers") - implementation(libs.s3mock.testcontainers) - - // Required for Spark integration tests - implementation(enforcedPlatform("org.scala-lang:scala-library:${scalaLibraryVersion}")) - implementation(enforcedPlatform("org.scala-lang:scala-reflect:${scalaLibraryVersion}")) - implementation(libs.javax.servlet.api) - implementation(libs.antlr4.runtime.spark40) - } - - targets.all { - testTask.configure { - environment( - "AWS_REGION", - providers.environmentVariable("AWS_REGION").getOrElse("us-west-2"), - ) - jvmArgs("--add-exports", "java.base/sun.nio.ch=ALL-UNNAMED") - // Need to allow a java security manager after Java 21, for Subject.getSubject to work - // "getSubject is supported only if a security manager is allowed". - systemProperty("java.security.manager", "allow") - val logsDir = project.layout.buildDirectory.get().asFile.resolve("logs") - // delete files from previous runs - doFirst { - // delete log files written by Polaris - logsDir.deleteRecursively() - } - withPolarisServer(configurations.polarisServer) { - environment.put( - "AWS_REGION", - providers.environmentVariable("AWS_REGION").orElse("us-west-2"), - ) - environment.put( - "AWS_ACCESS_KEY_ID", - providers.environmentVariable("AWS_ACCESS_KEY_ID").orElse("ap1"), - ) - environment.put( - "AWS_SECRET_ACCESS_KEY", - providers.environmentVariable("AWS_SECRET_ACCESS_KEY").orElse("s3cr3t"), - ) - environment.put("AWS_EC2_METADATA_DISABLED", "true") - environment.put("POLARIS_BOOTSTRAP_CREDENTIALS", "POLARIS,test-admin,test-secret") - systemProperties.put("quarkus.profile", "it") - systemProperties.put( - "quarkus.log.file.path", - logsDir.resolve("polaris.log").absolutePath, - ) - } - // For Spark integration tests - addSparkJvmOptions() - } - } - } - } -} - -sourceSets { - named("intTest") { - java { srcDir("../../common/src/intTest/java") } - resources { srcDir("../../common/src/intTest/resources") } - } -} - -// Force jakarta.servlet-api to 5.0.0 for Spark 4.0 compatibility -// Spark 4.0 requires version 5.0.0 which includes SingleThreadModel -// Quarkus BOM forces it to 6.x which removed SingleThreadModel -configurations.named("intTestRuntimeClasspath") { - resolutionStrategy { force("jakarta.servlet:jakarta.servlet-api:5.0.0") } -} diff --git a/plugins/spark/v4.0/spark/build.gradle.kts b/plugins/spark/v4.0/spark/build.gradle.kts index d2437cd1809..7fa04c77a4b 100644 --- a/plugins/spark/v4.0/spark/build.gradle.kts +++ b/plugins/spark/v4.0/spark/build.gradle.kts @@ -18,8 +18,13 @@ */ import com.github.jengelman.gradle.plugins.shadow.tasks.ShadowJar +import org.gradle.api.attributes.java.TargetJvmVersion +import org.gradle.api.plugins.jvm.JvmTestSuite -plugins { id("polaris-client") } +plugins { + id("polaris-client") + id("polaris-server-test-runner") +} checkstyle { configProperties = @@ -29,11 +34,6 @@ checkstyle { ) } -sourceSets { - main { java { srcDir("../../common/src/main/java") } } - test { java { srcDir("../../common/src/test/java") } } -} - // get version information val sparkMajorVersion = "4.0" val scalaVersion = getAndUseScalaVersionForProject() @@ -41,8 +41,12 @@ val icebergVersion = libs.versions.iceberg.get() val spark40Version = libs.versions.spark40.get() val scalaLibraryVersion = libs.versions.scala213.get() +val errorProneAnnotationsVersion = libs.errorprone.get().versionConstraint.requiredVersion +val intTestJvmVersion = 21 dependencies { + polarisServer(project(path = ":polaris-server", configuration = "quarkusRunner")) + // TODO: extract a polaris-rest module as a thin layer for // client to depends on. implementation(project(":polaris-core")) { isTransitive = false } @@ -85,6 +89,164 @@ dependencies { testImplementation("org.apache.logging.log4j:log4j-api:2.26.0") } +testing { + suites { + register("intTest") { + dependencies { + implementation( + "org.apache.iceberg:iceberg-spark-runtime-${sparkMajorVersion}_${scalaVersion}:${icebergVersion}" + ) + + implementation(project(":polaris-api-management-model")) + + implementation(project(":polaris-runtime-test-common")) + + implementation("org.apache.spark:spark-sql_${scalaVersion}:${spark40Version}") { + // exclude log4j dependencies. Explicit dependencies for the log4j libraries are + // enforced below to ensure the version compatibility + exclude("org.apache.logging.log4j", "log4j-slf4j2-impl") + exclude("org.apache.logging.log4j", "log4j-1.2-api") + exclude("org.apache.logging.log4j", "log4j-core") + exclude("org.slf4j", "jul-to-slf4j") + } + + // Add spark-hive for Hudi integration - provides HiveExternalCatalog that Hudi needs + runtimeOnly("org.apache.spark:spark-hive_${scalaVersion}:${spark40Version}") { + // exclude log4j dependencies to match spark-sql exclusions + exclude("org.apache.logging.log4j", "log4j-slf4j2-impl") + exclude("org.apache.logging.log4j", "log4j-1.2-api") + exclude("org.apache.logging.log4j", "log4j-core") + exclude("org.slf4j", "jul-to-slf4j") + // exclude old slf4j 1.x to log4j 2.x bridge that conflicts with slf4j 2.x bridge + exclude("org.apache.logging.log4j", "log4j-slf4j-impl") + } + // enforce the usage of log4j 2.24.3. This is for the log4j-api compatibility + // of spark-sql dependency + runtimeOnly("org.apache.logging.log4j:log4j-core:2.26.0") + + implementation("io.delta:delta-spark_${scalaVersion}:4.0.1") + implementation("org.apache.hudi:hudi-spark4.0-bundle_${scalaVersion}:1.1.1") { + // exclude log4j dependencies to match spark-sql exclusions + // exclude log4j dependencies to match spark-sql exclusions and prevent version conflicts + exclude("org.apache.logging.log4j", "log4j-slf4j2-impl") + exclude("org.apache.logging.log4j", "log4j-1.2-api") + exclude("org.apache.logging.log4j", "log4j-core") + exclude("org.slf4j", "jul-to-slf4j") + exclude("org.slf4j", "slf4j-log4j12") + exclude("org.slf4j", "slf4j-reload4j") + exclude("ch.qos.reload4j", "reload4j") + exclude("log4j", "log4j") + // exclude old slf4j 1.x to log4j 2.x bridge that conflicts with slf4j 2.x bridge + exclude("org.apache.logging.log4j", "log4j-slf4j-impl") + } + + // The hudi-spark-bundle includes most Hive libraries but excludes hive-exec to keep size + // manageable + // This matches what Spark 4.0 distribution provides (hive-exec-2.3.10-core.jar) + implementation("org.apache.hive:hive-exec:2.3.10:core") { + // Exclude conflicting dependencies to use Spark's versions + exclude("org.apache.hadoop", "*") + exclude("org.apache.commons", "*") + exclude("org.slf4j", "*") + exclude("log4j", "*") + exclude("org.apache.logging.log4j", "*") + exclude("org.pentaho", "*") + exclude("org.apache.calcite", "*") + exclude("org.apache.tez", "*") + } + + implementation(platform(libs.jackson.bom)) + implementation("com.fasterxml.jackson.jakarta.rs:jackson-jakarta-rs-json-provider") + implementation(libs.jakarta.ws.rs.api) + compileOnly("com.google.errorprone:error_prone_annotations:${errorProneAnnotationsVersion}") + + implementation(testFixtures(project(":polaris-runtime-service"))) + + implementation(platform(libs.awssdk.bom)) + implementation("software.amazon.awssdk:glue") + implementation("software.amazon.awssdk:kms") + implementation("software.amazon.awssdk:dynamodb") + + implementation(platform(libs.testcontainers.bom)) + implementation("org.testcontainers:testcontainers") + implementation(libs.s3mock.testcontainers) + + // Required for Spark integration tests + implementation(enforcedPlatform("org.scala-lang:scala-library:${scalaLibraryVersion}")) + implementation(enforcedPlatform("org.scala-lang:scala-reflect:${scalaLibraryVersion}")) + implementation(libs.javax.servlet.api) + implementation(libs.antlr4.runtime.spark40) + } + + targets.all { + testTask.configure { + environment( + "AWS_REGION", + providers.environmentVariable("AWS_REGION").getOrElse("us-west-2"), + ) + jvmArgs("--add-exports", "java.base/sun.nio.ch=ALL-UNNAMED") + // Need to allow a java security manager after Java 21, for Subject.getSubject to work + // "getSubject is supported only if a security manager is allowed". + systemProperty("java.security.manager", "allow") + val logsDir = project.layout.buildDirectory.get().asFile.resolve("logs") + // delete files from previous runs + doFirst { + // delete log files written by Polaris + logsDir.deleteRecursively() + } + withPolarisServer(configurations.polarisServer) { + environment.put( + "AWS_REGION", + providers.environmentVariable("AWS_REGION").orElse("us-west-2"), + ) + environment.put( + "AWS_ACCESS_KEY_ID", + providers.environmentVariable("AWS_ACCESS_KEY_ID").orElse("ap1"), + ) + environment.put( + "AWS_SECRET_ACCESS_KEY", + providers.environmentVariable("AWS_SECRET_ACCESS_KEY").orElse("s3cr3t"), + ) + environment.put("AWS_EC2_METADATA_DISABLED", "true") + environment.put("POLARIS_BOOTSTRAP_CREDENTIALS", "POLARIS,test-admin,test-secret") + systemProperties.put("quarkus.profile", "it") + systemProperties.put( + "quarkus.log.file.path", + logsDir.resolve("polaris.log").absolutePath, + ) + } + // For Spark integration tests + addSparkJvmOptions() + } + } + } + } +} + +sourceSets { + main { java { srcDir("../../common/src/main/java") } } + test { java { srcDir("../../common/src/test/java") } } + named("intTest") { + java { srcDir("../../common/src/intTest/java") } + resources { srcDir("../../common/src/intTest/resources") } + } +} + +listOf("intTestCompileClasspath", "intTestRuntimeClasspath").forEach { + // :polaris-runtime-test-common and :polaris-runtime-service (testFixtures) require JVM 21 + // compatibility. + configurations.named(it).configure { + attributes.attribute(TargetJvmVersion.TARGET_JVM_VERSION_ATTRIBUTE, intTestJvmVersion) + } +} + +// Force jakarta.servlet-api to 5.0.0 for Spark 4.0 compatibility +// Spark 4.0 requires version 5.0.0 which includes SingleThreadModel +// Quarkus BOM forces it to 6.x which removed SingleThreadModel +configurations.named("intTestRuntimeClasspath") { + resolutionStrategy { force("jakarta.servlet:jakarta.servlet-api:5.0.0") } +} + tasks.register("createPolarisSparkJar") { archiveClassifier = "bundle" isZip64 = true diff --git a/plugins/spark/v4.0/integration/src/intTest/java/org/apache/polaris/spark/quarkus/it/SparkCatalogBaseIT.java b/plugins/spark/v4.0/spark/src/intTest/java/org/apache/polaris/spark/quarkus/it/SparkCatalogBaseIT.java similarity index 100% rename from plugins/spark/v4.0/integration/src/intTest/java/org/apache/polaris/spark/quarkus/it/SparkCatalogBaseIT.java rename to plugins/spark/v4.0/spark/src/intTest/java/org/apache/polaris/spark/quarkus/it/SparkCatalogBaseIT.java diff --git a/settings.gradle.kts b/settings.gradle.kts index 6eefe52a29f..5eea168385a 100644 --- a/settings.gradle.kts +++ b/settings.gradle.kts @@ -99,20 +99,14 @@ for (sparkVersion in sparkVersions) { var first = true for (scalaVersion in scalaVersions) { val sparkArtifactId = "polaris-spark-${sparkVersion}_${scalaVersion}" - val sparkIntArtifactId = "polaris-spark-integration-${sparkVersion}_${scalaVersion}" polarisProject( "polaris-spark-${sparkVersion}_${scalaVersion}", file("${polarisSparkDir}/v${sparkVersion}/spark"), ) - polarisProject( - "polaris-spark-integration-${sparkVersion}_${scalaVersion}", - file("${polarisSparkDir}/v${sparkVersion}/integration"), - ) if (first) { first = false } else { noSourceChecksProjects.add(":$sparkArtifactId") - noSourceChecksProjects.add(":$sparkIntArtifactId") } // Skip all duplicated spark client projects while using Intelij IDE. // This is to avoid problems during dependency analysis and sync when From 8c11c6318a5d6b70fc7890d2b841028c7e9f144b Mon Sep 17 00:00:00 2001 From: Robert Stupp Date: Tue, 9 Jun 2026 12:05:19 +0200 Subject: [PATCH 06/22] Migrate Ranger integration tests to server runner --- extensions/auth/ranger/tests/build.gradle.kts | 124 +++++++++++------- .../ranger/test/RangerAdminServiceIT.java | 4 - .../test/RangerGenericTableHandlerIT.java | 4 - .../test/RangerIcebergCatalogHandlerIT.java | 4 - .../ranger/test/RangerIntegrationTest.java | 4 - .../test/RangerIntegrationTestBase.java | 14 ++ .../test/RangerPolicyCatalogHandlerIT.java | 4 - .../ranger/test/RangerPolicyConditionIT.java | 4 - .../auth/ranger/test/RangerTestProfiles.java | 51 ------- ...olaris.service.it.ext.PolarisServerManager | 20 +++ 10 files changed, 112 insertions(+), 121 deletions(-) delete mode 100644 extensions/auth/ranger/tests/src/intTest/java/org/apache/polaris/extension/auth/ranger/test/RangerTestProfiles.java create mode 100644 extensions/auth/ranger/tests/src/intTest/resources/META-INF/services/org.apache.polaris.service.it.ext.PolarisServerManager diff --git a/extensions/auth/ranger/tests/build.gradle.kts b/extensions/auth/ranger/tests/build.gradle.kts index 418d0c7fbaa..cea73fda802 100644 --- a/extensions/auth/ranger/tests/build.gradle.kts +++ b/extensions/auth/ranger/tests/build.gradle.kts @@ -17,62 +17,94 @@ * under the License. */ +import org.gradle.api.attributes.java.TargetJvmVersion +import org.gradle.api.plugins.jvm.JvmTestSuite + plugins { - alias(libs.plugins.quarkus) - id("org.kordamp.gradle.jandex") - id("polaris-runtime") + id("polaris-client") + id("polaris-server-test-runner") } -dependencies { - implementation(platform(libs.quarkus.bom)) - implementation("io.quarkus:quarkus-rest-jackson") +val intTestJvmVersion = 21 - implementation(project(":polaris-extensions-auth-ranger")) - implementation(project(":polaris-runtime-service")) +dependencies { polarisServer(project(path = ":polaris-server", configuration = "quarkusRunner")) } - testImplementation(project(":polaris-runtime-test-common")) +testing { + suites { + @Suppress("UnstableApiUsage") + register("intTest") { + dependencies { + implementation(platform(libs.quarkus.bom)) + implementation("io.rest-assured:rest-assured") + implementation(project(":polaris-tests")) + implementation(project(":polaris-runtime-test-common")) + implementation(project(":polaris-api-management-model")) + implementation(platform(libs.iceberg.bom)) + implementation("org.apache.iceberg:iceberg-api") + implementation("org.apache.iceberg:iceberg-core") - intTestImplementation("io.quarkus:quarkus-junit") - intTestImplementation("io.rest-assured:rest-assured") - intTestImplementation(project(":polaris-api-management-model")) - intTestImplementation(platform(libs.iceberg.bom)) - intTestImplementation("org.apache.iceberg:iceberg-api") - intTestImplementation("org.apache.iceberg:iceberg-core") + implementation(platform(libs.testcontainers.bom)) + implementation("org.testcontainers:testcontainers-junit-jupiter") + implementation(project(":polaris-container-spec-helper")) + } + targets { + val buildDir = project.layout.buildDirectory + val policyDir = project.layout.projectDirectory.dir("src/intTest/resources/authz_it_tests") + all { + testTask.configure { + environment( + "AWS_REGION", + providers.environmentVariable("AWS_REGION").getOrElse("us-west-2"), + ) + environment(mapOf("POLARIS_BOOTSTRAP_CREDENTIALS" to "POLARIS,test-admin,test-secret")) + val apiVersion = System.getenv("DOCKER_API_VERSION") ?: "1.44" + systemProperty("api.version", apiVersion) + jvmArgs("--add-exports", "java.base/sun.nio.ch=ALL-UNNAMED") + systemProperty("java.security.manager", "allow") + maxParallelForks = 1 - intTestImplementation(platform(libs.testcontainers.bom)) - intTestImplementation("org.testcontainers:testcontainers-junit-jupiter") - intTestImplementation(project(":polaris-container-spec-helper")) -} + val buildDirFile = buildDir.get().asFile + val logsDir = buildDirFile.resolve("logs") -sourceSets.named("intTest") { - resources.srcDir( - project(":polaris-extensions-auth-ranger").layout.projectDirectory.dir("src/test/resources") - ) -} + doFirst { + logsDir.deleteRecursively() + buildDirFile.resolve("quarkus.log").delete() + } -tasks.named("javadoc") { dependsOn("jandex") } - -tasks.withType { - if (System.getenv("AWS_REGION") == null) { - environment("AWS_REGION", "us-west-2") - } - environment("POLARIS_BOOTSTRAP_CREDENTIALS", "POLARIS,test-admin,test-secret") - val apiVersion = System.getenv("DOCKER_API_VERSION") ?: "1.44" - systemProperty("api.version", apiVersion) - jvmArgs("--add-exports", "java.base/sun.nio.ch=ALL-UNNAMED") - systemProperty("java.security.manager", "allow") - maxParallelForks = 1 - - val logsDir = project.layout.buildDirectory.get().asFile.resolve("logs") - - jvmArgumentProviders.add( - CommandLineArgumentProvider { - listOf("-Dquarkus.log.file.path=${logsDir.resolve("polaris.log").absolutePath}") + withPolarisServer(configurations.polarisServer) { + environment.put( + "AWS_REGION", + providers.environmentVariable("AWS_REGION").orElse("us-west-2"), + ) + environment.putAll( + mapOf("POLARIS_BOOTSTRAP_CREDENTIALS" to "POLARIS,test-admin,test-secret") + ) + systemProperties.putAll( + mapOf( + "quarkus.log.file.path" to logsDir.resolve("polaris.log").absolutePath, + "polaris.authorization.type" to "ranger", + "polaris.authorization.ranger.service-name" to "dev_polaris", + "polaris.authorization.ranger.authz.default.policy.source.impl" to + "org.apache.ranger.admin.client.LocalFolderPolicySource", + "polaris.authorization.ranger.authz.default.enable.implicit.userstore.enricher" to + "true", + "polaris.authorization.ranger.authz.default.policy.source.local_folder.path" to + policyDir.asFile.absolutePath, + "polaris.features.\"SUPPORTED_CATALOG_STORAGE_TYPES\"" to "[\"FILE\"]", + "polaris.features.\"ALLOW_INSECURE_STORAGE_TYPES\"" to "true", + "polaris.readiness.ignore-severe-issues" to "true", + ) + ) + } + } + } + } } - ) + } +} - doFirst { - logsDir.deleteRecursively() - project.layout.buildDirectory.get().asFile.resolve("quarkus.log").delete() +listOf("intTestCompileClasspath", "intTestRuntimeClasspath").forEach { + configurations.named(it).configure { + attributes.attribute(TargetJvmVersion.TARGET_JVM_VERSION_ATTRIBUTE, intTestJvmVersion) } } diff --git a/extensions/auth/ranger/tests/src/intTest/java/org/apache/polaris/extension/auth/ranger/test/RangerAdminServiceIT.java b/extensions/auth/ranger/tests/src/intTest/java/org/apache/polaris/extension/auth/ranger/test/RangerAdminServiceIT.java index 1bfcd1c3062..7b62a27e863 100644 --- a/extensions/auth/ranger/tests/src/intTest/java/org/apache/polaris/extension/auth/ranger/test/RangerAdminServiceIT.java +++ b/extensions/auth/ranger/tests/src/intTest/java/org/apache/polaris/extension/auth/ranger/test/RangerAdminServiceIT.java @@ -20,8 +20,6 @@ import static io.restassured.RestAssured.given; -import io.quarkus.test.junit.QuarkusTest; -import io.quarkus.test.junit.TestProfile; import io.restassured.http.ContentType; import java.net.URI; import java.nio.file.Files; @@ -38,8 +36,6 @@ import org.junit.jupiter.api.Test; import org.junit.jupiter.api.io.TempDir; -@QuarkusTest -@TestProfile(RangerTestProfiles.EmbeddedPolicy.class) public class RangerAdminServiceIT extends RangerIntegrationTestBase { private @TempDir Path baseCatalogLocationPath; diff --git a/extensions/auth/ranger/tests/src/intTest/java/org/apache/polaris/extension/auth/ranger/test/RangerGenericTableHandlerIT.java b/extensions/auth/ranger/tests/src/intTest/java/org/apache/polaris/extension/auth/ranger/test/RangerGenericTableHandlerIT.java index 54a83bdf302..f5bace0a9c7 100644 --- a/extensions/auth/ranger/tests/src/intTest/java/org/apache/polaris/extension/auth/ranger/test/RangerGenericTableHandlerIT.java +++ b/extensions/auth/ranger/tests/src/intTest/java/org/apache/polaris/extension/auth/ranger/test/RangerGenericTableHandlerIT.java @@ -20,8 +20,6 @@ import static io.restassured.RestAssured.given; -import io.quarkus.test.junit.QuarkusTest; -import io.quarkus.test.junit.TestProfile; import io.restassured.http.ContentType; import java.nio.file.Files; import java.nio.file.Path; @@ -32,8 +30,6 @@ import org.junit.jupiter.api.Test; import org.junit.jupiter.api.io.TempDir; -@QuarkusTest -@TestProfile(RangerTestProfiles.EmbeddedPolicy.class) public class RangerGenericTableHandlerIT extends RangerIntegrationTestBase { private String catalogName; diff --git a/extensions/auth/ranger/tests/src/intTest/java/org/apache/polaris/extension/auth/ranger/test/RangerIcebergCatalogHandlerIT.java b/extensions/auth/ranger/tests/src/intTest/java/org/apache/polaris/extension/auth/ranger/test/RangerIcebergCatalogHandlerIT.java index 18399ff57e7..d2c8d5a6b1c 100644 --- a/extensions/auth/ranger/tests/src/intTest/java/org/apache/polaris/extension/auth/ranger/test/RangerIcebergCatalogHandlerIT.java +++ b/extensions/auth/ranger/tests/src/intTest/java/org/apache/polaris/extension/auth/ranger/test/RangerIcebergCatalogHandlerIT.java @@ -20,8 +20,6 @@ import static io.restassured.RestAssured.given; -import io.quarkus.test.junit.QuarkusTest; -import io.quarkus.test.junit.TestProfile; import io.restassured.http.ContentType; import java.net.URI; import java.nio.file.Files; @@ -38,8 +36,6 @@ import org.junit.jupiter.api.Test; import org.junit.jupiter.api.io.TempDir; -@QuarkusTest -@TestProfile(RangerTestProfiles.EmbeddedPolicy.class) public class RangerIcebergCatalogHandlerIT extends RangerIntegrationTestBase { private String catalogName; diff --git a/extensions/auth/ranger/tests/src/intTest/java/org/apache/polaris/extension/auth/ranger/test/RangerIntegrationTest.java b/extensions/auth/ranger/tests/src/intTest/java/org/apache/polaris/extension/auth/ranger/test/RangerIntegrationTest.java index a808e238cc5..e9e10b11bf7 100644 --- a/extensions/auth/ranger/tests/src/intTest/java/org/apache/polaris/extension/auth/ranger/test/RangerIntegrationTest.java +++ b/extensions/auth/ranger/tests/src/intTest/java/org/apache/polaris/extension/auth/ranger/test/RangerIntegrationTest.java @@ -20,8 +20,6 @@ import static io.restassured.RestAssured.given; -import io.quarkus.test.junit.QuarkusTest; -import io.quarkus.test.junit.TestProfile; import io.restassured.http.ContentType; import java.nio.file.Path; import java.util.List; @@ -30,8 +28,6 @@ import org.junit.jupiter.api.Test; import org.junit.jupiter.api.io.TempDir; -@QuarkusTest -@TestProfile(RangerTestProfiles.EmbeddedPolicy.class) public class RangerIntegrationTest extends RangerIntegrationTestBase { @Test diff --git a/extensions/auth/ranger/tests/src/intTest/java/org/apache/polaris/extension/auth/ranger/test/RangerIntegrationTestBase.java b/extensions/auth/ranger/tests/src/intTest/java/org/apache/polaris/extension/auth/ranger/test/RangerIntegrationTestBase.java index 89403eea084..b4f150b59ad 100644 --- a/extensions/auth/ranger/tests/src/intTest/java/org/apache/polaris/extension/auth/ranger/test/RangerIntegrationTestBase.java +++ b/extensions/auth/ranger/tests/src/intTest/java/org/apache/polaris/extension/auth/ranger/test/RangerIntegrationTestBase.java @@ -23,6 +23,7 @@ import com.fasterxml.jackson.databind.JsonNode; import com.fasterxml.jackson.databind.json.JsonMapper; +import io.restassured.RestAssured; import io.restassured.http.ContentType; import java.io.UncheckedIOException; import java.util.ArrayList; @@ -30,7 +31,9 @@ import java.util.HashMap; import java.util.List; import java.util.Map; +import org.junit.jupiter.api.AfterAll; import org.junit.jupiter.api.AfterEach; +import org.junit.jupiter.api.BeforeAll; /** * Base class for Ranger integration tests providing common helpers for authentication and principal @@ -43,6 +46,17 @@ public abstract class RangerIntegrationTestBase { protected Map user2Token = new HashMap<>(); + @BeforeAll + static void configureRestAssured() { + RestAssured.baseURI = "http://localhost"; + RestAssured.port = Integer.getInteger("quarkus.http.test-port"); + } + + @AfterAll + static void resetRestAssured() { + RestAssured.reset(); + } + protected String toJson(Object value) { try { return mapper.writeValueAsString(value); diff --git a/extensions/auth/ranger/tests/src/intTest/java/org/apache/polaris/extension/auth/ranger/test/RangerPolicyCatalogHandlerIT.java b/extensions/auth/ranger/tests/src/intTest/java/org/apache/polaris/extension/auth/ranger/test/RangerPolicyCatalogHandlerIT.java index 8994693f604..5b9cb74206b 100644 --- a/extensions/auth/ranger/tests/src/intTest/java/org/apache/polaris/extension/auth/ranger/test/RangerPolicyCatalogHandlerIT.java +++ b/extensions/auth/ranger/tests/src/intTest/java/org/apache/polaris/extension/auth/ranger/test/RangerPolicyCatalogHandlerIT.java @@ -20,8 +20,6 @@ import static io.restassured.RestAssured.given; -import io.quarkus.test.junit.QuarkusTest; -import io.quarkus.test.junit.TestProfile; import io.restassured.http.ContentType; import java.nio.file.Files; import java.nio.file.Path; @@ -32,8 +30,6 @@ import org.junit.jupiter.api.Test; import org.junit.jupiter.api.io.TempDir; -@QuarkusTest -@TestProfile(RangerTestProfiles.EmbeddedPolicy.class) public class RangerPolicyCatalogHandlerIT extends RangerIntegrationTestBase { private String catalogName; diff --git a/extensions/auth/ranger/tests/src/intTest/java/org/apache/polaris/extension/auth/ranger/test/RangerPolicyConditionIT.java b/extensions/auth/ranger/tests/src/intTest/java/org/apache/polaris/extension/auth/ranger/test/RangerPolicyConditionIT.java index 2cdb05af415..a0e2167c6e9 100644 --- a/extensions/auth/ranger/tests/src/intTest/java/org/apache/polaris/extension/auth/ranger/test/RangerPolicyConditionIT.java +++ b/extensions/auth/ranger/tests/src/intTest/java/org/apache/polaris/extension/auth/ranger/test/RangerPolicyConditionIT.java @@ -20,13 +20,9 @@ import static io.restassured.RestAssured.given; -import io.quarkus.test.junit.QuarkusTest; -import io.quarkus.test.junit.TestProfile; import io.restassured.http.ContentType; import org.junit.jupiter.api.Test; -@QuarkusTest -@TestProfile(RangerTestProfiles.EmbeddedPolicy.class) public class RangerPolicyConditionIT extends RangerIntegrationTestBase { @Test diff --git a/extensions/auth/ranger/tests/src/intTest/java/org/apache/polaris/extension/auth/ranger/test/RangerTestProfiles.java b/extensions/auth/ranger/tests/src/intTest/java/org/apache/polaris/extension/auth/ranger/test/RangerTestProfiles.java deleted file mode 100644 index d91b6d4f174..00000000000 --- a/extensions/auth/ranger/tests/src/intTest/java/org/apache/polaris/extension/auth/ranger/test/RangerTestProfiles.java +++ /dev/null @@ -1,51 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ -package org.apache.polaris.extension.auth.ranger.test; - -import io.quarkus.test.junit.QuarkusTestProfile; -import java.util.HashMap; -import java.util.Map; - -/** Quarkus test profiles for Ranger integration tests using embedded policy fixtures. */ -public final class RangerTestProfiles { - - private RangerTestProfiles() {} - - /** Ranger authorizer with policies loaded from classpath ({@code /authz_tests}). */ - public static class EmbeddedPolicy implements QuarkusTestProfile { - @Override - public Map getConfigOverrides() { - Map config = new HashMap<>(); - config.put("polaris.authorization.type", "ranger"); - config.put("polaris.authorization.ranger.service-name", "dev_polaris"); - config.put( - "polaris.authorization.ranger.authz.default.policy.source.impl", - "org.apache.ranger.admin.client.EmbeddedResourcePolicySource"); - config.put( - "polaris.authorization.ranger.authz.default.enable.implicit.userstore.enricher", "true"); - config.put( - "polaris.authorization.ranger.authz.default.policy.source.embedded_resource.path", - "/authz_it_tests"); - config.put("polaris.features.\"SUPPORTED_CATALOG_STORAGE_TYPES\"", "[\"FILE\"]"); - config.put("polaris.features.\"ALLOW_INSECURE_STORAGE_TYPES\"", "true"); - config.put("polaris.readiness.ignore-severe-issues", "true"); - return config; - } - } -} diff --git a/extensions/auth/ranger/tests/src/intTest/resources/META-INF/services/org.apache.polaris.service.it.ext.PolarisServerManager b/extensions/auth/ranger/tests/src/intTest/resources/META-INF/services/org.apache.polaris.service.it.ext.PolarisServerManager new file mode 100644 index 00000000000..6aa2dac9e0c --- /dev/null +++ b/extensions/auth/ranger/tests/src/intTest/resources/META-INF/services/org.apache.polaris.service.it.ext.PolarisServerManager @@ -0,0 +1,20 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# + +org.apache.polaris.service.it.ext.ExternalPolarisServerManager From c06592ffb93c505cdfae1da273f87cb4f52e2484 Mon Sep 17 00:00:00 2001 From: Robert Stupp Date: Tue, 9 Jun 2026 12:06:48 +0200 Subject: [PATCH 07/22] Move Ranger integration tests into Ranger extension --- bom/build.gradle.kts | 1 - extensions/auth/ranger/impl/build.gradle.kts | 89 ++++++++++++++ .../ranger/test/RangerAdminServiceIT.java | 0 .../test/RangerGenericTableHandlerIT.java | 0 .../test/RangerIcebergCatalogHandlerIT.java | 0 .../ranger/test/RangerIntegrationTest.java | 0 .../test/RangerIntegrationTestBase.java | 0 .../test/RangerPolicyCatalogHandlerIT.java | 0 .../ranger/test/RangerPolicyConditionIT.java | 0 ...olaris.service.it.ext.PolarisServerManager | 0 .../resources/authz_it_tests/dev_polaris.json | 0 .../authz_it_tests/dev_polaris_roles.json | 0 .../authz_it_tests/dev_polaris_userstore.json | 0 extensions/auth/ranger/tests/build.gradle.kts | 110 ------------------ gradle/projects.main.properties | 1 - 15 files changed, 89 insertions(+), 112 deletions(-) rename extensions/auth/ranger/{tests => impl}/src/intTest/java/org/apache/polaris/extension/auth/ranger/test/RangerAdminServiceIT.java (100%) rename extensions/auth/ranger/{tests => impl}/src/intTest/java/org/apache/polaris/extension/auth/ranger/test/RangerGenericTableHandlerIT.java (100%) rename extensions/auth/ranger/{tests => impl}/src/intTest/java/org/apache/polaris/extension/auth/ranger/test/RangerIcebergCatalogHandlerIT.java (100%) rename extensions/auth/ranger/{tests => impl}/src/intTest/java/org/apache/polaris/extension/auth/ranger/test/RangerIntegrationTest.java (100%) rename extensions/auth/ranger/{tests => impl}/src/intTest/java/org/apache/polaris/extension/auth/ranger/test/RangerIntegrationTestBase.java (100%) rename extensions/auth/ranger/{tests => impl}/src/intTest/java/org/apache/polaris/extension/auth/ranger/test/RangerPolicyCatalogHandlerIT.java (100%) rename extensions/auth/ranger/{tests => impl}/src/intTest/java/org/apache/polaris/extension/auth/ranger/test/RangerPolicyConditionIT.java (100%) rename extensions/auth/ranger/{tests => impl}/src/intTest/resources/META-INF/services/org.apache.polaris.service.it.ext.PolarisServerManager (100%) rename extensions/auth/ranger/{tests => impl}/src/intTest/resources/authz_it_tests/dev_polaris.json (100%) rename extensions/auth/ranger/{tests => impl}/src/intTest/resources/authz_it_tests/dev_polaris_roles.json (100%) rename extensions/auth/ranger/{tests => impl}/src/intTest/resources/authz_it_tests/dev_polaris_userstore.json (100%) delete mode 100644 extensions/auth/ranger/tests/build.gradle.kts diff --git a/bom/build.gradle.kts b/bom/build.gradle.kts index b84a9ce0742..44034481a45 100644 --- a/bom/build.gradle.kts +++ b/bom/build.gradle.kts @@ -97,7 +97,6 @@ dependencies { api(project(":polaris-extensions-auth-opa")) api(project(":polaris-extensions-auth-opa-tests")) api(project(":polaris-extensions-auth-ranger")) - api(project(":polaris-extensions-auth-ranger-tests")) api(project(":polaris-extensions-federation-bigquery")) api(project(":polaris-extensions-federation-hadoop")) api(project(":polaris-extensions-federation-hive")) diff --git a/extensions/auth/ranger/impl/build.gradle.kts b/extensions/auth/ranger/impl/build.gradle.kts index c4437ce2335..46343917b64 100644 --- a/extensions/auth/ranger/impl/build.gradle.kts +++ b/extensions/auth/ranger/impl/build.gradle.kts @@ -17,12 +17,20 @@ * under the License. */ +import org.gradle.api.attributes.java.TargetJvmVersion +import org.gradle.api.plugins.jvm.JvmTestSuite + plugins { id("polaris-server") id("org.kordamp.gradle.jandex") + id("polaris-server-test-runner") } +val intTestJvmVersion = 21 + dependencies { + polarisServer(project(path = ":polaris-server", configuration = "quarkusRunner")) + implementation(project(":polaris-core")) implementation(libs.ranger.authz.embedded) { @@ -49,3 +57,84 @@ dependencies { runtimeOnly(libs.graalvm.polyglot.js) runtimeOnly(libs.graalvm.polyglot.polyglot) } + +testing { + suites { + @Suppress("UnstableApiUsage") + register("intTest") { + dependencies { + implementation(platform(libs.quarkus.bom)) + implementation("io.rest-assured:rest-assured") + implementation(project(":polaris-tests")) + implementation(project(":polaris-runtime-test-common")) + implementation(project(":polaris-api-management-model")) + implementation(platform(libs.iceberg.bom)) + implementation("org.apache.iceberg:iceberg-api") + implementation("org.apache.iceberg:iceberg-core") + + implementation(platform(libs.testcontainers.bom)) + implementation("org.testcontainers:testcontainers-junit-jupiter") + implementation(project(":polaris-container-spec-helper")) + } + targets { + all { + val buildDir = project.layout.buildDirectory + val policyDir = + project.layout.projectDirectory.dir("src/intTest/resources/authz_it_tests") + testTask.configure { + environment( + "AWS_REGION", + providers.environmentVariable("AWS_REGION").getOrElse("us-west-2"), + ) + environment(mapOf("POLARIS_BOOTSTRAP_CREDENTIALS" to "POLARIS,test-admin,test-secret")) + val apiVersion = providers.environmentVariable("DOCKER_API_VERSION").getOrElse("1.44") + systemProperty("api.version", apiVersion) + jvmArgs("--add-exports", "java.base/sun.nio.ch=ALL-UNNAMED") + systemProperty("java.security.manager", "allow") + maxParallelForks = 1 + + val buildDirFile = buildDir.get().asFile + val logsDir = buildDirFile.resolve("logs") + + doFirst { + logsDir.deleteRecursively() + buildDirFile.resolve("quarkus.log").delete() + } + + withPolarisServer(configurations.polarisServer) { + environment.put( + "AWS_REGION", + providers.environmentVariable("AWS_REGION").orElse("us-west-2"), + ) + environment.putAll( + mapOf("POLARIS_BOOTSTRAP_CREDENTIALS" to "POLARIS,test-admin,test-secret") + ) + systemProperties.putAll( + mapOf( + "quarkus.log.file.path" to logsDir.resolve("polaris.log").absolutePath, + "polaris.authorization.type" to "ranger", + "polaris.authorization.ranger.service-name" to "dev_polaris", + "polaris.authorization.ranger.authz.default.policy.source.impl" to + "org.apache.ranger.admin.client.LocalFolderPolicySource", + "polaris.authorization.ranger.authz.default.enable.implicit.userstore.enricher" to + "true", + "polaris.authorization.ranger.authz.default.policy.source.local_folder.path" to + policyDir.asFile.absolutePath, + "polaris.features.\"SUPPORTED_CATALOG_STORAGE_TYPES\"" to "[\"FILE\"]", + "polaris.features.\"ALLOW_INSECURE_STORAGE_TYPES\"" to "true", + "polaris.readiness.ignore-severe-issues" to "true", + ) + ) + } + } + } + } + } + } +} + +listOf("intTestCompileClasspath", "intTestRuntimeClasspath").forEach { + configurations.named(it).configure { + attributes.attribute(TargetJvmVersion.TARGET_JVM_VERSION_ATTRIBUTE, intTestJvmVersion) + } +} diff --git a/extensions/auth/ranger/tests/src/intTest/java/org/apache/polaris/extension/auth/ranger/test/RangerAdminServiceIT.java b/extensions/auth/ranger/impl/src/intTest/java/org/apache/polaris/extension/auth/ranger/test/RangerAdminServiceIT.java similarity index 100% rename from extensions/auth/ranger/tests/src/intTest/java/org/apache/polaris/extension/auth/ranger/test/RangerAdminServiceIT.java rename to extensions/auth/ranger/impl/src/intTest/java/org/apache/polaris/extension/auth/ranger/test/RangerAdminServiceIT.java diff --git a/extensions/auth/ranger/tests/src/intTest/java/org/apache/polaris/extension/auth/ranger/test/RangerGenericTableHandlerIT.java b/extensions/auth/ranger/impl/src/intTest/java/org/apache/polaris/extension/auth/ranger/test/RangerGenericTableHandlerIT.java similarity index 100% rename from extensions/auth/ranger/tests/src/intTest/java/org/apache/polaris/extension/auth/ranger/test/RangerGenericTableHandlerIT.java rename to extensions/auth/ranger/impl/src/intTest/java/org/apache/polaris/extension/auth/ranger/test/RangerGenericTableHandlerIT.java diff --git a/extensions/auth/ranger/tests/src/intTest/java/org/apache/polaris/extension/auth/ranger/test/RangerIcebergCatalogHandlerIT.java b/extensions/auth/ranger/impl/src/intTest/java/org/apache/polaris/extension/auth/ranger/test/RangerIcebergCatalogHandlerIT.java similarity index 100% rename from extensions/auth/ranger/tests/src/intTest/java/org/apache/polaris/extension/auth/ranger/test/RangerIcebergCatalogHandlerIT.java rename to extensions/auth/ranger/impl/src/intTest/java/org/apache/polaris/extension/auth/ranger/test/RangerIcebergCatalogHandlerIT.java diff --git a/extensions/auth/ranger/tests/src/intTest/java/org/apache/polaris/extension/auth/ranger/test/RangerIntegrationTest.java b/extensions/auth/ranger/impl/src/intTest/java/org/apache/polaris/extension/auth/ranger/test/RangerIntegrationTest.java similarity index 100% rename from extensions/auth/ranger/tests/src/intTest/java/org/apache/polaris/extension/auth/ranger/test/RangerIntegrationTest.java rename to extensions/auth/ranger/impl/src/intTest/java/org/apache/polaris/extension/auth/ranger/test/RangerIntegrationTest.java diff --git a/extensions/auth/ranger/tests/src/intTest/java/org/apache/polaris/extension/auth/ranger/test/RangerIntegrationTestBase.java b/extensions/auth/ranger/impl/src/intTest/java/org/apache/polaris/extension/auth/ranger/test/RangerIntegrationTestBase.java similarity index 100% rename from extensions/auth/ranger/tests/src/intTest/java/org/apache/polaris/extension/auth/ranger/test/RangerIntegrationTestBase.java rename to extensions/auth/ranger/impl/src/intTest/java/org/apache/polaris/extension/auth/ranger/test/RangerIntegrationTestBase.java diff --git a/extensions/auth/ranger/tests/src/intTest/java/org/apache/polaris/extension/auth/ranger/test/RangerPolicyCatalogHandlerIT.java b/extensions/auth/ranger/impl/src/intTest/java/org/apache/polaris/extension/auth/ranger/test/RangerPolicyCatalogHandlerIT.java similarity index 100% rename from extensions/auth/ranger/tests/src/intTest/java/org/apache/polaris/extension/auth/ranger/test/RangerPolicyCatalogHandlerIT.java rename to extensions/auth/ranger/impl/src/intTest/java/org/apache/polaris/extension/auth/ranger/test/RangerPolicyCatalogHandlerIT.java diff --git a/extensions/auth/ranger/tests/src/intTest/java/org/apache/polaris/extension/auth/ranger/test/RangerPolicyConditionIT.java b/extensions/auth/ranger/impl/src/intTest/java/org/apache/polaris/extension/auth/ranger/test/RangerPolicyConditionIT.java similarity index 100% rename from extensions/auth/ranger/tests/src/intTest/java/org/apache/polaris/extension/auth/ranger/test/RangerPolicyConditionIT.java rename to extensions/auth/ranger/impl/src/intTest/java/org/apache/polaris/extension/auth/ranger/test/RangerPolicyConditionIT.java diff --git a/extensions/auth/ranger/tests/src/intTest/resources/META-INF/services/org.apache.polaris.service.it.ext.PolarisServerManager b/extensions/auth/ranger/impl/src/intTest/resources/META-INF/services/org.apache.polaris.service.it.ext.PolarisServerManager similarity index 100% rename from extensions/auth/ranger/tests/src/intTest/resources/META-INF/services/org.apache.polaris.service.it.ext.PolarisServerManager rename to extensions/auth/ranger/impl/src/intTest/resources/META-INF/services/org.apache.polaris.service.it.ext.PolarisServerManager diff --git a/extensions/auth/ranger/tests/src/intTest/resources/authz_it_tests/dev_polaris.json b/extensions/auth/ranger/impl/src/intTest/resources/authz_it_tests/dev_polaris.json similarity index 100% rename from extensions/auth/ranger/tests/src/intTest/resources/authz_it_tests/dev_polaris.json rename to extensions/auth/ranger/impl/src/intTest/resources/authz_it_tests/dev_polaris.json diff --git a/extensions/auth/ranger/tests/src/intTest/resources/authz_it_tests/dev_polaris_roles.json b/extensions/auth/ranger/impl/src/intTest/resources/authz_it_tests/dev_polaris_roles.json similarity index 100% rename from extensions/auth/ranger/tests/src/intTest/resources/authz_it_tests/dev_polaris_roles.json rename to extensions/auth/ranger/impl/src/intTest/resources/authz_it_tests/dev_polaris_roles.json diff --git a/extensions/auth/ranger/tests/src/intTest/resources/authz_it_tests/dev_polaris_userstore.json b/extensions/auth/ranger/impl/src/intTest/resources/authz_it_tests/dev_polaris_userstore.json similarity index 100% rename from extensions/auth/ranger/tests/src/intTest/resources/authz_it_tests/dev_polaris_userstore.json rename to extensions/auth/ranger/impl/src/intTest/resources/authz_it_tests/dev_polaris_userstore.json diff --git a/extensions/auth/ranger/tests/build.gradle.kts b/extensions/auth/ranger/tests/build.gradle.kts deleted file mode 100644 index cea73fda802..00000000000 --- a/extensions/auth/ranger/tests/build.gradle.kts +++ /dev/null @@ -1,110 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -import org.gradle.api.attributes.java.TargetJvmVersion -import org.gradle.api.plugins.jvm.JvmTestSuite - -plugins { - id("polaris-client") - id("polaris-server-test-runner") -} - -val intTestJvmVersion = 21 - -dependencies { polarisServer(project(path = ":polaris-server", configuration = "quarkusRunner")) } - -testing { - suites { - @Suppress("UnstableApiUsage") - register("intTest") { - dependencies { - implementation(platform(libs.quarkus.bom)) - implementation("io.rest-assured:rest-assured") - implementation(project(":polaris-tests")) - implementation(project(":polaris-runtime-test-common")) - implementation(project(":polaris-api-management-model")) - implementation(platform(libs.iceberg.bom)) - implementation("org.apache.iceberg:iceberg-api") - implementation("org.apache.iceberg:iceberg-core") - - implementation(platform(libs.testcontainers.bom)) - implementation("org.testcontainers:testcontainers-junit-jupiter") - implementation(project(":polaris-container-spec-helper")) - } - targets { - val buildDir = project.layout.buildDirectory - val policyDir = project.layout.projectDirectory.dir("src/intTest/resources/authz_it_tests") - all { - testTask.configure { - environment( - "AWS_REGION", - providers.environmentVariable("AWS_REGION").getOrElse("us-west-2"), - ) - environment(mapOf("POLARIS_BOOTSTRAP_CREDENTIALS" to "POLARIS,test-admin,test-secret")) - val apiVersion = System.getenv("DOCKER_API_VERSION") ?: "1.44" - systemProperty("api.version", apiVersion) - jvmArgs("--add-exports", "java.base/sun.nio.ch=ALL-UNNAMED") - systemProperty("java.security.manager", "allow") - maxParallelForks = 1 - - val buildDirFile = buildDir.get().asFile - val logsDir = buildDirFile.resolve("logs") - - doFirst { - logsDir.deleteRecursively() - buildDirFile.resolve("quarkus.log").delete() - } - - withPolarisServer(configurations.polarisServer) { - environment.put( - "AWS_REGION", - providers.environmentVariable("AWS_REGION").orElse("us-west-2"), - ) - environment.putAll( - mapOf("POLARIS_BOOTSTRAP_CREDENTIALS" to "POLARIS,test-admin,test-secret") - ) - systemProperties.putAll( - mapOf( - "quarkus.log.file.path" to logsDir.resolve("polaris.log").absolutePath, - "polaris.authorization.type" to "ranger", - "polaris.authorization.ranger.service-name" to "dev_polaris", - "polaris.authorization.ranger.authz.default.policy.source.impl" to - "org.apache.ranger.admin.client.LocalFolderPolicySource", - "polaris.authorization.ranger.authz.default.enable.implicit.userstore.enricher" to - "true", - "polaris.authorization.ranger.authz.default.policy.source.local_folder.path" to - policyDir.asFile.absolutePath, - "polaris.features.\"SUPPORTED_CATALOG_STORAGE_TYPES\"" to "[\"FILE\"]", - "polaris.features.\"ALLOW_INSECURE_STORAGE_TYPES\"" to "true", - "polaris.readiness.ignore-severe-issues" to "true", - ) - ) - } - } - } - } - } - } -} - -listOf("intTestCompileClasspath", "intTestRuntimeClasspath").forEach { - configurations.named(it).configure { - attributes.attribute(TargetJvmVersion.TARGET_JVM_VERSION_ATTRIBUTE, intTestJvmVersion) - } -} diff --git a/gradle/projects.main.properties b/gradle/projects.main.properties index 322510c5578..21f94659d95 100644 --- a/gradle/projects.main.properties +++ b/gradle/projects.main.properties @@ -50,7 +50,6 @@ polaris-extensions-federation-bigquery=extensions/federation/bigquery polaris-extensions-auth-opa=extensions/auth/opa/impl polaris-extensions-auth-opa-tests=extensions/auth/opa/tests polaris-extensions-auth-ranger=extensions/auth/ranger/impl -polaris-extensions-auth-ranger-tests=extensions/auth/ranger/tests polaris-config-docs-annotations=tools/config-docs/annotations polaris-config-docs-generator=tools/config-docs/generator From e2c747e46d2e4e273879f144e9d98586b1644581 Mon Sep 17 00:00:00 2001 From: Robert Stupp Date: Tue, 9 Jun 2026 12:17:05 +0200 Subject: [PATCH 08/22] Move Ranger source up --- extensions/auth/ranger/{impl => }/README.md | 0 extensions/auth/ranger/{impl => }/build.gradle.kts | 0 .../extension/auth/ranger/test/RangerAdminServiceIT.java | 0 .../extension/auth/ranger/test/RangerGenericTableHandlerIT.java | 0 .../auth/ranger/test/RangerIcebergCatalogHandlerIT.java | 0 .../extension/auth/ranger/test/RangerIntegrationTest.java | 0 .../extension/auth/ranger/test/RangerIntegrationTestBase.java | 0 .../auth/ranger/test/RangerPolicyCatalogHandlerIT.java | 0 .../extension/auth/ranger/test/RangerPolicyConditionIT.java | 0 .../org.apache.polaris.service.it.ext.PolarisServerManager | 0 .../src/intTest/resources/authz_it_tests/dev_polaris.json | 0 .../src/intTest/resources/authz_it_tests/dev_polaris_roles.json | 0 .../intTest/resources/authz_it_tests/dev_polaris_userstore.json | 0 .../polaris/extension/auth/ranger/RangerPolarisAuthorizer.java | 0 .../extension/auth/ranger/RangerPolarisAuthorizerConfig.java | 0 .../extension/auth/ranger/RangerPolarisAuthorizerFactory.java | 0 .../extension/auth/ranger/RangerPolarisOperationSemantics.java | 0 .../apache/polaris/extension/auth/ranger/utils/RangerUtils.java | 0 .../auth/ranger/RangerPolarisAuthorizerFactoryTest.java | 0 .../extension/auth/ranger/RangerPolarisAuthorizerTest.java | 0 .../apache/polaris/extension/auth/ranger/RangerTestUtils.java | 0 .../{impl => }/src/test/resources/authz_tests/dev_polaris.json | 0 .../src/test/resources/authz_tests/tests_authz_catalog.json | 0 .../src/test/resources/authz_tests/tests_authz_namespace.json | 0 .../src/test/resources/authz_tests/tests_authz_policy.json | 0 .../src/test/resources/authz_tests/tests_authz_principal.json | 0 .../src/test/resources/authz_tests/tests_authz_root.json | 0 .../src/test/resources/authz_tests/tests_authz_table.json | 0 .../src/test/resources/authz_tests/tests_authz_unsupported.json | 0 .../auth/ranger/{impl => }/src/test/resources/logback-test.xml | 0 gradle/projects.main.properties | 2 +- 31 files changed, 1 insertion(+), 1 deletion(-) rename extensions/auth/ranger/{impl => }/README.md (100%) rename extensions/auth/ranger/{impl => }/build.gradle.kts (100%) rename extensions/auth/ranger/{impl => }/src/intTest/java/org/apache/polaris/extension/auth/ranger/test/RangerAdminServiceIT.java (100%) rename extensions/auth/ranger/{impl => }/src/intTest/java/org/apache/polaris/extension/auth/ranger/test/RangerGenericTableHandlerIT.java (100%) rename extensions/auth/ranger/{impl => }/src/intTest/java/org/apache/polaris/extension/auth/ranger/test/RangerIcebergCatalogHandlerIT.java (100%) rename extensions/auth/ranger/{impl => }/src/intTest/java/org/apache/polaris/extension/auth/ranger/test/RangerIntegrationTest.java (100%) rename extensions/auth/ranger/{impl => }/src/intTest/java/org/apache/polaris/extension/auth/ranger/test/RangerIntegrationTestBase.java (100%) rename extensions/auth/ranger/{impl => }/src/intTest/java/org/apache/polaris/extension/auth/ranger/test/RangerPolicyCatalogHandlerIT.java (100%) rename extensions/auth/ranger/{impl => }/src/intTest/java/org/apache/polaris/extension/auth/ranger/test/RangerPolicyConditionIT.java (100%) rename extensions/auth/ranger/{impl => }/src/intTest/resources/META-INF/services/org.apache.polaris.service.it.ext.PolarisServerManager (100%) rename extensions/auth/ranger/{impl => }/src/intTest/resources/authz_it_tests/dev_polaris.json (100%) rename extensions/auth/ranger/{impl => }/src/intTest/resources/authz_it_tests/dev_polaris_roles.json (100%) rename extensions/auth/ranger/{impl => }/src/intTest/resources/authz_it_tests/dev_polaris_userstore.json (100%) rename extensions/auth/ranger/{impl => }/src/main/java/org/apache/polaris/extension/auth/ranger/RangerPolarisAuthorizer.java (100%) rename extensions/auth/ranger/{impl => }/src/main/java/org/apache/polaris/extension/auth/ranger/RangerPolarisAuthorizerConfig.java (100%) rename extensions/auth/ranger/{impl => }/src/main/java/org/apache/polaris/extension/auth/ranger/RangerPolarisAuthorizerFactory.java (100%) rename extensions/auth/ranger/{impl => }/src/main/java/org/apache/polaris/extension/auth/ranger/RangerPolarisOperationSemantics.java (100%) rename extensions/auth/ranger/{impl => }/src/main/java/org/apache/polaris/extension/auth/ranger/utils/RangerUtils.java (100%) rename extensions/auth/ranger/{impl => }/src/test/java/org/apache/polaris/extension/auth/ranger/RangerPolarisAuthorizerFactoryTest.java (100%) rename extensions/auth/ranger/{impl => }/src/test/java/org/apache/polaris/extension/auth/ranger/RangerPolarisAuthorizerTest.java (100%) rename extensions/auth/ranger/{impl => }/src/test/java/org/apache/polaris/extension/auth/ranger/RangerTestUtils.java (100%) rename extensions/auth/ranger/{impl => }/src/test/resources/authz_tests/dev_polaris.json (100%) rename extensions/auth/ranger/{impl => }/src/test/resources/authz_tests/tests_authz_catalog.json (100%) rename extensions/auth/ranger/{impl => }/src/test/resources/authz_tests/tests_authz_namespace.json (100%) rename extensions/auth/ranger/{impl => }/src/test/resources/authz_tests/tests_authz_policy.json (100%) rename extensions/auth/ranger/{impl => }/src/test/resources/authz_tests/tests_authz_principal.json (100%) rename extensions/auth/ranger/{impl => }/src/test/resources/authz_tests/tests_authz_root.json (100%) rename extensions/auth/ranger/{impl => }/src/test/resources/authz_tests/tests_authz_table.json (100%) rename extensions/auth/ranger/{impl => }/src/test/resources/authz_tests/tests_authz_unsupported.json (100%) rename extensions/auth/ranger/{impl => }/src/test/resources/logback-test.xml (100%) diff --git a/extensions/auth/ranger/impl/README.md b/extensions/auth/ranger/README.md similarity index 100% rename from extensions/auth/ranger/impl/README.md rename to extensions/auth/ranger/README.md diff --git a/extensions/auth/ranger/impl/build.gradle.kts b/extensions/auth/ranger/build.gradle.kts similarity index 100% rename from extensions/auth/ranger/impl/build.gradle.kts rename to extensions/auth/ranger/build.gradle.kts diff --git a/extensions/auth/ranger/impl/src/intTest/java/org/apache/polaris/extension/auth/ranger/test/RangerAdminServiceIT.java b/extensions/auth/ranger/src/intTest/java/org/apache/polaris/extension/auth/ranger/test/RangerAdminServiceIT.java similarity index 100% rename from extensions/auth/ranger/impl/src/intTest/java/org/apache/polaris/extension/auth/ranger/test/RangerAdminServiceIT.java rename to extensions/auth/ranger/src/intTest/java/org/apache/polaris/extension/auth/ranger/test/RangerAdminServiceIT.java diff --git a/extensions/auth/ranger/impl/src/intTest/java/org/apache/polaris/extension/auth/ranger/test/RangerGenericTableHandlerIT.java b/extensions/auth/ranger/src/intTest/java/org/apache/polaris/extension/auth/ranger/test/RangerGenericTableHandlerIT.java similarity index 100% rename from extensions/auth/ranger/impl/src/intTest/java/org/apache/polaris/extension/auth/ranger/test/RangerGenericTableHandlerIT.java rename to extensions/auth/ranger/src/intTest/java/org/apache/polaris/extension/auth/ranger/test/RangerGenericTableHandlerIT.java diff --git a/extensions/auth/ranger/impl/src/intTest/java/org/apache/polaris/extension/auth/ranger/test/RangerIcebergCatalogHandlerIT.java b/extensions/auth/ranger/src/intTest/java/org/apache/polaris/extension/auth/ranger/test/RangerIcebergCatalogHandlerIT.java similarity index 100% rename from extensions/auth/ranger/impl/src/intTest/java/org/apache/polaris/extension/auth/ranger/test/RangerIcebergCatalogHandlerIT.java rename to extensions/auth/ranger/src/intTest/java/org/apache/polaris/extension/auth/ranger/test/RangerIcebergCatalogHandlerIT.java diff --git a/extensions/auth/ranger/impl/src/intTest/java/org/apache/polaris/extension/auth/ranger/test/RangerIntegrationTest.java b/extensions/auth/ranger/src/intTest/java/org/apache/polaris/extension/auth/ranger/test/RangerIntegrationTest.java similarity index 100% rename from extensions/auth/ranger/impl/src/intTest/java/org/apache/polaris/extension/auth/ranger/test/RangerIntegrationTest.java rename to extensions/auth/ranger/src/intTest/java/org/apache/polaris/extension/auth/ranger/test/RangerIntegrationTest.java diff --git a/extensions/auth/ranger/impl/src/intTest/java/org/apache/polaris/extension/auth/ranger/test/RangerIntegrationTestBase.java b/extensions/auth/ranger/src/intTest/java/org/apache/polaris/extension/auth/ranger/test/RangerIntegrationTestBase.java similarity index 100% rename from extensions/auth/ranger/impl/src/intTest/java/org/apache/polaris/extension/auth/ranger/test/RangerIntegrationTestBase.java rename to extensions/auth/ranger/src/intTest/java/org/apache/polaris/extension/auth/ranger/test/RangerIntegrationTestBase.java diff --git a/extensions/auth/ranger/impl/src/intTest/java/org/apache/polaris/extension/auth/ranger/test/RangerPolicyCatalogHandlerIT.java b/extensions/auth/ranger/src/intTest/java/org/apache/polaris/extension/auth/ranger/test/RangerPolicyCatalogHandlerIT.java similarity index 100% rename from extensions/auth/ranger/impl/src/intTest/java/org/apache/polaris/extension/auth/ranger/test/RangerPolicyCatalogHandlerIT.java rename to extensions/auth/ranger/src/intTest/java/org/apache/polaris/extension/auth/ranger/test/RangerPolicyCatalogHandlerIT.java diff --git a/extensions/auth/ranger/impl/src/intTest/java/org/apache/polaris/extension/auth/ranger/test/RangerPolicyConditionIT.java b/extensions/auth/ranger/src/intTest/java/org/apache/polaris/extension/auth/ranger/test/RangerPolicyConditionIT.java similarity index 100% rename from extensions/auth/ranger/impl/src/intTest/java/org/apache/polaris/extension/auth/ranger/test/RangerPolicyConditionIT.java rename to extensions/auth/ranger/src/intTest/java/org/apache/polaris/extension/auth/ranger/test/RangerPolicyConditionIT.java diff --git a/extensions/auth/ranger/impl/src/intTest/resources/META-INF/services/org.apache.polaris.service.it.ext.PolarisServerManager b/extensions/auth/ranger/src/intTest/resources/META-INF/services/org.apache.polaris.service.it.ext.PolarisServerManager similarity index 100% rename from extensions/auth/ranger/impl/src/intTest/resources/META-INF/services/org.apache.polaris.service.it.ext.PolarisServerManager rename to extensions/auth/ranger/src/intTest/resources/META-INF/services/org.apache.polaris.service.it.ext.PolarisServerManager diff --git a/extensions/auth/ranger/impl/src/intTest/resources/authz_it_tests/dev_polaris.json b/extensions/auth/ranger/src/intTest/resources/authz_it_tests/dev_polaris.json similarity index 100% rename from extensions/auth/ranger/impl/src/intTest/resources/authz_it_tests/dev_polaris.json rename to extensions/auth/ranger/src/intTest/resources/authz_it_tests/dev_polaris.json diff --git a/extensions/auth/ranger/impl/src/intTest/resources/authz_it_tests/dev_polaris_roles.json b/extensions/auth/ranger/src/intTest/resources/authz_it_tests/dev_polaris_roles.json similarity index 100% rename from extensions/auth/ranger/impl/src/intTest/resources/authz_it_tests/dev_polaris_roles.json rename to extensions/auth/ranger/src/intTest/resources/authz_it_tests/dev_polaris_roles.json diff --git a/extensions/auth/ranger/impl/src/intTest/resources/authz_it_tests/dev_polaris_userstore.json b/extensions/auth/ranger/src/intTest/resources/authz_it_tests/dev_polaris_userstore.json similarity index 100% rename from extensions/auth/ranger/impl/src/intTest/resources/authz_it_tests/dev_polaris_userstore.json rename to extensions/auth/ranger/src/intTest/resources/authz_it_tests/dev_polaris_userstore.json diff --git a/extensions/auth/ranger/impl/src/main/java/org/apache/polaris/extension/auth/ranger/RangerPolarisAuthorizer.java b/extensions/auth/ranger/src/main/java/org/apache/polaris/extension/auth/ranger/RangerPolarisAuthorizer.java similarity index 100% rename from extensions/auth/ranger/impl/src/main/java/org/apache/polaris/extension/auth/ranger/RangerPolarisAuthorizer.java rename to extensions/auth/ranger/src/main/java/org/apache/polaris/extension/auth/ranger/RangerPolarisAuthorizer.java diff --git a/extensions/auth/ranger/impl/src/main/java/org/apache/polaris/extension/auth/ranger/RangerPolarisAuthorizerConfig.java b/extensions/auth/ranger/src/main/java/org/apache/polaris/extension/auth/ranger/RangerPolarisAuthorizerConfig.java similarity index 100% rename from extensions/auth/ranger/impl/src/main/java/org/apache/polaris/extension/auth/ranger/RangerPolarisAuthorizerConfig.java rename to extensions/auth/ranger/src/main/java/org/apache/polaris/extension/auth/ranger/RangerPolarisAuthorizerConfig.java diff --git a/extensions/auth/ranger/impl/src/main/java/org/apache/polaris/extension/auth/ranger/RangerPolarisAuthorizerFactory.java b/extensions/auth/ranger/src/main/java/org/apache/polaris/extension/auth/ranger/RangerPolarisAuthorizerFactory.java similarity index 100% rename from extensions/auth/ranger/impl/src/main/java/org/apache/polaris/extension/auth/ranger/RangerPolarisAuthorizerFactory.java rename to extensions/auth/ranger/src/main/java/org/apache/polaris/extension/auth/ranger/RangerPolarisAuthorizerFactory.java diff --git a/extensions/auth/ranger/impl/src/main/java/org/apache/polaris/extension/auth/ranger/RangerPolarisOperationSemantics.java b/extensions/auth/ranger/src/main/java/org/apache/polaris/extension/auth/ranger/RangerPolarisOperationSemantics.java similarity index 100% rename from extensions/auth/ranger/impl/src/main/java/org/apache/polaris/extension/auth/ranger/RangerPolarisOperationSemantics.java rename to extensions/auth/ranger/src/main/java/org/apache/polaris/extension/auth/ranger/RangerPolarisOperationSemantics.java diff --git a/extensions/auth/ranger/impl/src/main/java/org/apache/polaris/extension/auth/ranger/utils/RangerUtils.java b/extensions/auth/ranger/src/main/java/org/apache/polaris/extension/auth/ranger/utils/RangerUtils.java similarity index 100% rename from extensions/auth/ranger/impl/src/main/java/org/apache/polaris/extension/auth/ranger/utils/RangerUtils.java rename to extensions/auth/ranger/src/main/java/org/apache/polaris/extension/auth/ranger/utils/RangerUtils.java diff --git a/extensions/auth/ranger/impl/src/test/java/org/apache/polaris/extension/auth/ranger/RangerPolarisAuthorizerFactoryTest.java b/extensions/auth/ranger/src/test/java/org/apache/polaris/extension/auth/ranger/RangerPolarisAuthorizerFactoryTest.java similarity index 100% rename from extensions/auth/ranger/impl/src/test/java/org/apache/polaris/extension/auth/ranger/RangerPolarisAuthorizerFactoryTest.java rename to extensions/auth/ranger/src/test/java/org/apache/polaris/extension/auth/ranger/RangerPolarisAuthorizerFactoryTest.java diff --git a/extensions/auth/ranger/impl/src/test/java/org/apache/polaris/extension/auth/ranger/RangerPolarisAuthorizerTest.java b/extensions/auth/ranger/src/test/java/org/apache/polaris/extension/auth/ranger/RangerPolarisAuthorizerTest.java similarity index 100% rename from extensions/auth/ranger/impl/src/test/java/org/apache/polaris/extension/auth/ranger/RangerPolarisAuthorizerTest.java rename to extensions/auth/ranger/src/test/java/org/apache/polaris/extension/auth/ranger/RangerPolarisAuthorizerTest.java diff --git a/extensions/auth/ranger/impl/src/test/java/org/apache/polaris/extension/auth/ranger/RangerTestUtils.java b/extensions/auth/ranger/src/test/java/org/apache/polaris/extension/auth/ranger/RangerTestUtils.java similarity index 100% rename from extensions/auth/ranger/impl/src/test/java/org/apache/polaris/extension/auth/ranger/RangerTestUtils.java rename to extensions/auth/ranger/src/test/java/org/apache/polaris/extension/auth/ranger/RangerTestUtils.java diff --git a/extensions/auth/ranger/impl/src/test/resources/authz_tests/dev_polaris.json b/extensions/auth/ranger/src/test/resources/authz_tests/dev_polaris.json similarity index 100% rename from extensions/auth/ranger/impl/src/test/resources/authz_tests/dev_polaris.json rename to extensions/auth/ranger/src/test/resources/authz_tests/dev_polaris.json diff --git a/extensions/auth/ranger/impl/src/test/resources/authz_tests/tests_authz_catalog.json b/extensions/auth/ranger/src/test/resources/authz_tests/tests_authz_catalog.json similarity index 100% rename from extensions/auth/ranger/impl/src/test/resources/authz_tests/tests_authz_catalog.json rename to extensions/auth/ranger/src/test/resources/authz_tests/tests_authz_catalog.json diff --git a/extensions/auth/ranger/impl/src/test/resources/authz_tests/tests_authz_namespace.json b/extensions/auth/ranger/src/test/resources/authz_tests/tests_authz_namespace.json similarity index 100% rename from extensions/auth/ranger/impl/src/test/resources/authz_tests/tests_authz_namespace.json rename to extensions/auth/ranger/src/test/resources/authz_tests/tests_authz_namespace.json diff --git a/extensions/auth/ranger/impl/src/test/resources/authz_tests/tests_authz_policy.json b/extensions/auth/ranger/src/test/resources/authz_tests/tests_authz_policy.json similarity index 100% rename from extensions/auth/ranger/impl/src/test/resources/authz_tests/tests_authz_policy.json rename to extensions/auth/ranger/src/test/resources/authz_tests/tests_authz_policy.json diff --git a/extensions/auth/ranger/impl/src/test/resources/authz_tests/tests_authz_principal.json b/extensions/auth/ranger/src/test/resources/authz_tests/tests_authz_principal.json similarity index 100% rename from extensions/auth/ranger/impl/src/test/resources/authz_tests/tests_authz_principal.json rename to extensions/auth/ranger/src/test/resources/authz_tests/tests_authz_principal.json diff --git a/extensions/auth/ranger/impl/src/test/resources/authz_tests/tests_authz_root.json b/extensions/auth/ranger/src/test/resources/authz_tests/tests_authz_root.json similarity index 100% rename from extensions/auth/ranger/impl/src/test/resources/authz_tests/tests_authz_root.json rename to extensions/auth/ranger/src/test/resources/authz_tests/tests_authz_root.json diff --git a/extensions/auth/ranger/impl/src/test/resources/authz_tests/tests_authz_table.json b/extensions/auth/ranger/src/test/resources/authz_tests/tests_authz_table.json similarity index 100% rename from extensions/auth/ranger/impl/src/test/resources/authz_tests/tests_authz_table.json rename to extensions/auth/ranger/src/test/resources/authz_tests/tests_authz_table.json diff --git a/extensions/auth/ranger/impl/src/test/resources/authz_tests/tests_authz_unsupported.json b/extensions/auth/ranger/src/test/resources/authz_tests/tests_authz_unsupported.json similarity index 100% rename from extensions/auth/ranger/impl/src/test/resources/authz_tests/tests_authz_unsupported.json rename to extensions/auth/ranger/src/test/resources/authz_tests/tests_authz_unsupported.json diff --git a/extensions/auth/ranger/impl/src/test/resources/logback-test.xml b/extensions/auth/ranger/src/test/resources/logback-test.xml similarity index 100% rename from extensions/auth/ranger/impl/src/test/resources/logback-test.xml rename to extensions/auth/ranger/src/test/resources/logback-test.xml diff --git a/gradle/projects.main.properties b/gradle/projects.main.properties index 21f94659d95..0e37e570d7f 100644 --- a/gradle/projects.main.properties +++ b/gradle/projects.main.properties @@ -49,7 +49,7 @@ polaris-extensions-federation-hive=extensions/federation/hive polaris-extensions-federation-bigquery=extensions/federation/bigquery polaris-extensions-auth-opa=extensions/auth/opa/impl polaris-extensions-auth-opa-tests=extensions/auth/opa/tests -polaris-extensions-auth-ranger=extensions/auth/ranger/impl +polaris-extensions-auth-ranger=extensions/auth/ranger polaris-config-docs-annotations=tools/config-docs/annotations polaris-config-docs-generator=tools/config-docs/generator From 64a90b213f506a634061f577f103923509e56b4b Mon Sep 17 00:00:00 2001 From: Robert Stupp Date: Tue, 9 Jun 2026 14:20:31 +0200 Subject: [PATCH 09/22] Migrate OPA integration tests to server runner --- extensions/auth/opa/impl/build.gradle.kts | 4 +- extensions/auth/opa/tests/build.gradle.kts | 200 ++++++++++++++---- .../auth/opa/test/OpaAdminServiceIT.java | 4 - .../opa/test/OpaGenericTableHandlerIT.java | 4 - .../opa/test/OpaIcebergCatalogHandlerIT.java | 4 - .../auth/opa/test/OpaIntegrationTest.java | 4 - .../opa/test/OpaPolicyCatalogHandlerIT.java | 4 - .../auth/opa/test/OpaTestProfiles.java | 87 -------- .../auth/opa/test/OpaIntegrationTestBase.java | 14 ++ ...olaris.service.it.ext.PolarisServerManager | 20 ++ .../opa/test/OpaFileTokenIntegrationTest.java | 6 +- .../auth/opa/test/OpaStartupAction.java} | 120 +++++------ .../auth/opa/test/Dockerfile-opa-version | 0 13 files changed, 247 insertions(+), 224 deletions(-) rename extensions/auth/opa/tests/src/{intTest => bearerTokenIntTest}/java/org/apache/polaris/extension/auth/opa/test/OpaAdminServiceIT.java (99%) rename extensions/auth/opa/tests/src/{intTest => bearerTokenIntTest}/java/org/apache/polaris/extension/auth/opa/test/OpaGenericTableHandlerIT.java (96%) rename extensions/auth/opa/tests/src/{intTest => bearerTokenIntTest}/java/org/apache/polaris/extension/auth/opa/test/OpaIcebergCatalogHandlerIT.java (97%) rename extensions/auth/opa/tests/src/{intTest => bearerTokenIntTest}/java/org/apache/polaris/extension/auth/opa/test/OpaIntegrationTest.java (94%) rename extensions/auth/opa/tests/src/{intTest => bearerTokenIntTest}/java/org/apache/polaris/extension/auth/opa/test/OpaPolicyCatalogHandlerIT.java (97%) delete mode 100644 extensions/auth/opa/tests/src/intTest/java/org/apache/polaris/extension/auth/opa/test/OpaTestProfiles.java rename extensions/auth/opa/tests/src/{intTest => intTestBase}/java/org/apache/polaris/extension/auth/opa/test/OpaIntegrationTestBase.java (95%) create mode 100644 extensions/auth/opa/tests/src/intTestBase/resources/META-INF/services/org.apache.polaris.service.it.ext.PolarisServerManager rename extensions/auth/opa/tests/src/{intTest => opaFileTokenIntTest}/java/org/apache/polaris/extension/auth/opa/test/OpaFileTokenIntegrationTest.java (95%) rename extensions/auth/opa/tests/src/{intTest/java/org/apache/polaris/extension/auth/opa/test/OpaTestResource.java => opaStartupAction/java/org/apache/polaris/extension/auth/opa/test/OpaStartupAction.java} (53%) rename extensions/auth/opa/tests/src/{intTest => opaStartupAction}/resources/org/apache/polaris/extension/auth/opa/test/Dockerfile-opa-version (100%) diff --git a/extensions/auth/opa/impl/build.gradle.kts b/extensions/auth/opa/impl/build.gradle.kts index 165c19d9b79..fe152e26e11 100644 --- a/extensions/auth/opa/impl/build.gradle.kts +++ b/extensions/auth/opa/impl/build.gradle.kts @@ -124,6 +124,8 @@ tasks.register("validateOpaSchema") { errorOutput = outStream } + val prjDir = projectDir + doLast { outStream?.close() @@ -138,7 +140,7 @@ tasks.register("validateOpaSchema") { ) } - val generatedContent = projectDir.resolve(tempSchemaFile).readText().trim() + val generatedContent = prjDir.resolve(tempSchemaFile).readText().trim() val committedContent = committedSchemaFile.readText().trim() if (generatedContent != committedContent) { diff --git a/extensions/auth/opa/tests/build.gradle.kts b/extensions/auth/opa/tests/build.gradle.kts index c83bde936fc..68c79a16151 100644 --- a/extensions/auth/opa/tests/build.gradle.kts +++ b/extensions/auth/opa/tests/build.gradle.kts @@ -17,63 +17,175 @@ * under the License. */ +import org.gradle.api.attributes.java.TargetJvmVersion +import org.gradle.api.plugins.jvm.JvmTestSuite +import org.gradle.language.base.plugins.LifecycleBasePlugin + plugins { - alias(libs.plugins.quarkus) - id("org.kordamp.gradle.jandex") - id("polaris-runtime") + id("polaris-client") + id("polaris-server-test-runner") } +val intTestJvmVersion = 21 +val intTestBase by + configurations.creating { + isCanBeConsumed = false + isCanBeResolved = false + } +val intTestBaseSources = sourceSets.create("intTestBase") +val opaStartupAction = sourceSets.create("opaStartupAction") +val opaStartupActionCompileOnly by configurations.getting +val opaStartupActionImplementation by configurations.getting +val opaBearerTokenRefreshIntervalProperty = + "polaris.authorization.opa.auth.bearer.file-based.refresh-interval" + +listOf( + intTestBaseSources.implementationConfigurationName, + opaStartupAction.runtimeOnlyConfigurationName, + ) + .forEach { configurations.named(it) { extendsFrom(intTestBase) } } + dependencies { - // Quarkus platform - implementation(enforcedPlatform(libs.quarkus.bom)) - implementation("io.quarkus:quarkus-rest-jackson") - - // Add the OPA implementation as RUNTIME dependency to include in Quarkus app - implementation(project(":polaris-extensions-auth-opa")) - - // Include all runtime-service dependencies - implementation(project(":polaris-runtime-service")) - - // Test common for integration testing - testImplementation(project(":polaris-runtime-test-common")) - - // Test dependencies - intTestImplementation("io.quarkus:quarkus-junit") - intTestImplementation("io.rest-assured:rest-assured") - intTestImplementation(project(":polaris-api-management-model")) - intTestImplementation(platform(libs.iceberg.bom)) - intTestImplementation("org.apache.iceberg:iceberg-api") - intTestImplementation("org.apache.iceberg:iceberg-core") - - // Test container dependencies - intTestImplementation(platform(libs.testcontainers.bom)) - intTestImplementation("org.testcontainers:testcontainers-junit-jupiter") - intTestImplementation(project(":polaris-container-spec-helper")) + polarisServer(project(path = ":polaris-server", configuration = "quarkusRunner")) + + opaStartupActionCompileOnly("org.apache.polaris.server-test-runner:polaris-server-test-runner") + opaStartupActionImplementation(platform(libs.testcontainers.bom)) + opaStartupActionImplementation("org.testcontainers:testcontainers") + opaStartupActionImplementation(project(":polaris-container-spec-helper")) + + intTestBase(platform(libs.junit.bom)) + intTestBase("org.junit.jupiter:junit-jupiter") + intTestBase("org.junit.platform:junit-platform-launcher") + intTestBase(enforcedPlatform(libs.quarkus.bom)) + intTestBase("io.rest-assured:rest-assured") + intTestBase(project(":polaris-tests")) + intTestBase(project(":polaris-runtime-test-common")) + intTestBase(project(":polaris-api-management-model")) + intTestBase(platform(libs.iceberg.bom)) + intTestBase("org.apache.iceberg:iceberg-api") + intTestBase("org.apache.iceberg:iceberg-core") } -tasks.named("javadoc") { dependsOn("jandex") } +fun Test.configureOpaTestTask( + buildDir: DirectoryProperty, + staticToken: String? = null, + fileTokenPath: Provider? = null, +) { + description = "Runs OPA integration tests against an external Polaris server." + group = LifecycleBasePlugin.VERIFICATION_GROUP -tasks.withType { - if (System.getenv("AWS_REGION") == null) { - environment("AWS_REGION", "us-west-2") - } - environment("POLARIS_BOOTSTRAP_CREDENTIALS", "POLARIS,test-admin,test-secret") - val apiVersion = System.getenv("DOCKER_API_VERSION") ?: "1.44" + environment("AWS_REGION", providers.environmentVariable("AWS_REGION").getOrElse("us-west-2")) + environment(mapOf("POLARIS_BOOTSTRAP_CREDENTIALS" to "POLARIS,test-admin,test-secret")) + val apiVersion = providers.environmentVariable("DOCKER_API_VERSION").getOrElse("1.44") systemProperty("api.version", apiVersion) jvmArgs("--add-exports", "java.base/sun.nio.ch=ALL-UNNAMED") systemProperty("java.security.manager", "allow") - maxParallelForks = 1 - val logsDir = project.layout.buildDirectory.get().asFile.resolve("logs") - - jvmArgumentProviders.add( - CommandLineArgumentProvider { - listOf("-Dquarkus.log.file.path=${logsDir.resolve("polaris.log").absolutePath}") - } - ) + val logsDir = buildDir.get().asFile.resolve("logs/$name") doFirst { logsDir.deleteRecursively() - project.layout.buildDirectory.get().asFile.resolve("quarkus.log").delete() + buildDir.get().asFile.resolve("quarkus.log").delete() + } + + withPolarisServer(configurations.polarisServer) { + startupActionClasspath.from(opaStartupAction.runtimeClasspath) + startupActionClass.set("org.apache.polaris.extension.auth.opa.test.OpaStartupAction") + + environment.put("AWS_REGION", providers.environmentVariable("AWS_REGION").orElse("us-west-2")) + environment.putAll(mapOf("POLARIS_BOOTSTRAP_CREDENTIALS" to "POLARIS,test-admin,test-secret")) + systemProperties.putAll( + mapOf( + "quarkus.log.file.path" to logsDir.resolve("polaris.log").absolutePath, + "polaris.authorization.type" to "opa", + "polaris.authorization.opa.auth.type" to "bearer", + "polaris.features.\"SUPPORTED_CATALOG_STORAGE_TYPES\"" to "[\"FILE\"]", + "polaris.features.\"ALLOW_INSECURE_STORAGE_TYPES\"" to "true", + "polaris.readiness.ignore-severe-issues" to "true", + ) + ) + staticToken?.let { + systemProperties.putAll( + mapOf("polaris.authorization.opa.auth.bearer.static-token.value" to it) + ) + } + fileTokenPath?.orNull?.let { + systemProperties.putAll( + mapOf( + "polaris.authorization.opa.auth.bearer.file-based.path" to it, + opaBearerTokenRefreshIntervalProperty to "PT1S", + ) + ) + } + } +} + +@Suppress("UnstableApiUsage") +fun JvmTestSuite.configureIntTest() { + useJUnitJupiter() + + configurations.named(sources.implementationConfigurationName) { extendsFrom(intTestBase) } + sources.compileClasspath += intTestBaseSources.output + sources.runtimeClasspath += intTestBaseSources.output +} + +testing { + suites { + val buildDir = project.layout.buildDirectory + + @Suppress("UnstableApiUsage") + register("bearerTokenIntTest") { + configureIntTest() + targets { + all { + testTask.configure { + configureOpaTestTask(buildDir, staticToken = "test-opa-bearer-token-12345") + } + } + } + } + + @Suppress("UnstableApiUsage") + register("opaFileTokenIntTest") { + configureIntTest() + val tokenFile = buildDir.file("opa-file-token/token.txt") + targets { + all { + testTask.configure { + configureOpaTestTask(buildDir, fileTokenPath = tokenFile.map { it.asFile.absolutePath }) + + doFirst { + val file = tokenFile.get().asFile + file.parentFile.mkdirs() + file.writeText("test-opa-bearer-token-from-file") + } + } + } + } + } + } +} + +listOf( + intTestBaseSources.compileClasspathConfigurationName, + intTestBaseSources.runtimeClasspathConfigurationName, + "bearerTokenIntTestCompileClasspath", + "bearerTokenIntTestRuntimeClasspath", + "opaFileTokenIntTestCompileClasspath", + "opaFileTokenIntTestRuntimeClasspath", + "opaStartupActionCompileClasspath", + "opaStartupActionRuntimeClasspath", + ) + .forEach { + configurations.named(it).configure { + attributes.attribute(TargetJvmVersion.TARGET_JVM_VERSION_ATTRIBUTE, intTestJvmVersion) + } } + +tasks.register("intTest") { + description = "Runs all OPA integration tests." + group = LifecycleBasePlugin.VERIFICATION_GROUP + dependsOn("bearerTokenIntTest", "opaFileTokenIntTest") } + +tasks.named("check") { dependsOn("intTest") } diff --git a/extensions/auth/opa/tests/src/intTest/java/org/apache/polaris/extension/auth/opa/test/OpaAdminServiceIT.java b/extensions/auth/opa/tests/src/bearerTokenIntTest/java/org/apache/polaris/extension/auth/opa/test/OpaAdminServiceIT.java similarity index 99% rename from extensions/auth/opa/tests/src/intTest/java/org/apache/polaris/extension/auth/opa/test/OpaAdminServiceIT.java rename to extensions/auth/opa/tests/src/bearerTokenIntTest/java/org/apache/polaris/extension/auth/opa/test/OpaAdminServiceIT.java index 86be8d058cb..e2f259e8ae8 100644 --- a/extensions/auth/opa/tests/src/intTest/java/org/apache/polaris/extension/auth/opa/test/OpaAdminServiceIT.java +++ b/extensions/auth/opa/tests/src/bearerTokenIntTest/java/org/apache/polaris/extension/auth/opa/test/OpaAdminServiceIT.java @@ -20,8 +20,6 @@ import static io.restassured.RestAssured.given; -import io.quarkus.test.junit.QuarkusTest; -import io.quarkus.test.junit.TestProfile; import io.restassured.http.ContentType; import java.net.URI; import java.nio.file.Files; @@ -49,8 +47,6 @@ *

  • Catalog-role grant listings * */ -@QuarkusTest -@TestProfile(OpaTestProfiles.StaticToken.class) public class OpaAdminServiceIT extends OpaIntegrationTestBase { private @TempDir Path baseCatalogLocationPath; diff --git a/extensions/auth/opa/tests/src/intTest/java/org/apache/polaris/extension/auth/opa/test/OpaGenericTableHandlerIT.java b/extensions/auth/opa/tests/src/bearerTokenIntTest/java/org/apache/polaris/extension/auth/opa/test/OpaGenericTableHandlerIT.java similarity index 96% rename from extensions/auth/opa/tests/src/intTest/java/org/apache/polaris/extension/auth/opa/test/OpaGenericTableHandlerIT.java rename to extensions/auth/opa/tests/src/bearerTokenIntTest/java/org/apache/polaris/extension/auth/opa/test/OpaGenericTableHandlerIT.java index 54a8d41a45d..ef221979a95 100644 --- a/extensions/auth/opa/tests/src/intTest/java/org/apache/polaris/extension/auth/opa/test/OpaGenericTableHandlerIT.java +++ b/extensions/auth/opa/tests/src/bearerTokenIntTest/java/org/apache/polaris/extension/auth/opa/test/OpaGenericTableHandlerIT.java @@ -20,8 +20,6 @@ import static io.restassured.RestAssured.given; -import io.quarkus.test.junit.QuarkusTest; -import io.quarkus.test.junit.TestProfile; import io.restassured.http.ContentType; import java.nio.file.Files; import java.nio.file.Path; @@ -41,8 +39,6 @@ *
  • Drop generic table * */ -@QuarkusTest -@TestProfile(OpaTestProfiles.StaticToken.class) public class OpaGenericTableHandlerIT extends OpaIntegrationTestBase { private String catalogName; diff --git a/extensions/auth/opa/tests/src/intTest/java/org/apache/polaris/extension/auth/opa/test/OpaIcebergCatalogHandlerIT.java b/extensions/auth/opa/tests/src/bearerTokenIntTest/java/org/apache/polaris/extension/auth/opa/test/OpaIcebergCatalogHandlerIT.java similarity index 97% rename from extensions/auth/opa/tests/src/intTest/java/org/apache/polaris/extension/auth/opa/test/OpaIcebergCatalogHandlerIT.java rename to extensions/auth/opa/tests/src/bearerTokenIntTest/java/org/apache/polaris/extension/auth/opa/test/OpaIcebergCatalogHandlerIT.java index 882de5adfc3..6ddfca80bcb 100644 --- a/extensions/auth/opa/tests/src/intTest/java/org/apache/polaris/extension/auth/opa/test/OpaIcebergCatalogHandlerIT.java +++ b/extensions/auth/opa/tests/src/bearerTokenIntTest/java/org/apache/polaris/extension/auth/opa/test/OpaIcebergCatalogHandlerIT.java @@ -20,8 +20,6 @@ import static io.restassured.RestAssured.given; -import io.quarkus.test.junit.QuarkusTest; -import io.quarkus.test.junit.TestProfile; import io.restassured.http.ContentType; import java.net.URI; import java.nio.file.Files; @@ -47,8 +45,6 @@ *
  • Drop table * */ -@QuarkusTest -@TestProfile(OpaTestProfiles.StaticToken.class) public class OpaIcebergCatalogHandlerIT extends OpaIntegrationTestBase { private String catalogName; diff --git a/extensions/auth/opa/tests/src/intTest/java/org/apache/polaris/extension/auth/opa/test/OpaIntegrationTest.java b/extensions/auth/opa/tests/src/bearerTokenIntTest/java/org/apache/polaris/extension/auth/opa/test/OpaIntegrationTest.java similarity index 94% rename from extensions/auth/opa/tests/src/intTest/java/org/apache/polaris/extension/auth/opa/test/OpaIntegrationTest.java rename to extensions/auth/opa/tests/src/bearerTokenIntTest/java/org/apache/polaris/extension/auth/opa/test/OpaIntegrationTest.java index 1cf4cb6deb2..8891801dcbf 100644 --- a/extensions/auth/opa/tests/src/intTest/java/org/apache/polaris/extension/auth/opa/test/OpaIntegrationTest.java +++ b/extensions/auth/opa/tests/src/bearerTokenIntTest/java/org/apache/polaris/extension/auth/opa/test/OpaIntegrationTest.java @@ -21,12 +21,8 @@ import static io.restassured.RestAssured.given; import static org.assertj.core.api.Assertions.assertThatNoException; -import io.quarkus.test.junit.QuarkusTest; -import io.quarkus.test.junit.TestProfile; import org.junit.jupiter.api.Test; -@QuarkusTest -@TestProfile(OpaTestProfiles.StaticToken.class) public class OpaIntegrationTest extends OpaIntegrationTestBase { @Test diff --git a/extensions/auth/opa/tests/src/intTest/java/org/apache/polaris/extension/auth/opa/test/OpaPolicyCatalogHandlerIT.java b/extensions/auth/opa/tests/src/bearerTokenIntTest/java/org/apache/polaris/extension/auth/opa/test/OpaPolicyCatalogHandlerIT.java similarity index 97% rename from extensions/auth/opa/tests/src/intTest/java/org/apache/polaris/extension/auth/opa/test/OpaPolicyCatalogHandlerIT.java rename to extensions/auth/opa/tests/src/bearerTokenIntTest/java/org/apache/polaris/extension/auth/opa/test/OpaPolicyCatalogHandlerIT.java index 6400a47c042..44bb5cd750f 100644 --- a/extensions/auth/opa/tests/src/intTest/java/org/apache/polaris/extension/auth/opa/test/OpaPolicyCatalogHandlerIT.java +++ b/extensions/auth/opa/tests/src/bearerTokenIntTest/java/org/apache/polaris/extension/auth/opa/test/OpaPolicyCatalogHandlerIT.java @@ -20,8 +20,6 @@ import static io.restassured.RestAssured.given; -import io.quarkus.test.junit.QuarkusTest; -import io.quarkus.test.junit.TestProfile; import io.restassured.http.ContentType; import java.nio.file.Files; import java.nio.file.Path; @@ -41,8 +39,6 @@ *
  • Delete policies * */ -@QuarkusTest -@TestProfile(OpaTestProfiles.StaticToken.class) public class OpaPolicyCatalogHandlerIT extends OpaIntegrationTestBase { private String catalogName; diff --git a/extensions/auth/opa/tests/src/intTest/java/org/apache/polaris/extension/auth/opa/test/OpaTestProfiles.java b/extensions/auth/opa/tests/src/intTest/java/org/apache/polaris/extension/auth/opa/test/OpaTestProfiles.java deleted file mode 100644 index a4864946af2..00000000000 --- a/extensions/auth/opa/tests/src/intTest/java/org/apache/polaris/extension/auth/opa/test/OpaTestProfiles.java +++ /dev/null @@ -1,87 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ -package org.apache.polaris.extension.auth.opa.test; - -import io.quarkus.test.junit.QuarkusTestProfile; -import java.io.IOException; -import java.nio.file.Files; -import java.nio.file.Path; -import java.util.HashMap; -import java.util.List; -import java.util.Map; - -/** Shared Quarkus test profiles for OPA integration tests. */ -public final class OpaTestProfiles { - - private OpaTestProfiles() {} - - /** OPA profile using a static bearer token. */ - public static class StaticToken implements QuarkusTestProfile { - @Override - public Map getConfigOverrides() { - Map config = new HashMap<>(); - config.put("polaris.authorization.type", "opa"); - config.put("polaris.authorization.opa.auth.type", "bearer"); - config.put( - "polaris.authorization.opa.auth.bearer.static-token.value", - "test-opa-bearer-token-12345"); - config.put("polaris.features.\"SUPPORTED_CATALOG_STORAGE_TYPES\"", "[\"FILE\"]"); - config.put("polaris.features.\"ALLOW_INSECURE_STORAGE_TYPES\"", "true"); - config.put("polaris.readiness.ignore-severe-issues", "true"); - return config; - } - - @Override - public List testResources() { - return List.of(new TestResourceEntry(OpaTestResource.class)); - } - } - - /** OPA profile using a bearer token read from a file. */ - public static class FileToken implements QuarkusTestProfile { - // Exposed for tests that may need to inspect the created token file. - public static Path tokenFilePath; - - @Override - public Map getConfigOverrides() { - try { - tokenFilePath = Files.createTempFile("opa-test-token", ".txt"); - Files.writeString(tokenFilePath, "test-opa-bearer-token-from-file"); - - Map config = new HashMap<>(); - config.put("polaris.authorization.type", "opa"); - config.put("polaris.authorization.opa.auth.type", "bearer"); - config.put( - "polaris.authorization.opa.auth.bearer.file-based.path", tokenFilePath.toString()); - config.put("polaris.authorization.opa.auth.bearer.file-based.refresh-interval", "PT1S"); - config.put("polaris.features.\"SUPPORTED_CATALOG_STORAGE_TYPES\"", "[\"FILE\"]"); - config.put("polaris.features.\"ALLOW_INSECURE_STORAGE_TYPES\"", "true"); - config.put("polaris.readiness.ignore-severe-issues", "true"); - return config; - } catch (IOException e) { - throw new RuntimeException("Failed to create test token file", e); - } - } - - @Override - public List testResources() { - return List.of(new TestResourceEntry(OpaTestResource.class)); - } - } -} diff --git a/extensions/auth/opa/tests/src/intTest/java/org/apache/polaris/extension/auth/opa/test/OpaIntegrationTestBase.java b/extensions/auth/opa/tests/src/intTestBase/java/org/apache/polaris/extension/auth/opa/test/OpaIntegrationTestBase.java similarity index 95% rename from extensions/auth/opa/tests/src/intTest/java/org/apache/polaris/extension/auth/opa/test/OpaIntegrationTestBase.java rename to extensions/auth/opa/tests/src/intTestBase/java/org/apache/polaris/extension/auth/opa/test/OpaIntegrationTestBase.java index aa121c521f1..269be94e8be 100644 --- a/extensions/auth/opa/tests/src/intTest/java/org/apache/polaris/extension/auth/opa/test/OpaIntegrationTestBase.java +++ b/extensions/auth/opa/tests/src/intTestBase/java/org/apache/polaris/extension/auth/opa/test/OpaIntegrationTestBase.java @@ -23,13 +23,16 @@ import com.fasterxml.jackson.databind.JsonNode; import com.fasterxml.jackson.databind.json.JsonMapper; +import io.restassured.RestAssured; import io.restassured.http.ContentType; import java.io.UncheckedIOException; import java.util.ArrayList; import java.util.Collections; import java.util.List; import java.util.Map; +import org.junit.jupiter.api.AfterAll; import org.junit.jupiter.api.AfterEach; +import org.junit.jupiter.api.BeforeAll; /** * Base class for OPA integration tests providing common helper methods for authentication and @@ -40,6 +43,17 @@ public abstract class OpaIntegrationTestBase { private static final JsonMapper mapper = JsonMapper.builder().build(); private final List catalogsToCleanup = new ArrayList<>(); + @BeforeAll + static void configureRestAssured() { + RestAssured.baseURI = "http://localhost"; + RestAssured.port = Integer.getInteger("quarkus.http.test-port"); + } + + @AfterAll + static void resetRestAssured() { + RestAssured.reset(); + } + protected String toJson(Object value) { try { return mapper.writeValueAsString(value); diff --git a/extensions/auth/opa/tests/src/intTestBase/resources/META-INF/services/org.apache.polaris.service.it.ext.PolarisServerManager b/extensions/auth/opa/tests/src/intTestBase/resources/META-INF/services/org.apache.polaris.service.it.ext.PolarisServerManager new file mode 100644 index 00000000000..6aa2dac9e0c --- /dev/null +++ b/extensions/auth/opa/tests/src/intTestBase/resources/META-INF/services/org.apache.polaris.service.it.ext.PolarisServerManager @@ -0,0 +1,20 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# + +org.apache.polaris.service.it.ext.ExternalPolarisServerManager diff --git a/extensions/auth/opa/tests/src/intTest/java/org/apache/polaris/extension/auth/opa/test/OpaFileTokenIntegrationTest.java b/extensions/auth/opa/tests/src/opaFileTokenIntTest/java/org/apache/polaris/extension/auth/opa/test/OpaFileTokenIntegrationTest.java similarity index 95% rename from extensions/auth/opa/tests/src/intTest/java/org/apache/polaris/extension/auth/opa/test/OpaFileTokenIntegrationTest.java rename to extensions/auth/opa/tests/src/opaFileTokenIntTest/java/org/apache/polaris/extension/auth/opa/test/OpaFileTokenIntegrationTest.java index 81c2a04364a..e473b9325c1 100644 --- a/extensions/auth/opa/tests/src/intTest/java/org/apache/polaris/extension/auth/opa/test/OpaFileTokenIntegrationTest.java +++ b/extensions/auth/opa/tests/src/opaFileTokenIntTest/java/org/apache/polaris/extension/auth/opa/test/OpaFileTokenIntegrationTest.java @@ -21,8 +21,7 @@ import static io.restassured.RestAssured.given; import static org.assertj.core.api.Assertions.assertThatNoException; -import io.quarkus.test.junit.QuarkusTest; -import io.quarkus.test.junit.TestProfile; +import org.junit.jupiter.api.Tag; import org.junit.jupiter.api.Test; /** @@ -31,8 +30,7 @@ *

    These tests verify that OpaPolarisAuthorizer correctly reads bearer tokens from a file and * uses them to authenticate with OPA. */ -@QuarkusTest -@TestProfile(OpaTestProfiles.FileToken.class) +@Tag("opa-file-token") public class OpaFileTokenIntegrationTest extends OpaIntegrationTestBase { @Test diff --git a/extensions/auth/opa/tests/src/intTest/java/org/apache/polaris/extension/auth/opa/test/OpaTestResource.java b/extensions/auth/opa/tests/src/opaStartupAction/java/org/apache/polaris/extension/auth/opa/test/OpaStartupAction.java similarity index 53% rename from extensions/auth/opa/tests/src/intTest/java/org/apache/polaris/extension/auth/opa/test/OpaTestResource.java rename to extensions/auth/opa/tests/src/opaStartupAction/java/org/apache/polaris/extension/auth/opa/test/OpaStartupAction.java index 456f380c0fc..b663ebba256 100644 --- a/extensions/auth/opa/tests/src/intTest/java/org/apache/polaris/extension/auth/opa/test/OpaTestResource.java +++ b/extensions/auth/opa/tests/src/opaStartupAction/java/org/apache/polaris/extension/auth/opa/test/OpaStartupAction.java @@ -18,84 +18,60 @@ */ package org.apache.polaris.extension.auth.opa.test; -import io.quarkus.test.common.QuarkusTestResourceLifecycleManager; import java.io.OutputStream; import java.net.HttpURLConnection; import java.net.URI; import java.net.URL; import java.nio.charset.StandardCharsets; import java.time.Duration; -import java.util.HashMap; -import java.util.Map; import org.apache.polaris.containerspec.ContainerSpecHelper; +import org.apache.polaris.server.test.runner.spi.PolarisServerStartupAction; +import org.apache.polaris.server.test.runner.spi.PolarisServerStartupContext; import org.testcontainers.containers.GenericContainer; import org.testcontainers.containers.wait.strategy.Wait; -public class OpaTestResource implements QuarkusTestResourceLifecycleManager { - private static GenericContainer opa; +/** Starts an OPA test server before the external Polaris server process starts. */ +public class OpaStartupAction implements PolarisServerStartupAction { + private static final int OPA_PORT = 8181; + private static final String POLICY_NAME = "polaris-authz"; - @SuppressWarnings({"resource", "HttpUrlsUsage"}) - @Override - public Map start() { - try { - // Reuse container across tests to speed up execution - if (opa == null || !opa.isRunning()) { - opa = - new GenericContainer<>( - ContainerSpecHelper.containerSpecHelper("opa", OpaTestResource.class) - .dockerImageName(null)) - .withExposedPorts(8181) - .withReuse(true) - .withCommand("run", "--server", "--addr=0.0.0.0:8181") - .waitingFor( - Wait.forHttp("/health") - .forPort(8181) - .forStatusCode(200) - .withStartupTimeout(Duration.ofSeconds(120))); - - opa.start(); - } - - int mappedPort = opa.getMappedPort(8181); - String containerHost = opa.getHost(); - String baseUrl = "http://" + containerHost + ":" + mappedPort; - - // Load Opa Polaris Authorizer Rego policy into OPA - String polarisPolicyName = "polaris-authz"; - String polarisRegoPolicy = - """ - package polaris.authz - - default allow := false - - # Allow root user for all operations - allow if { - input.actor.principal == "root" - } + private GenericContainer opa; - # Allow admin user for all operations - allow if { - input.actor.principal == "admin" - } - """; - loadRegoPolicy(baseUrl, polarisPolicyName, polarisRegoPolicy); - - Map config = new HashMap<>(); - config.put("polaris.authorization.opa.policy-uri", baseUrl + "/v1/data/polaris/authz"); - - return config; + @Override + @SuppressWarnings("resource") + public void start(PolarisServerStartupContext context) { + opa = + new GenericContainer<>( + ContainerSpecHelper.containerSpecHelper("opa", OpaStartupAction.class) + .dockerImageName(null)) + .withExposedPorts(OPA_PORT) + .withCommand("run", "--server", "--addr=0.0.0.0:8181") + .waitingFor( + Wait.forHttp("/health") + .forPort(OPA_PORT) + .forStatusCode(200) + .withStartupTimeout(Duration.ofSeconds(120))); + + opa.start(); + + String baseUrl = "http://" + opa.getHost() + ":" + opa.getMappedPort(OPA_PORT); + loadRegoPolicy(baseUrl, POLICY_NAME, polarisRegoPolicy()); + context + .getSystemProperties() + .put("polaris.authorization.opa.policy-uri", baseUrl + "/v1/data/polaris/authz"); + } - } catch (Exception e) { - throw new RuntimeException("Failed to start OPA test resource", e); + @Override + public void close() { + if (opa != null) { + opa.stop(); + opa = null; } } private void loadRegoPolicy(String baseUrl, String policyName, String regoPolicy) { - // Hardcode the policy directly instead of loading through QuarkusTestProfile try { URL url = URI.create(baseUrl + "/v1/policies/" + policyName).toURL(); - System.out.println("Uploading policy to: " + url); - HttpURLConnection conn = (HttpURLConnection) url.openConnection(); conn.setRequestMethod("PUT"); conn.setDoOutput(true); @@ -106,27 +82,35 @@ private void loadRegoPolicy(String baseUrl, String policyName, String regoPolicy } int code = conn.getResponseCode(); - System.out.println("OPA policy upload response code: " + code); - if (code < 200 || code >= 300) { throw new RuntimeException("OPA policy upload failed, HTTP " + code); } - - System.out.println("Successfully uploaded policy to OPA"); } catch (Exception e) { - // Surface container logs to help debug on CI String logs = ""; try { logs = opa.getLogs(); } catch (Throwable ignored) { + // ignore logging failures while reporting the original startup failure } throw new RuntimeException("Failed to load OPA policy. Container logs:\n" + logs, e); } } - @Override - public void stop() { - // Quarkus takes care of reusing the test resource across tests - opa.stop(); + private String polarisRegoPolicy() { + return """ + package polaris.authz + + default allow := false + + # Allow root user for all operations + allow if { + input.actor.principal == "root" + } + + # Allow admin user for all operations + allow if { + input.actor.principal == "admin" + } + """; } } diff --git a/extensions/auth/opa/tests/src/intTest/resources/org/apache/polaris/extension/auth/opa/test/Dockerfile-opa-version b/extensions/auth/opa/tests/src/opaStartupAction/resources/org/apache/polaris/extension/auth/opa/test/Dockerfile-opa-version similarity index 100% rename from extensions/auth/opa/tests/src/intTest/resources/org/apache/polaris/extension/auth/opa/test/Dockerfile-opa-version rename to extensions/auth/opa/tests/src/opaStartupAction/resources/org/apache/polaris/extension/auth/opa/test/Dockerfile-opa-version From 623dad25637e61983e9cd549dfe41fadb713c32f Mon Sep 17 00:00:00 2001 From: Robert Stupp Date: Tue, 9 Jun 2026 14:36:22 +0200 Subject: [PATCH 10/22] Move OPA tests into OPA extension --- bom/build.gradle.kts | 1 - extensions/auth/opa/{impl => }/SCHEMA.md | 2 +- .../auth/opa/{tests => }/build.gradle.kts | 144 ++++++++++++++- extensions/auth/opa/impl/build.gradle.kts | 171 ------------------ .../auth/opa/{impl => }/opa-input-schema.json | 0 .../auth/opa/test/OpaAdminServiceIT.java | 0 .../opa/test/OpaGenericTableHandlerIT.java | 0 .../opa/test/OpaIcebergCatalogHandlerIT.java | 0 .../auth/opa/test/OpaIntegrationTest.java | 0 .../opa/test/OpaPolicyCatalogHandlerIT.java | 0 .../auth/opa/test/OpaIntegrationTestBase.java | 0 ...olaris.service.it.ext.PolarisServerManager | 0 .../auth/opa/model/OpaSchemaGenerator.java | 0 .../auth/opa/OpaAuthorizationConfig.java | 0 .../auth/opa/OpaHttpClientFactory.java | 0 .../auth/opa/OpaPolarisAuthorizer.java | 0 .../auth/opa/OpaPolarisAuthorizerFactory.java | 0 .../opa/OpaProductionReadinessChecks.java | 0 .../extension/auth/opa/model/Actor.java | 0 .../extension/auth/opa/model/Context.java | 0 .../auth/opa/model/OpaAuthorizationInput.java | 0 .../extension/auth/opa/model/OpaRequest.java | 0 .../extension/auth/opa/model/README.md | 2 +- .../extension/auth/opa/model/Resource.java | 0 .../auth/opa/model/ResourceEntity.java | 0 .../auth/opa/model/package-info.java | 0 .../auth/opa/token/BearerTokenProvider.java | 0 .../opa/token/FileBearerTokenProvider.java | 0 .../opa/token/StaticBearerTokenProvider.java | 0 .../opa/test/OpaFileTokenIntegrationTest.java | 0 .../auth/opa/test/OpaStartupAction.java | 0 .../auth/opa/test/Dockerfile-opa-version | 0 .../auth/opa/OpaHttpClientFactoryTest.java | 0 .../opa/OpaPolarisAuthorizerFactoryTest.java | 0 .../auth/opa/OpaPolarisAuthorizerTest.java | 0 .../token/FileBearerTokenProviderTest.java | 0 .../token/StaticBearerTokenProviderTest.java | 0 gradle/projects.main.properties | 3 +- 38 files changed, 145 insertions(+), 178 deletions(-) rename extensions/auth/opa/{impl => }/SCHEMA.md (98%) rename extensions/auth/opa/{tests => }/build.gradle.kts (56%) delete mode 100644 extensions/auth/opa/impl/build.gradle.kts rename extensions/auth/opa/{impl => }/opa-input-schema.json (100%) rename extensions/auth/opa/{tests => }/src/bearerTokenIntTest/java/org/apache/polaris/extension/auth/opa/test/OpaAdminServiceIT.java (100%) rename extensions/auth/opa/{tests => }/src/bearerTokenIntTest/java/org/apache/polaris/extension/auth/opa/test/OpaGenericTableHandlerIT.java (100%) rename extensions/auth/opa/{tests => }/src/bearerTokenIntTest/java/org/apache/polaris/extension/auth/opa/test/OpaIcebergCatalogHandlerIT.java (100%) rename extensions/auth/opa/{tests => }/src/bearerTokenIntTest/java/org/apache/polaris/extension/auth/opa/test/OpaIntegrationTest.java (100%) rename extensions/auth/opa/{tests => }/src/bearerTokenIntTest/java/org/apache/polaris/extension/auth/opa/test/OpaPolicyCatalogHandlerIT.java (100%) rename extensions/auth/opa/{tests => }/src/intTestBase/java/org/apache/polaris/extension/auth/opa/test/OpaIntegrationTestBase.java (100%) rename extensions/auth/opa/{tests => }/src/intTestBase/resources/META-INF/services/org.apache.polaris.service.it.ext.PolarisServerManager (100%) rename extensions/auth/opa/{impl => }/src/jsonSchemaGenerator/java/org/apache/polaris/extension/auth/opa/model/OpaSchemaGenerator.java (100%) rename extensions/auth/opa/{impl => }/src/main/java/org/apache/polaris/extension/auth/opa/OpaAuthorizationConfig.java (100%) rename extensions/auth/opa/{impl => }/src/main/java/org/apache/polaris/extension/auth/opa/OpaHttpClientFactory.java (100%) rename extensions/auth/opa/{impl => }/src/main/java/org/apache/polaris/extension/auth/opa/OpaPolarisAuthorizer.java (100%) rename extensions/auth/opa/{impl => }/src/main/java/org/apache/polaris/extension/auth/opa/OpaPolarisAuthorizerFactory.java (100%) rename extensions/auth/opa/{impl => }/src/main/java/org/apache/polaris/extension/auth/opa/OpaProductionReadinessChecks.java (100%) rename extensions/auth/opa/{impl => }/src/main/java/org/apache/polaris/extension/auth/opa/model/Actor.java (100%) rename extensions/auth/opa/{impl => }/src/main/java/org/apache/polaris/extension/auth/opa/model/Context.java (100%) rename extensions/auth/opa/{impl => }/src/main/java/org/apache/polaris/extension/auth/opa/model/OpaAuthorizationInput.java (100%) rename extensions/auth/opa/{impl => }/src/main/java/org/apache/polaris/extension/auth/opa/model/OpaRequest.java (100%) rename extensions/auth/opa/{impl => }/src/main/java/org/apache/polaris/extension/auth/opa/model/README.md (96%) rename extensions/auth/opa/{impl => }/src/main/java/org/apache/polaris/extension/auth/opa/model/Resource.java (100%) rename extensions/auth/opa/{impl => }/src/main/java/org/apache/polaris/extension/auth/opa/model/ResourceEntity.java (100%) rename extensions/auth/opa/{impl => }/src/main/java/org/apache/polaris/extension/auth/opa/model/package-info.java (100%) rename extensions/auth/opa/{impl => }/src/main/java/org/apache/polaris/extension/auth/opa/token/BearerTokenProvider.java (100%) rename extensions/auth/opa/{impl => }/src/main/java/org/apache/polaris/extension/auth/opa/token/FileBearerTokenProvider.java (100%) rename extensions/auth/opa/{impl => }/src/main/java/org/apache/polaris/extension/auth/opa/token/StaticBearerTokenProvider.java (100%) rename extensions/auth/opa/{tests => }/src/opaFileTokenIntTest/java/org/apache/polaris/extension/auth/opa/test/OpaFileTokenIntegrationTest.java (100%) rename extensions/auth/opa/{tests => }/src/opaStartupAction/java/org/apache/polaris/extension/auth/opa/test/OpaStartupAction.java (100%) rename extensions/auth/opa/{tests => }/src/opaStartupAction/resources/org/apache/polaris/extension/auth/opa/test/Dockerfile-opa-version (100%) rename extensions/auth/opa/{impl => }/src/test/java/org/apache/polaris/extension/auth/opa/OpaHttpClientFactoryTest.java (100%) rename extensions/auth/opa/{impl => }/src/test/java/org/apache/polaris/extension/auth/opa/OpaPolarisAuthorizerFactoryTest.java (100%) rename extensions/auth/opa/{impl => }/src/test/java/org/apache/polaris/extension/auth/opa/OpaPolarisAuthorizerTest.java (100%) rename extensions/auth/opa/{impl => }/src/test/java/org/apache/polaris/extension/auth/opa/token/FileBearerTokenProviderTest.java (100%) rename extensions/auth/opa/{impl => }/src/test/java/org/apache/polaris/extension/auth/opa/token/StaticBearerTokenProviderTest.java (100%) diff --git a/bom/build.gradle.kts b/bom/build.gradle.kts index 44034481a45..0a9530f60cc 100644 --- a/bom/build.gradle.kts +++ b/bom/build.gradle.kts @@ -95,7 +95,6 @@ dependencies { api(project(":polaris-relational-jdbc")) api(project(":polaris-extensions-auth-opa")) - api(project(":polaris-extensions-auth-opa-tests")) api(project(":polaris-extensions-auth-ranger")) api(project(":polaris-extensions-federation-bigquery")) api(project(":polaris-extensions-federation-hadoop")) diff --git a/extensions/auth/opa/impl/SCHEMA.md b/extensions/auth/opa/SCHEMA.md similarity index 98% rename from extensions/auth/opa/impl/SCHEMA.md rename to extensions/auth/opa/SCHEMA.md index ffebd5002bd..7905d6ab981 100644 --- a/extensions/auth/opa/impl/SCHEMA.md +++ b/extensions/auth/opa/SCHEMA.md @@ -96,7 +96,7 @@ Generates the JSON Schema from model classes. ./gradlew :polaris-extensions-auth-opa:generateOpaSchema ``` -**Output**: `extensions/auth/opa/impl/opa-input-schema.json` +**Output**: `extensions/auth/opa/opa-input-schema.json` ### `validateOpaSchema` Validates that committed schema matches the code. diff --git a/extensions/auth/opa/tests/build.gradle.kts b/extensions/auth/opa/build.gradle.kts similarity index 56% rename from extensions/auth/opa/tests/build.gradle.kts rename to extensions/auth/opa/build.gradle.kts index 68c79a16151..5298d287294 100644 --- a/extensions/auth/opa/tests/build.gradle.kts +++ b/extensions/auth/opa/build.gradle.kts @@ -17,13 +17,15 @@ * under the License. */ +import java.io.OutputStream import org.gradle.api.attributes.java.TargetJvmVersion import org.gradle.api.plugins.jvm.JvmTestSuite import org.gradle.language.base.plugins.LifecycleBasePlugin plugins { - id("polaris-client") + id("polaris-server") id("polaris-server-test-runner") + id("org.kordamp.gradle.jandex") } val intTestJvmVersion = 21 @@ -33,6 +35,8 @@ val intTestBase by isCanBeResolved = false } val intTestBaseSources = sourceSets.create("intTestBase") +val jsonSchemaGenerator = sourceSets.create("jsonSchemaGenerator") +val jsonSchemaGeneratorImplementation by configurations.getting val opaStartupAction = sourceSets.create("opaStartupAction") val opaStartupActionCompileOnly by configurations.getting val opaStartupActionImplementation by configurations.getting @@ -48,6 +52,46 @@ listOf( dependencies { polarisServer(project(path = ":polaris-server", configuration = "quarkusRunner")) + implementation(project(":polaris-core")) + implementation(libs.apache.httpclient5) + implementation(platform(libs.jackson.bom)) + implementation("com.fasterxml.jackson.core:jackson-core") + implementation("com.fasterxml.jackson.core:jackson-databind") + implementation(libs.guava) + implementation(libs.slf4j.api) + implementation(libs.auth0.jwt) + implementation(project(":polaris-async-api")) + + jsonSchemaGeneratorImplementation(project(":polaris-extensions-auth-opa")) + jsonSchemaGeneratorImplementation(platform(libs.jackson.bom)) + jsonSchemaGeneratorImplementation("com.fasterxml.jackson.module:jackson-module-jsonSchema") + + // Iceberg dependency for ForbiddenException + implementation(platform(libs.iceberg.bom)) + implementation("org.apache.iceberg:iceberg-api") + + compileOnly(project(":polaris-immutables")) + annotationProcessor(project(":polaris-immutables", configuration = "processor")) + + compileOnly(libs.jspecify) + compileOnly(libs.jakarta.annotation.api) + compileOnly(libs.jakarta.enterprise.cdi.api) + compileOnly(libs.jakarta.inject.api) + compileOnly(libs.smallrye.config.core) + + testCompileOnly(project(":polaris-immutables")) + testAnnotationProcessor(project(":polaris-immutables", configuration = "processor")) + + testImplementation(testFixtures(project(":polaris-core"))) + testImplementation(platform(libs.junit.bom)) + testImplementation("org.junit.jupiter:junit-jupiter") + testImplementation(libs.assertj.core) + testImplementation(libs.mockito.core) + testImplementation(libs.threeten.extra) + testImplementation(testFixtures(project(":polaris-async-api"))) + testImplementation(project(":polaris-async-java")) + testImplementation(project(":polaris-idgen-mocks")) + opaStartupActionCompileOnly("org.apache.polaris.server-test-runner:polaris-server-test-runner") opaStartupActionImplementation(platform(libs.testcontainers.bom)) opaStartupActionImplementation("org.testcontainers:testcontainers") @@ -66,6 +110,102 @@ dependencies { intTestBase("org.apache.iceberg:iceberg-core") } +// Task to generate JSON Schema from model classes +tasks.register("generateOpaSchema") { + group = "documentation" + description = "Generates JSON Schema for OPA authorization input" + + dependsOn(tasks.compileJava, tasks.named("jandex")) + + // Only execute generation if anything changed + outputs.cacheIf { true } + outputs.file("./opa-input-schema.json") + inputs.files(jsonSchemaGenerator.runtimeClasspath).withNormalizer(ClasspathNormalizer::class.java) + + classpath = jsonSchemaGenerator.runtimeClasspath + mainClass.set("org.apache.polaris.extension.auth.opa.model.OpaSchemaGenerator") + args("./opa-input-schema.json") +} + +// Task to validate that the committed schema matches the generated schema +tasks.register("validateOpaSchema") { + group = "verification" + description = "Validates that the committed OPA schema matches the generated schema" + + dependsOn(tasks.compileJava, tasks.named("jandex")) + + val tempSchemaFile = + layout.buildDirectory + .file("opa-schema/opa-input-schema-generated.json") + .get() + .asFile + .relativeTo(projectDir) + val committedSchemaFile = file("${projectDir}/opa-input-schema.json") + val logFile = layout.buildDirectory.file("opa-schema/generator.log") + + // Only execute validation if anything changed + outputs.cacheIf { true } + outputs.file(tempSchemaFile) + inputs.file(committedSchemaFile).withPathSensitivity(PathSensitivity.RELATIVE) + inputs.files(jsonSchemaGenerator.runtimeClasspath).withNormalizer(ClasspathNormalizer::class.java) + + classpath = jsonSchemaGenerator.runtimeClasspath + mainClass.set("org.apache.polaris.extension.auth.opa.model.OpaSchemaGenerator") + args(tempSchemaFile) + isIgnoreExitValue = true + + var outStream: OutputStream? = null + doFirst { + // Ensure temp directory exists + tempSchemaFile.parentFile.mkdirs() + outStream = logFile.get().asFile.outputStream() + standardOutput = outStream + errorOutput = outStream + } + + val prjDir = projectDir + + doLast { + outStream?.close() + + if (executionResult.get().exitValue != 0) { + throw GradleException( + """ + |OPA Schema validation failed! + | + |${logFile.get().asFile.readText()} + """ + .trimMargin() + ) + } + + val generatedContent = prjDir.resolve(tempSchemaFile).readText().trim() + val committedContent = committedSchemaFile.readText().trim() + + if (generatedContent != committedContent) { + throw GradleException( + """ + |OPA Schema validation failed! + | + |The committed opa-input-schema.json does not match the generated schema. + |This means the schema is out of sync with the model classes. + | + |To fix this, run: + | ./gradlew :polaris-extensions-auth-opa:generateOpaSchema + | + |Then commit the updated opa-input-schema.json file. + | + |Committed file: ${committedSchemaFile.absolutePath} + |Generated file: ${tempSchemaFile.absolutePath} + """ + .trimMargin() + ) + } + + logger.info("OPA schema validation passed - schema is up to date") + } +} + fun Test.configureOpaTestTask( buildDir: DirectoryProperty, staticToken: String? = null, @@ -188,4 +328,4 @@ tasks.register("intTest") { dependsOn("bearerTokenIntTest", "opaFileTokenIntTest") } -tasks.named("check") { dependsOn("intTest") } +tasks.named("check") { dependsOn("validateOpaSchema", "intTest") } diff --git a/extensions/auth/opa/impl/build.gradle.kts b/extensions/auth/opa/impl/build.gradle.kts deleted file mode 100644 index fe152e26e11..00000000000 --- a/extensions/auth/opa/impl/build.gradle.kts +++ /dev/null @@ -1,171 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -import java.io.OutputStream - -plugins { - id("polaris-server") - id("org.kordamp.gradle.jandex") -} - -val jsonSchemaGenerator = sourceSets.create("jsonSchemaGenerator") - -dependencies { - implementation(project(":polaris-core")) - implementation(libs.apache.httpclient5) - implementation(platform(libs.jackson.bom)) - implementation("com.fasterxml.jackson.core:jackson-core") - implementation("com.fasterxml.jackson.core:jackson-databind") - implementation(libs.guava) - implementation(libs.slf4j.api) - implementation(libs.auth0.jwt) - implementation(project(":polaris-async-api")) - - add(jsonSchemaGenerator.implementationConfigurationName, project(":polaris-extensions-auth-opa")) - add(jsonSchemaGenerator.implementationConfigurationName, platform(libs.jackson.bom)) - add( - jsonSchemaGenerator.implementationConfigurationName, - "com.fasterxml.jackson.module:jackson-module-jsonSchema", - ) - - // Iceberg dependency for ForbiddenException - implementation(platform(libs.iceberg.bom)) - implementation("org.apache.iceberg:iceberg-api") - - compileOnly(project(":polaris-immutables")) - annotationProcessor(project(":polaris-immutables", configuration = "processor")) - - compileOnly(libs.jspecify) - compileOnly(libs.jakarta.annotation.api) - compileOnly(libs.jakarta.enterprise.cdi.api) - compileOnly(libs.jakarta.inject.api) - compileOnly(libs.smallrye.config.core) - - testCompileOnly(project(":polaris-immutables")) - testAnnotationProcessor(project(":polaris-immutables", configuration = "processor")) - - testImplementation(testFixtures(project(":polaris-core"))) - testImplementation(platform(libs.junit.bom)) - testImplementation("org.junit.jupiter:junit-jupiter") - testImplementation(libs.assertj.core) - testImplementation(libs.mockito.core) - testImplementation(libs.threeten.extra) - testImplementation(testFixtures(project(":polaris-async-api"))) - testImplementation(project(":polaris-async-java")) - testImplementation(project(":polaris-idgen-mocks")) -} - -// Task to generate JSON Schema from model classes -tasks.register("generateOpaSchema") { - group = "documentation" - description = "Generates JSON Schema for OPA authorization input" - - dependsOn(tasks.compileJava, tasks.named("jandex")) - - // Only execute generation if anything changed - outputs.cacheIf { true } - outputs.file("./opa-input-schema.json") - inputs.files(jsonSchemaGenerator.runtimeClasspath).withNormalizer(ClasspathNormalizer::class.java) - - classpath = jsonSchemaGenerator.runtimeClasspath - mainClass.set("org.apache.polaris.extension.auth.opa.model.OpaSchemaGenerator") - args("./opa-input-schema.json") -} - -// Task to validate that the committed schema matches the generated schema -tasks.register("validateOpaSchema") { - group = "verification" - description = "Validates that the committed OPA schema matches the generated schema" - - dependsOn(tasks.compileJava, tasks.named("jandex")) - - val tempSchemaFile = - layout.buildDirectory - .file("opa-schema/opa-input-schema-generated.json") - .get() - .asFile - .relativeTo(projectDir) - val committedSchemaFile = file("${projectDir}/opa-input-schema.json") - val logFile = layout.buildDirectory.file("opa-schema/generator.log") - - // Only execute validation if anything changed - outputs.cacheIf { true } - outputs.file(tempSchemaFile) - inputs.file(committedSchemaFile).withPathSensitivity(PathSensitivity.RELATIVE) - inputs.files(jsonSchemaGenerator.runtimeClasspath).withNormalizer(ClasspathNormalizer::class.java) - - classpath = jsonSchemaGenerator.runtimeClasspath - mainClass.set("org.apache.polaris.extension.auth.opa.model.OpaSchemaGenerator") - args(tempSchemaFile) - isIgnoreExitValue = true - - var outStream: OutputStream? = null - doFirst { - // Ensure temp directory exists - tempSchemaFile.parentFile.mkdirs() - outStream = logFile.get().asFile.outputStream() - standardOutput = outStream - errorOutput = outStream - } - - val prjDir = projectDir - - doLast { - outStream?.close() - - if (executionResult.get().exitValue != 0) { - throw GradleException( - """ - |OPA Schema validation failed! - | - |${logFile.get().asFile.readText()} - """ - .trimMargin() - ) - } - - val generatedContent = prjDir.resolve(tempSchemaFile).readText().trim() - val committedContent = committedSchemaFile.readText().trim() - - if (generatedContent != committedContent) { - throw GradleException( - """ - |OPA Schema validation failed! - | - |The committed opa-input-schema.json does not match the generated schema. - |This means the schema is out of sync with the model classes. - | - |To fix this, run: - | ./gradlew :polaris-extensions-auth-opa:generateOpaSchema - | - |Then commit the updated opa-input-schema.json file. - | - |Committed file: ${committedSchemaFile.absolutePath} - |Generated file: ${tempSchemaFile.absolutePath} - """ - .trimMargin() - ) - } - - logger.info("OPA schema validation passed - schema is up to date") - } -} - -// Add schema validation to the check task -tasks.named("check") { dependsOn("validateOpaSchema") } diff --git a/extensions/auth/opa/impl/opa-input-schema.json b/extensions/auth/opa/opa-input-schema.json similarity index 100% rename from extensions/auth/opa/impl/opa-input-schema.json rename to extensions/auth/opa/opa-input-schema.json diff --git a/extensions/auth/opa/tests/src/bearerTokenIntTest/java/org/apache/polaris/extension/auth/opa/test/OpaAdminServiceIT.java b/extensions/auth/opa/src/bearerTokenIntTest/java/org/apache/polaris/extension/auth/opa/test/OpaAdminServiceIT.java similarity index 100% rename from extensions/auth/opa/tests/src/bearerTokenIntTest/java/org/apache/polaris/extension/auth/opa/test/OpaAdminServiceIT.java rename to extensions/auth/opa/src/bearerTokenIntTest/java/org/apache/polaris/extension/auth/opa/test/OpaAdminServiceIT.java diff --git a/extensions/auth/opa/tests/src/bearerTokenIntTest/java/org/apache/polaris/extension/auth/opa/test/OpaGenericTableHandlerIT.java b/extensions/auth/opa/src/bearerTokenIntTest/java/org/apache/polaris/extension/auth/opa/test/OpaGenericTableHandlerIT.java similarity index 100% rename from extensions/auth/opa/tests/src/bearerTokenIntTest/java/org/apache/polaris/extension/auth/opa/test/OpaGenericTableHandlerIT.java rename to extensions/auth/opa/src/bearerTokenIntTest/java/org/apache/polaris/extension/auth/opa/test/OpaGenericTableHandlerIT.java diff --git a/extensions/auth/opa/tests/src/bearerTokenIntTest/java/org/apache/polaris/extension/auth/opa/test/OpaIcebergCatalogHandlerIT.java b/extensions/auth/opa/src/bearerTokenIntTest/java/org/apache/polaris/extension/auth/opa/test/OpaIcebergCatalogHandlerIT.java similarity index 100% rename from extensions/auth/opa/tests/src/bearerTokenIntTest/java/org/apache/polaris/extension/auth/opa/test/OpaIcebergCatalogHandlerIT.java rename to extensions/auth/opa/src/bearerTokenIntTest/java/org/apache/polaris/extension/auth/opa/test/OpaIcebergCatalogHandlerIT.java diff --git a/extensions/auth/opa/tests/src/bearerTokenIntTest/java/org/apache/polaris/extension/auth/opa/test/OpaIntegrationTest.java b/extensions/auth/opa/src/bearerTokenIntTest/java/org/apache/polaris/extension/auth/opa/test/OpaIntegrationTest.java similarity index 100% rename from extensions/auth/opa/tests/src/bearerTokenIntTest/java/org/apache/polaris/extension/auth/opa/test/OpaIntegrationTest.java rename to extensions/auth/opa/src/bearerTokenIntTest/java/org/apache/polaris/extension/auth/opa/test/OpaIntegrationTest.java diff --git a/extensions/auth/opa/tests/src/bearerTokenIntTest/java/org/apache/polaris/extension/auth/opa/test/OpaPolicyCatalogHandlerIT.java b/extensions/auth/opa/src/bearerTokenIntTest/java/org/apache/polaris/extension/auth/opa/test/OpaPolicyCatalogHandlerIT.java similarity index 100% rename from extensions/auth/opa/tests/src/bearerTokenIntTest/java/org/apache/polaris/extension/auth/opa/test/OpaPolicyCatalogHandlerIT.java rename to extensions/auth/opa/src/bearerTokenIntTest/java/org/apache/polaris/extension/auth/opa/test/OpaPolicyCatalogHandlerIT.java diff --git a/extensions/auth/opa/tests/src/intTestBase/java/org/apache/polaris/extension/auth/opa/test/OpaIntegrationTestBase.java b/extensions/auth/opa/src/intTestBase/java/org/apache/polaris/extension/auth/opa/test/OpaIntegrationTestBase.java similarity index 100% rename from extensions/auth/opa/tests/src/intTestBase/java/org/apache/polaris/extension/auth/opa/test/OpaIntegrationTestBase.java rename to extensions/auth/opa/src/intTestBase/java/org/apache/polaris/extension/auth/opa/test/OpaIntegrationTestBase.java diff --git a/extensions/auth/opa/tests/src/intTestBase/resources/META-INF/services/org.apache.polaris.service.it.ext.PolarisServerManager b/extensions/auth/opa/src/intTestBase/resources/META-INF/services/org.apache.polaris.service.it.ext.PolarisServerManager similarity index 100% rename from extensions/auth/opa/tests/src/intTestBase/resources/META-INF/services/org.apache.polaris.service.it.ext.PolarisServerManager rename to extensions/auth/opa/src/intTestBase/resources/META-INF/services/org.apache.polaris.service.it.ext.PolarisServerManager diff --git a/extensions/auth/opa/impl/src/jsonSchemaGenerator/java/org/apache/polaris/extension/auth/opa/model/OpaSchemaGenerator.java b/extensions/auth/opa/src/jsonSchemaGenerator/java/org/apache/polaris/extension/auth/opa/model/OpaSchemaGenerator.java similarity index 100% rename from extensions/auth/opa/impl/src/jsonSchemaGenerator/java/org/apache/polaris/extension/auth/opa/model/OpaSchemaGenerator.java rename to extensions/auth/opa/src/jsonSchemaGenerator/java/org/apache/polaris/extension/auth/opa/model/OpaSchemaGenerator.java diff --git a/extensions/auth/opa/impl/src/main/java/org/apache/polaris/extension/auth/opa/OpaAuthorizationConfig.java b/extensions/auth/opa/src/main/java/org/apache/polaris/extension/auth/opa/OpaAuthorizationConfig.java similarity index 100% rename from extensions/auth/opa/impl/src/main/java/org/apache/polaris/extension/auth/opa/OpaAuthorizationConfig.java rename to extensions/auth/opa/src/main/java/org/apache/polaris/extension/auth/opa/OpaAuthorizationConfig.java diff --git a/extensions/auth/opa/impl/src/main/java/org/apache/polaris/extension/auth/opa/OpaHttpClientFactory.java b/extensions/auth/opa/src/main/java/org/apache/polaris/extension/auth/opa/OpaHttpClientFactory.java similarity index 100% rename from extensions/auth/opa/impl/src/main/java/org/apache/polaris/extension/auth/opa/OpaHttpClientFactory.java rename to extensions/auth/opa/src/main/java/org/apache/polaris/extension/auth/opa/OpaHttpClientFactory.java diff --git a/extensions/auth/opa/impl/src/main/java/org/apache/polaris/extension/auth/opa/OpaPolarisAuthorizer.java b/extensions/auth/opa/src/main/java/org/apache/polaris/extension/auth/opa/OpaPolarisAuthorizer.java similarity index 100% rename from extensions/auth/opa/impl/src/main/java/org/apache/polaris/extension/auth/opa/OpaPolarisAuthorizer.java rename to extensions/auth/opa/src/main/java/org/apache/polaris/extension/auth/opa/OpaPolarisAuthorizer.java diff --git a/extensions/auth/opa/impl/src/main/java/org/apache/polaris/extension/auth/opa/OpaPolarisAuthorizerFactory.java b/extensions/auth/opa/src/main/java/org/apache/polaris/extension/auth/opa/OpaPolarisAuthorizerFactory.java similarity index 100% rename from extensions/auth/opa/impl/src/main/java/org/apache/polaris/extension/auth/opa/OpaPolarisAuthorizerFactory.java rename to extensions/auth/opa/src/main/java/org/apache/polaris/extension/auth/opa/OpaPolarisAuthorizerFactory.java diff --git a/extensions/auth/opa/impl/src/main/java/org/apache/polaris/extension/auth/opa/OpaProductionReadinessChecks.java b/extensions/auth/opa/src/main/java/org/apache/polaris/extension/auth/opa/OpaProductionReadinessChecks.java similarity index 100% rename from extensions/auth/opa/impl/src/main/java/org/apache/polaris/extension/auth/opa/OpaProductionReadinessChecks.java rename to extensions/auth/opa/src/main/java/org/apache/polaris/extension/auth/opa/OpaProductionReadinessChecks.java diff --git a/extensions/auth/opa/impl/src/main/java/org/apache/polaris/extension/auth/opa/model/Actor.java b/extensions/auth/opa/src/main/java/org/apache/polaris/extension/auth/opa/model/Actor.java similarity index 100% rename from extensions/auth/opa/impl/src/main/java/org/apache/polaris/extension/auth/opa/model/Actor.java rename to extensions/auth/opa/src/main/java/org/apache/polaris/extension/auth/opa/model/Actor.java diff --git a/extensions/auth/opa/impl/src/main/java/org/apache/polaris/extension/auth/opa/model/Context.java b/extensions/auth/opa/src/main/java/org/apache/polaris/extension/auth/opa/model/Context.java similarity index 100% rename from extensions/auth/opa/impl/src/main/java/org/apache/polaris/extension/auth/opa/model/Context.java rename to extensions/auth/opa/src/main/java/org/apache/polaris/extension/auth/opa/model/Context.java diff --git a/extensions/auth/opa/impl/src/main/java/org/apache/polaris/extension/auth/opa/model/OpaAuthorizationInput.java b/extensions/auth/opa/src/main/java/org/apache/polaris/extension/auth/opa/model/OpaAuthorizationInput.java similarity index 100% rename from extensions/auth/opa/impl/src/main/java/org/apache/polaris/extension/auth/opa/model/OpaAuthorizationInput.java rename to extensions/auth/opa/src/main/java/org/apache/polaris/extension/auth/opa/model/OpaAuthorizationInput.java diff --git a/extensions/auth/opa/impl/src/main/java/org/apache/polaris/extension/auth/opa/model/OpaRequest.java b/extensions/auth/opa/src/main/java/org/apache/polaris/extension/auth/opa/model/OpaRequest.java similarity index 100% rename from extensions/auth/opa/impl/src/main/java/org/apache/polaris/extension/auth/opa/model/OpaRequest.java rename to extensions/auth/opa/src/main/java/org/apache/polaris/extension/auth/opa/model/OpaRequest.java diff --git a/extensions/auth/opa/impl/src/main/java/org/apache/polaris/extension/auth/opa/model/README.md b/extensions/auth/opa/src/main/java/org/apache/polaris/extension/auth/opa/model/README.md similarity index 96% rename from extensions/auth/opa/impl/src/main/java/org/apache/polaris/extension/auth/opa/model/README.md rename to extensions/auth/opa/src/main/java/org/apache/polaris/extension/auth/opa/model/README.md index d478e60f8b1..8c3bd5ee9c6 100644 --- a/extensions/auth/opa/impl/src/main/java/org/apache/polaris/extension/auth/opa/model/README.md +++ b/extensions/auth/opa/src/main/java/org/apache/polaris/extension/auth/opa/model/README.md @@ -33,7 +33,7 @@ Run the Gradle task to regenerate the schema: ./gradlew :polaris-extensions-auth-opa:generateOpaSchema ``` -The schema will be generated at: `extensions/auth/opa/impl/opa-input-schema.json` +The schema will be generated at: `extensions/auth/opa/opa-input-schema.json` ## Model Classes diff --git a/extensions/auth/opa/impl/src/main/java/org/apache/polaris/extension/auth/opa/model/Resource.java b/extensions/auth/opa/src/main/java/org/apache/polaris/extension/auth/opa/model/Resource.java similarity index 100% rename from extensions/auth/opa/impl/src/main/java/org/apache/polaris/extension/auth/opa/model/Resource.java rename to extensions/auth/opa/src/main/java/org/apache/polaris/extension/auth/opa/model/Resource.java diff --git a/extensions/auth/opa/impl/src/main/java/org/apache/polaris/extension/auth/opa/model/ResourceEntity.java b/extensions/auth/opa/src/main/java/org/apache/polaris/extension/auth/opa/model/ResourceEntity.java similarity index 100% rename from extensions/auth/opa/impl/src/main/java/org/apache/polaris/extension/auth/opa/model/ResourceEntity.java rename to extensions/auth/opa/src/main/java/org/apache/polaris/extension/auth/opa/model/ResourceEntity.java diff --git a/extensions/auth/opa/impl/src/main/java/org/apache/polaris/extension/auth/opa/model/package-info.java b/extensions/auth/opa/src/main/java/org/apache/polaris/extension/auth/opa/model/package-info.java similarity index 100% rename from extensions/auth/opa/impl/src/main/java/org/apache/polaris/extension/auth/opa/model/package-info.java rename to extensions/auth/opa/src/main/java/org/apache/polaris/extension/auth/opa/model/package-info.java diff --git a/extensions/auth/opa/impl/src/main/java/org/apache/polaris/extension/auth/opa/token/BearerTokenProvider.java b/extensions/auth/opa/src/main/java/org/apache/polaris/extension/auth/opa/token/BearerTokenProvider.java similarity index 100% rename from extensions/auth/opa/impl/src/main/java/org/apache/polaris/extension/auth/opa/token/BearerTokenProvider.java rename to extensions/auth/opa/src/main/java/org/apache/polaris/extension/auth/opa/token/BearerTokenProvider.java diff --git a/extensions/auth/opa/impl/src/main/java/org/apache/polaris/extension/auth/opa/token/FileBearerTokenProvider.java b/extensions/auth/opa/src/main/java/org/apache/polaris/extension/auth/opa/token/FileBearerTokenProvider.java similarity index 100% rename from extensions/auth/opa/impl/src/main/java/org/apache/polaris/extension/auth/opa/token/FileBearerTokenProvider.java rename to extensions/auth/opa/src/main/java/org/apache/polaris/extension/auth/opa/token/FileBearerTokenProvider.java diff --git a/extensions/auth/opa/impl/src/main/java/org/apache/polaris/extension/auth/opa/token/StaticBearerTokenProvider.java b/extensions/auth/opa/src/main/java/org/apache/polaris/extension/auth/opa/token/StaticBearerTokenProvider.java similarity index 100% rename from extensions/auth/opa/impl/src/main/java/org/apache/polaris/extension/auth/opa/token/StaticBearerTokenProvider.java rename to extensions/auth/opa/src/main/java/org/apache/polaris/extension/auth/opa/token/StaticBearerTokenProvider.java diff --git a/extensions/auth/opa/tests/src/opaFileTokenIntTest/java/org/apache/polaris/extension/auth/opa/test/OpaFileTokenIntegrationTest.java b/extensions/auth/opa/src/opaFileTokenIntTest/java/org/apache/polaris/extension/auth/opa/test/OpaFileTokenIntegrationTest.java similarity index 100% rename from extensions/auth/opa/tests/src/opaFileTokenIntTest/java/org/apache/polaris/extension/auth/opa/test/OpaFileTokenIntegrationTest.java rename to extensions/auth/opa/src/opaFileTokenIntTest/java/org/apache/polaris/extension/auth/opa/test/OpaFileTokenIntegrationTest.java diff --git a/extensions/auth/opa/tests/src/opaStartupAction/java/org/apache/polaris/extension/auth/opa/test/OpaStartupAction.java b/extensions/auth/opa/src/opaStartupAction/java/org/apache/polaris/extension/auth/opa/test/OpaStartupAction.java similarity index 100% rename from extensions/auth/opa/tests/src/opaStartupAction/java/org/apache/polaris/extension/auth/opa/test/OpaStartupAction.java rename to extensions/auth/opa/src/opaStartupAction/java/org/apache/polaris/extension/auth/opa/test/OpaStartupAction.java diff --git a/extensions/auth/opa/tests/src/opaStartupAction/resources/org/apache/polaris/extension/auth/opa/test/Dockerfile-opa-version b/extensions/auth/opa/src/opaStartupAction/resources/org/apache/polaris/extension/auth/opa/test/Dockerfile-opa-version similarity index 100% rename from extensions/auth/opa/tests/src/opaStartupAction/resources/org/apache/polaris/extension/auth/opa/test/Dockerfile-opa-version rename to extensions/auth/opa/src/opaStartupAction/resources/org/apache/polaris/extension/auth/opa/test/Dockerfile-opa-version diff --git a/extensions/auth/opa/impl/src/test/java/org/apache/polaris/extension/auth/opa/OpaHttpClientFactoryTest.java b/extensions/auth/opa/src/test/java/org/apache/polaris/extension/auth/opa/OpaHttpClientFactoryTest.java similarity index 100% rename from extensions/auth/opa/impl/src/test/java/org/apache/polaris/extension/auth/opa/OpaHttpClientFactoryTest.java rename to extensions/auth/opa/src/test/java/org/apache/polaris/extension/auth/opa/OpaHttpClientFactoryTest.java diff --git a/extensions/auth/opa/impl/src/test/java/org/apache/polaris/extension/auth/opa/OpaPolarisAuthorizerFactoryTest.java b/extensions/auth/opa/src/test/java/org/apache/polaris/extension/auth/opa/OpaPolarisAuthorizerFactoryTest.java similarity index 100% rename from extensions/auth/opa/impl/src/test/java/org/apache/polaris/extension/auth/opa/OpaPolarisAuthorizerFactoryTest.java rename to extensions/auth/opa/src/test/java/org/apache/polaris/extension/auth/opa/OpaPolarisAuthorizerFactoryTest.java diff --git a/extensions/auth/opa/impl/src/test/java/org/apache/polaris/extension/auth/opa/OpaPolarisAuthorizerTest.java b/extensions/auth/opa/src/test/java/org/apache/polaris/extension/auth/opa/OpaPolarisAuthorizerTest.java similarity index 100% rename from extensions/auth/opa/impl/src/test/java/org/apache/polaris/extension/auth/opa/OpaPolarisAuthorizerTest.java rename to extensions/auth/opa/src/test/java/org/apache/polaris/extension/auth/opa/OpaPolarisAuthorizerTest.java diff --git a/extensions/auth/opa/impl/src/test/java/org/apache/polaris/extension/auth/opa/token/FileBearerTokenProviderTest.java b/extensions/auth/opa/src/test/java/org/apache/polaris/extension/auth/opa/token/FileBearerTokenProviderTest.java similarity index 100% rename from extensions/auth/opa/impl/src/test/java/org/apache/polaris/extension/auth/opa/token/FileBearerTokenProviderTest.java rename to extensions/auth/opa/src/test/java/org/apache/polaris/extension/auth/opa/token/FileBearerTokenProviderTest.java diff --git a/extensions/auth/opa/impl/src/test/java/org/apache/polaris/extension/auth/opa/token/StaticBearerTokenProviderTest.java b/extensions/auth/opa/src/test/java/org/apache/polaris/extension/auth/opa/token/StaticBearerTokenProviderTest.java similarity index 100% rename from extensions/auth/opa/impl/src/test/java/org/apache/polaris/extension/auth/opa/token/StaticBearerTokenProviderTest.java rename to extensions/auth/opa/src/test/java/org/apache/polaris/extension/auth/opa/token/StaticBearerTokenProviderTest.java diff --git a/gradle/projects.main.properties b/gradle/projects.main.properties index 0e37e570d7f..cae568b92e2 100644 --- a/gradle/projects.main.properties +++ b/gradle/projects.main.properties @@ -47,8 +47,7 @@ polaris-misc-types=tools/misc-types polaris-extensions-federation-hadoop=extensions/federation/hadoop polaris-extensions-federation-hive=extensions/federation/hive polaris-extensions-federation-bigquery=extensions/federation/bigquery -polaris-extensions-auth-opa=extensions/auth/opa/impl -polaris-extensions-auth-opa-tests=extensions/auth/opa/tests +polaris-extensions-auth-opa=extensions/auth/opa polaris-extensions-auth-ranger=extensions/auth/ranger polaris-config-docs-annotations=tools/config-docs/annotations From 6de1733a3b9535105aa0bc3a5a653e8d25228b20 Mon Sep 17 00:00:00 2001 From: Robert Stupp Date: Tue, 9 Jun 2026 17:19:33 +0200 Subject: [PATCH 11/22] Migrate runtime-spark-tests to server runner --- runtime/spark-tests/build.gradle.kts | 233 ++++++++++++++---- .../service/spark/it/CatalogFederationIT.java | 23 ++ ...olaris.service.it.ext.PolarisServerManager | 2 +- .../spark/it/HiveCatalogFederationIT.java | 32 +++ .../spark/it/PropertiesRustfsAccess.java | 128 ++++++++++ ...olaris.service.it.ext.PolarisServerManager | 20 ++ .../service/spark/it/CatalogFederationIT.java | 49 ---- .../spark/it/HiveCatalogFederationIT.java | 108 -------- .../polaris/service/spark/it/SparkIT.java | 2 - ...olaris.service.it.ext.PolarisServerManager | 20 ++ .../spark/it/SparkTestsStartupAction.java | 75 ++++++ 11 files changed, 490 insertions(+), 202 deletions(-) create mode 100644 runtime/spark-tests/src/catalogFederationIntTest/java/org/apache/polaris/service/spark/it/CatalogFederationIT.java rename runtime/spark-tests/src/{intTest => catalogFederationIntTest}/resources/META-INF/services/org.apache.polaris.service.it.ext.PolarisServerManager (92%) create mode 100644 runtime/spark-tests/src/hiveFederationIntTest/java/org/apache/polaris/service/spark/it/HiveCatalogFederationIT.java create mode 100644 runtime/spark-tests/src/hiveFederationIntTest/java/org/apache/polaris/service/spark/it/PropertiesRustfsAccess.java create mode 100644 runtime/spark-tests/src/hiveFederationIntTest/resources/META-INF/services/org.apache.polaris.service.it.ext.PolarisServerManager delete mode 100644 runtime/spark-tests/src/intTest/java/org/apache/polaris/service/spark/it/CatalogFederationIT.java delete mode 100644 runtime/spark-tests/src/intTest/java/org/apache/polaris/service/spark/it/HiveCatalogFederationIT.java rename runtime/spark-tests/src/{intTest => sparkIntTest}/java/org/apache/polaris/service/spark/it/SparkIT.java (92%) create mode 100644 runtime/spark-tests/src/sparkIntTest/resources/META-INF/services/org.apache.polaris.service.it.ext.PolarisServerManager create mode 100644 runtime/spark-tests/src/sparkStartupAction/java/org/apache/polaris/service/spark/it/SparkTestsStartupAction.java diff --git a/runtime/spark-tests/build.gradle.kts b/runtime/spark-tests/build.gradle.kts index 089aedb1163..011590f397a 100644 --- a/runtime/spark-tests/build.gradle.kts +++ b/runtime/spark-tests/build.gradle.kts @@ -17,13 +17,33 @@ * under the License. */ +import io.quarkus.gradle.tasks.QuarkusBuild +import org.gradle.api.attributes.java.TargetJvmVersion +import org.gradle.api.plugins.jvm.JvmTestSuite +import org.gradle.language.base.plugins.LifecycleBasePlugin + plugins { alias(libs.plugins.quarkus) id("org.kordamp.gradle.jandex") id("polaris-runtime") + id("polaris-server-test-runner") +} + +val intTestJvmVersion = 21 +val sparkStartupAction = sourceSets.create("sparkStartupAction") +val sparkStartupActionCompileOnly by configurations.getting +val sparkStartupActionImplementation by configurations.getting +val quarkusBuild = tasks.named("quarkusBuild") +val localPolarisServer = files(provider { quarkusBuild.get().fastJar.resolve("quarkus-run.jar") }) + +localPolarisServer.builtBy(quarkusBuild) + +configurations.named(sparkStartupAction.runtimeOnlyConfigurationName) { + extendsFrom(configurations.getByName(sparkStartupAction.implementationConfigurationName)) } dependencies { + polarisServer(localPolarisServer) // must be enforced to get a consistent and validated set of dependencies implementation(enforcedPlatform(libs.quarkus.bom)) { @@ -33,6 +53,7 @@ dependencies { } implementation(project(":polaris-runtime-service")) + runtimeOnly(project(":polaris-extensions-federation-hadoop")) runtimeOnly(project(":polaris-extensions-federation-hive")) { // Brings shaded parquet 1.10 which conflicts with Iceberg's parquet 1.16 in test code. exclude("org.apache.parquet", "parquet-hadoop-bundle") @@ -54,57 +75,185 @@ dependencies { // (the bundle fat-jar provided them); add it explicitly at the BOM-managed version. runtimeOnly("software.amazon.awssdk:s3-transfer-manager") - testImplementation(project(":polaris-tests")) - testImplementation(project(":polaris-rustfs-testcontainer")) - testImplementation(testFixtures(project(":polaris-runtime-service"))) - testImplementation(project(":polaris-runtime-test-common")) - - testImplementation(platform(libs.quarkus.bom)) - testImplementation("io.quarkus:quarkus-junit") - testImplementation("io.quarkus:quarkus-rest-client") - testImplementation("io.quarkus:quarkus-rest-client-jackson") - - testImplementation(platform(libs.awssdk.bom)) - testImplementation("software.amazon.awssdk:glue") - testImplementation("software.amazon.awssdk:kms") - testImplementation("software.amazon.awssdk:dynamodb") - - testImplementation(platform(libs.testcontainers.bom)) - testImplementation("org.testcontainers:testcontainers") - testImplementation(libs.s3mock.testcontainers) - - // Required for Spark integration tests - testImplementation(enforcedPlatform(libs.scala212.lang.library)) - testImplementation(enforcedPlatform(libs.scala212.lang.reflect)) - testImplementation(libs.javax.servlet.api) - testImplementation(libs.antlr4.runtime.spark35) + sparkStartupActionCompileOnly("org.apache.polaris.server-test-runner:polaris-server-test-runner") + sparkStartupActionImplementation(project(":polaris-rustfs-testcontainer")) } -tasks.named("intTest").configure { - if (System.getenv("AWS_REGION") == null) { - environment("AWS_REGION", "us-west-2") +@Suppress("UnstableApiUsage") +fun JvmTestSuite.configureSparkIntegrationDependencies() { + dependencies { + implementation(project(":polaris-tests")) + implementation(testFixtures(project(":polaris-runtime-service"))) + implementation(project(":polaris-runtime-test-common")) + + implementation(platform(libs.awssdk.bom)) + implementation("software.amazon.awssdk:glue") + implementation("software.amazon.awssdk:kms") + implementation("software.amazon.awssdk:dynamodb") + implementation("software.amazon.awssdk:url-connection-client") + + // Required for Spark integration tests + implementation(enforcedPlatform(libs.scala212.lang.library)) + implementation(enforcedPlatform(libs.scala212.lang.reflect)) + implementation(libs.javax.servlet.api) + implementation(libs.antlr4.runtime.spark35) } - // Note: the test secrets are referenced in - // org.apache.polaris.service.it.ServerManager - environment("POLARIS_BOOTSTRAP_CREDENTIALS", "POLARIS,test-admin,test-secret") +} + +fun Test.configureSparkIntegrationTestTask( + suiteName: String, + skipCredentialSubscoping: Boolean, + storageAccessKey: String? = null, + storageSecretKey: String? = null, + withHiveRustfsStartupAction: Boolean = false, +) { + environment("AWS_REGION", providers.environmentVariable("AWS_REGION").getOrElse("us-west-2")) jvmArgs("--add-exports", "java.base/sun.nio.ch=ALL-UNNAMED") // Need to allow a java security manager after Java 21, for Subject.getSubject to work // "getSubject is supported only if a security manager is allowed". systemProperty("java.security.manager", "allow") - // Same issue as above: allow a java security manager after Java 21 - // (this setting is for the application under test, while the setting above is for test code). - systemProperty("quarkus.test.arg-line", "-Djava.security.manager=allow") - val logsDir = project.layout.buildDirectory.get().asFile.resolve("logs") - // delete files from previous runs + + val logsDir = project.layout.buildDirectory.dir("logs/$suiteName") + val hiveRustfsProperties = project.layout.buildDirectory.file("$suiteName/hive-rustfs.properties") + doFirst { - // delete log files written by Polaris - logsDir.deleteRecursively() - // delete quarkus.log file (captured Polaris stdout/stderr) - project.layout.buildDirectory.get().asFile.resolve("quarkus.log").delete() + val logsDirFile = logsDir.get().asFile + logsDirFile.deleteRecursively() + logsDirFile.mkdirs() + } + + if (withHiveRustfsStartupAction) { + systemProperty( + "polaris.spark-tests.hive-rustfs.properties", + hiveRustfsProperties.get().asFile.absolutePath, + ) + } + + withPolarisServer(configurations.polarisServer) { + if (withHiveRustfsStartupAction) { + startupActionClasspath.from(sparkStartupAction.runtimeClasspath) + startupActionClass.set("org.apache.polaris.service.spark.it.SparkTestsStartupAction") + startupActionParameters.put( + "hiveRustfsProperties", + hiveRustfsProperties.get().asFile.absolutePath, + ) + } + + environment.put("AWS_REGION", providers.environmentVariable("AWS_REGION").orElse("us-west-2")) + environment.put( + "AWS_ACCESS_KEY_ID", + providers.environmentVariable("AWS_ACCESS_KEY_ID").orElse("ap1"), + ) + environment.put( + "AWS_SECRET_ACCESS_KEY", + providers.environmentVariable("AWS_SECRET_ACCESS_KEY").orElse("s3cr3t"), + ) + environment.put("AWS_EC2_METADATA_DISABLED", "true") + environment.put("POLARIS_BOOTSTRAP_CREDENTIALS", "POLARIS,test-admin,test-secret") + + systemProperties.putAll( + mapOf( + "quarkus.profile" to "it", + "java.security.manager" to "allow", + "quarkus.log.file.enabled" to "true", + "quarkus.log.file.path" to logsDir.get().asFile.resolve("polaris.log").absolutePath, + "polaris.features.\"SUPPORTED_CATALOG_STORAGE_TYPES\"" to + "[\"FILE\",\"S3\",\"GCS\",\"AZURE\"]", + "polaris.features.\"ALLOW_INSECURE_STORAGE_TYPES\"" to "true", + "polaris.features.\"DROP_WITH_PURGE_ENABLED\"" to "true", + "polaris.features.\"ALLOW_OVERLAPPING_CATALOG_URLS\"" to "true", + "polaris.features.\"SKIP_CREDENTIAL_SUBSCOPING_INDIRECTION\"" to + skipCredentialSubscoping.toString(), + "polaris.features.\"ENABLE_CATALOG_FEDERATION\"" to "true", + "polaris.features.\"SUPPORTED_CATALOG_CONNECTION_TYPES\"" to "[\"ICEBERG_REST\",\"HIVE\"]", + "polaris.features.\"SUPPORTED_EXTERNAL_CATALOG_AUTHENTICATION_TYPES\"" to + "[\"IMPLICIT\",\"OAUTH\"]", + "polaris.features.\"ENABLE_SUB_CATALOG_RBAC_FOR_FEDERATED_CATALOGS\"" to "true", + "polaris.features.\"ALLOW_DROPPING_NON_EMPTY_PASSTHROUGH_FACADE_CATALOG\"" to "true", + "polaris.features.\"ALLOW_EXTERNAL_CATALOG_CREDENTIAL_VENDING\"" to "true", + "polaris.features.\"ALLOW_FEDERATED_CATALOGS_CREDENTIAL_VENDING\"" to "true", + ) + ) + storageAccessKey?.let { systemProperties.put("polaris.storage.aws.access-key", it) } + storageSecretKey?.let { systemProperties.put("polaris.storage.aws.secret-key", it) } } - // This property is not honored in a per-profile application.properties file, - // so we need to set it here. - systemProperty("quarkus.log.file.path", logsDir.resolve("polaris.log").absolutePath) // For Spark integration tests addSparkJvmOptions() } + +@Suppress("UnstableApiUsage") +testing { + suites { + register("sparkIntTest") { + useJUnitJupiter() + configureSparkIntegrationDependencies() + + targets.all { + testTask.configure { + configureSparkIntegrationTestTask( + suiteName = "sparkIntTest", + skipCredentialSubscoping = true, + ) + } + } + } + + register("catalogFederationIntTest") { + useJUnitJupiter() + configureSparkIntegrationDependencies() + + targets.all { + testTask.configure { + configureSparkIntegrationTestTask( + suiteName = "catalogFederationIntTest", + skipCredentialSubscoping = false, + storageAccessKey = "test-ak-123-catalog-federation", + storageSecretKey = "test-sk-123-catalog-federation", + ) + } + } + } + + register("hiveFederationIntTest") { + useJUnitJupiter() + configureSparkIntegrationDependencies() + dependencies { implementation(project(":polaris-rustfs-testcontainer")) } + + targets.all { + testTask.configure { + configureSparkIntegrationTestTask( + suiteName = "hiveFederationIntTest", + skipCredentialSubscoping = false, + storageAccessKey = "test-ak-123-hive-federation", + storageSecretKey = "test-sk-123-hive-federation", + withHiveRustfsStartupAction = true, + ) + } + } + } + } +} + +listOf( + "sparkIntTestCompileClasspath", + "sparkIntTestRuntimeClasspath", + "catalogFederationIntTestCompileClasspath", + "catalogFederationIntTestRuntimeClasspath", + "hiveFederationIntTestCompileClasspath", + "hiveFederationIntTestRuntimeClasspath", + "sparkStartupActionCompileClasspath", + "sparkStartupActionRuntimeClasspath", + ) + .forEach { + configurations.named(it).configure { + attributes.attribute(TargetJvmVersion.TARGET_JVM_VERSION_ATTRIBUTE, intTestJvmVersion) + } + } + +tasks.named("intTest") { + description = "Runs all Spark integration tests." + group = LifecycleBasePlugin.VERIFICATION_GROUP + dependsOn("sparkIntTest", "catalogFederationIntTest", "hiveFederationIntTest") + testClassesDirs = files() + classpath = files() +} diff --git a/runtime/spark-tests/src/catalogFederationIntTest/java/org/apache/polaris/service/spark/it/CatalogFederationIT.java b/runtime/spark-tests/src/catalogFederationIntTest/java/org/apache/polaris/service/spark/it/CatalogFederationIT.java new file mode 100644 index 00000000000..f7002ce08c9 --- /dev/null +++ b/runtime/spark-tests/src/catalogFederationIntTest/java/org/apache/polaris/service/spark/it/CatalogFederationIT.java @@ -0,0 +1,23 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.polaris.service.spark.it; + +import org.apache.polaris.service.it.test.CatalogFederationIntegrationTest; + +public class CatalogFederationIT extends CatalogFederationIntegrationTest {} diff --git a/runtime/spark-tests/src/intTest/resources/META-INF/services/org.apache.polaris.service.it.ext.PolarisServerManager b/runtime/spark-tests/src/catalogFederationIntTest/resources/META-INF/services/org.apache.polaris.service.it.ext.PolarisServerManager similarity index 92% rename from runtime/spark-tests/src/intTest/resources/META-INF/services/org.apache.polaris.service.it.ext.PolarisServerManager rename to runtime/spark-tests/src/catalogFederationIntTest/resources/META-INF/services/org.apache.polaris.service.it.ext.PolarisServerManager index 07ecf4b1fd0..6aa2dac9e0c 100644 --- a/runtime/spark-tests/src/intTest/resources/META-INF/services/org.apache.polaris.service.it.ext.PolarisServerManager +++ b/runtime/spark-tests/src/catalogFederationIntTest/resources/META-INF/services/org.apache.polaris.service.it.ext.PolarisServerManager @@ -17,4 +17,4 @@ # under the License. # -org.apache.polaris.service.it.ServerManager \ No newline at end of file +org.apache.polaris.service.it.ext.ExternalPolarisServerManager diff --git a/runtime/spark-tests/src/hiveFederationIntTest/java/org/apache/polaris/service/spark/it/HiveCatalogFederationIT.java b/runtime/spark-tests/src/hiveFederationIntTest/java/org/apache/polaris/service/spark/it/HiveCatalogFederationIT.java new file mode 100644 index 00000000000..9fc1f1e86a6 --- /dev/null +++ b/runtime/spark-tests/src/hiveFederationIntTest/java/org/apache/polaris/service/spark/it/HiveCatalogFederationIT.java @@ -0,0 +1,32 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.polaris.service.spark.it; + +import java.nio.file.Path; +import org.apache.polaris.service.it.test.HiveCatalogFederationIntegrationTest; +import org.apache.polaris.test.rustfs.RustfsAccess; + +public class HiveCatalogFederationIT extends HiveCatalogFederationIntegrationTest { + + @Override + protected RustfsAccess rustfsAccess() { + return PropertiesRustfsAccess.from( + Path.of(System.getProperty("polaris.spark-tests.hive-rustfs.properties"))); + } +} diff --git a/runtime/spark-tests/src/hiveFederationIntTest/java/org/apache/polaris/service/spark/it/PropertiesRustfsAccess.java b/runtime/spark-tests/src/hiveFederationIntTest/java/org/apache/polaris/service/spark/it/PropertiesRustfsAccess.java new file mode 100644 index 00000000000..d5bbada494b --- /dev/null +++ b/runtime/spark-tests/src/hiveFederationIntTest/java/org/apache/polaris/service/spark/it/PropertiesRustfsAccess.java @@ -0,0 +1,128 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.polaris.service.spark.it; + +import java.io.IOException; +import java.io.InputStream; +import java.net.URI; +import java.nio.file.Path; +import java.util.HashMap; +import java.util.Map; +import java.util.Optional; +import java.util.Properties; +import org.apache.polaris.test.rustfs.RustfsAccess; +import software.amazon.awssdk.auth.credentials.AwsBasicCredentials; +import software.amazon.awssdk.auth.credentials.StaticCredentialsProvider; +import software.amazon.awssdk.http.urlconnection.UrlConnectionHttpClient; +import software.amazon.awssdk.regions.Region; +import software.amazon.awssdk.services.s3.S3Client; + +/** Test-side view of the RustFS container owned by {@link SparkTestsStartupAction}. */ +final class PropertiesRustfsAccess implements RustfsAccess { + private final Properties properties; + + private PropertiesRustfsAccess(Properties properties) { + this.properties = properties; + } + + static PropertiesRustfsAccess from(Path path) { + Properties properties = new Properties(); + try (InputStream input = java.nio.file.Files.newInputStream(path)) { + properties.load(input); + } catch (IOException e) { + throw new IllegalStateException("Failed to load RustFS properties from " + path, e); + } + return new PropertiesRustfsAccess(properties); + } + + @Override + public String hostPort() { + return value("hostPort"); + } + + @Override + public String accessKey() { + return value("accessKey"); + } + + @Override + public String secretKey() { + return value("secretKey"); + } + + @Override + public String bucket() { + return value("bucket"); + } + + @Override + public Optional region() { + return Optional.ofNullable(properties.getProperty("region")); + } + + @Override + public String s3endpoint() { + return value("s3endpoint"); + } + + @Override + public S3Client s3Client() { + return S3Client.builder() + .httpClientBuilder(UrlConnectionHttpClient.builder()) + .endpointOverride(URI.create(s3endpoint())) + .applyMutation(builder -> region().ifPresent(region -> builder.region(Region.of(region)))) + .credentialsProvider( + StaticCredentialsProvider.create(AwsBasicCredentials.create(accessKey(), secretKey()))) + .build(); + } + + @Override + public Map icebergProperties() { + Map props = new HashMap<>(); + props.put("s3.access-key-id", accessKey()); + props.put("s3.secret-access-key", secretKey()); + props.put("s3.endpoint", s3endpoint()); + props.put("http-client.type", "urlconnection"); + region().ifPresent(region -> props.put("client.region", region)); + return props; + } + + @Override + public Map hadoopConfig() { + Map props = new HashMap<>(); + props.put("fs.s3.impl", "org.apache.hadoop.fs.s3a.S3AFileSystem"); + props.put("fs.s3a.access.key", accessKey()); + props.put("fs.s3a.secret.key", secretKey()); + props.put("fs.s3a.endpoint", s3endpoint()); + return props; + } + + @Override + public URI s3BucketUri(String path) { + return URI.create(String.format("s3://%s/", bucket())).resolve(path); + } + + private String value(String name) { + String value = properties.getProperty(name); + if (value == null || value.isBlank()) { + throw new IllegalStateException("Missing RustFS property: " + name); + } + return value; + } +} diff --git a/runtime/spark-tests/src/hiveFederationIntTest/resources/META-INF/services/org.apache.polaris.service.it.ext.PolarisServerManager b/runtime/spark-tests/src/hiveFederationIntTest/resources/META-INF/services/org.apache.polaris.service.it.ext.PolarisServerManager new file mode 100644 index 00000000000..6aa2dac9e0c --- /dev/null +++ b/runtime/spark-tests/src/hiveFederationIntTest/resources/META-INF/services/org.apache.polaris.service.it.ext.PolarisServerManager @@ -0,0 +1,20 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# + +org.apache.polaris.service.it.ext.ExternalPolarisServerManager diff --git a/runtime/spark-tests/src/intTest/java/org/apache/polaris/service/spark/it/CatalogFederationIT.java b/runtime/spark-tests/src/intTest/java/org/apache/polaris/service/spark/it/CatalogFederationIT.java deleted file mode 100644 index a05211b3016..00000000000 --- a/runtime/spark-tests/src/intTest/java/org/apache/polaris/service/spark/it/CatalogFederationIT.java +++ /dev/null @@ -1,49 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ -package org.apache.polaris.service.spark.it; - -import com.google.common.collect.ImmutableMap; -import io.quarkus.test.junit.QuarkusIntegrationTest; -import io.quarkus.test.junit.QuarkusTestProfile; -import io.quarkus.test.junit.TestProfile; -import java.util.Map; -import org.apache.polaris.service.it.test.CatalogFederationIntegrationTest; - -@TestProfile(CatalogFederationIT.CatalogFederationProfile.class) -@QuarkusIntegrationTest -public class CatalogFederationIT extends CatalogFederationIntegrationTest { - - public static class CatalogFederationProfile implements QuarkusTestProfile { - @Override - public Map getConfigOverrides() { - return ImmutableMap.builder() - .put("polaris.features.\"ENABLE_CATALOG_FEDERATION\"", "true") - .put("polaris.features.\"SUPPORTED_CATALOG_CONNECTION_TYPES\"", "[\"ICEBERG_REST\"]") - .put("polaris.features.\"ALLOW_OVERLAPPING_CATALOG_URLS\"", "true") - .put("polaris.features.\"ENABLE_SUB_CATALOG_RBAC_FOR_FEDERATED_CATALOGS\"", "true") - .put("polaris.features.\"ALLOW_DROPPING_NON_EMPTY_PASSTHROUGH_FACADE_CATALOG\"", "true") - .put("polaris.features.\"SKIP_CREDENTIAL_SUBSCOPING_INDIRECTION\"", "false") - .put("polaris.features.\"ALLOW_EXTERNAL_CATALOG_CREDENTIAL_VENDING\"", "true") - .put("polaris.features.\"ALLOW_FEDERATED_CATALOGS_CREDENTIAL_VENDING\"", "true") - .put("polaris.storage.aws.access-key", CatalogFederationIntegrationTest.RUSTFS_ACCESS_KEY) - .put("polaris.storage.aws.secret-key", CatalogFederationIntegrationTest.RUSTFS_SECRET_KEY) - .build(); - } - } -} diff --git a/runtime/spark-tests/src/intTest/java/org/apache/polaris/service/spark/it/HiveCatalogFederationIT.java b/runtime/spark-tests/src/intTest/java/org/apache/polaris/service/spark/it/HiveCatalogFederationIT.java deleted file mode 100644 index ccfdc4249e6..00000000000 --- a/runtime/spark-tests/src/intTest/java/org/apache/polaris/service/spark/it/HiveCatalogFederationIT.java +++ /dev/null @@ -1,108 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ -package org.apache.polaris.service.spark.it; - -import com.google.common.collect.ImmutableMap; -import io.quarkus.test.common.QuarkusTestResource; -import io.quarkus.test.common.QuarkusTestResourceLifecycleManager; -import io.quarkus.test.junit.QuarkusIntegrationTest; -import io.quarkus.test.junit.QuarkusTestProfile; -import io.quarkus.test.junit.TestProfile; -import java.util.Map; -import org.apache.polaris.service.it.test.HiveCatalogFederationIntegrationTest; -import org.apache.polaris.test.rustfs.RustfsAccess; -import org.apache.polaris.test.rustfs.RustfsContainer; - -@TestProfile(HiveCatalogFederationIT.HiveFederationProfile.class) -@QuarkusTestResource(HiveCatalogFederationIT.RustfsResource.class) -@QuarkusIntegrationTest -public class HiveCatalogFederationIT extends HiveCatalogFederationIntegrationTest { - - /** Populated by {@link RustfsResource#inject} before {@code @BeforeAll} runs. */ - private RustfsAccess rustfsAccess; - - @Override - protected RustfsAccess rustfsAccess() { - return rustfsAccess; - } - - public static class HiveFederationProfile implements QuarkusTestProfile { - @Override - public Map getConfigOverrides() { - return ImmutableMap.builder() - .put("polaris.features.\"ENABLE_CATALOG_FEDERATION\"", "true") - .put("polaris.features.\"SUPPORTED_CATALOG_CONNECTION_TYPES\"", "[\"HIVE\"]") - .put( - "polaris.features.\"SUPPORTED_EXTERNAL_CATALOG_AUTHENTICATION_TYPES\"", - "[\"IMPLICIT\"]") - .put("polaris.features.\"ALLOW_OVERLAPPING_CATALOG_URLS\"", "true") - .put("polaris.features.\"ENABLE_SUB_CATALOG_RBAC_FOR_FEDERATED_CATALOGS\"", "true") - .put("polaris.features.\"ALLOW_DROPPING_NON_EMPTY_PASSTHROUGH_FACADE_CATALOG\"", "true") - .put("polaris.features.\"SKIP_CREDENTIAL_SUBSCOPING_INDIRECTION\"", "false") - .put("polaris.features.\"ALLOW_EXTERNAL_CATALOG_CREDENTIAL_VENDING\"", "true") - .put("polaris.features.\"ALLOW_FEDERATED_CATALOGS_CREDENTIAL_VENDING\"", "true") - .put( - "polaris.storage.aws.access-key", - HiveCatalogFederationIntegrationTest.RUSTFS_ACCESS_KEY) - .put( - "polaris.storage.aws.secret-key", - HiveCatalogFederationIntegrationTest.RUSTFS_SECRET_KEY) - .build(); - } - } - - /** - * Starts a RustFS container , exposes its endpoint and credentials, and injects the running - * container into {@link HiveCatalogFederationIT#rustfsAccess} on the test instance. - */ - public static class RustfsResource implements QuarkusTestResourceLifecycleManager { - - private RustfsContainer container; - - @Override - public Map start() { - container = - new RustfsContainer( - null, - HiveCatalogFederationIntegrationTest.RUSTFS_ACCESS_KEY, - HiveCatalogFederationIntegrationTest.RUSTFS_SECRET_KEY, - null, - null) - .withStartupAttempts(5); - container.start(); - return ImmutableMap.of( - "polaris.s3.endpoint", container.s3endpoint(), - "polaris.s3.access-key", container.accessKey(), - "polaris.s3.secret-key", container.secretKey()); - } - - @Override - public void inject(TestInjector testInjector) { - testInjector.injectIntoFields(container, new TestInjector.MatchesType(RustfsAccess.class)); - } - - @Override - public void stop() { - if (container != null) { - container.close(); - container = null; - } - } - } -} diff --git a/runtime/spark-tests/src/intTest/java/org/apache/polaris/service/spark/it/SparkIT.java b/runtime/spark-tests/src/sparkIntTest/java/org/apache/polaris/service/spark/it/SparkIT.java similarity index 92% rename from runtime/spark-tests/src/intTest/java/org/apache/polaris/service/spark/it/SparkIT.java rename to runtime/spark-tests/src/sparkIntTest/java/org/apache/polaris/service/spark/it/SparkIT.java index f82140e9c1c..4b9a21a663f 100644 --- a/runtime/spark-tests/src/intTest/java/org/apache/polaris/service/spark/it/SparkIT.java +++ b/runtime/spark-tests/src/sparkIntTest/java/org/apache/polaris/service/spark/it/SparkIT.java @@ -18,8 +18,6 @@ */ package org.apache.polaris.service.spark.it; -import io.quarkus.test.junit.QuarkusIntegrationTest; import org.apache.polaris.service.it.test.PolarisSparkIntegrationTest; -@QuarkusIntegrationTest public class SparkIT extends PolarisSparkIntegrationTest {} diff --git a/runtime/spark-tests/src/sparkIntTest/resources/META-INF/services/org.apache.polaris.service.it.ext.PolarisServerManager b/runtime/spark-tests/src/sparkIntTest/resources/META-INF/services/org.apache.polaris.service.it.ext.PolarisServerManager new file mode 100644 index 00000000000..6aa2dac9e0c --- /dev/null +++ b/runtime/spark-tests/src/sparkIntTest/resources/META-INF/services/org.apache.polaris.service.it.ext.PolarisServerManager @@ -0,0 +1,20 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# + +org.apache.polaris.service.it.ext.ExternalPolarisServerManager diff --git a/runtime/spark-tests/src/sparkStartupAction/java/org/apache/polaris/service/spark/it/SparkTestsStartupAction.java b/runtime/spark-tests/src/sparkStartupAction/java/org/apache/polaris/service/spark/it/SparkTestsStartupAction.java new file mode 100644 index 00000000000..df8b75fd4e6 --- /dev/null +++ b/runtime/spark-tests/src/sparkStartupAction/java/org/apache/polaris/service/spark/it/SparkTestsStartupAction.java @@ -0,0 +1,75 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.polaris.service.spark.it; + +import java.io.IOException; +import java.io.OutputStream; +import java.nio.file.Files; +import java.nio.file.Path; +import java.util.Properties; +import org.apache.polaris.server.test.runner.spi.PolarisServerStartupAction; +import org.apache.polaris.server.test.runner.spi.PolarisServerStartupContext; +import org.apache.polaris.test.rustfs.RustfsContainer; + +/** Starts services required before the custom Spark-test Polaris server process starts. */ +public class SparkTestsStartupAction implements PolarisServerStartupAction { + private static final String RUSTFS_ACCESS_KEY = "test-ak-123-hive-federation"; + private static final String RUSTFS_SECRET_KEY = "test-sk-123-hive-federation"; + + private RustfsContainer hiveRustfs; + + @Override + public void start(PolarisServerStartupContext context) throws IOException { + hiveRustfs = + new RustfsContainer(null, RUSTFS_ACCESS_KEY, RUSTFS_SECRET_KEY, null, "us-west-2") + .withStartupAttempts(5); + hiveRustfs.start(); + + context.getSystemProperties().put("polaris.s3.endpoint", hiveRustfs.s3endpoint()); + context.getSystemProperties().put("polaris.s3.access-key", hiveRustfs.accessKey()); + context.getSystemProperties().put("polaris.s3.secret-key", hiveRustfs.secretKey()); + + String propertiesFile = context.getParameters().get("hiveRustfsProperties"); + if (propertiesFile != null) { + writeHiveRustfsProperties(Path.of(propertiesFile)); + } + } + + private void writeHiveRustfsProperties(Path path) throws IOException { + Files.createDirectories(path.getParent()); + Properties properties = new Properties(); + properties.setProperty("hostPort", hiveRustfs.hostPort()); + properties.setProperty("accessKey", hiveRustfs.accessKey()); + properties.setProperty("secretKey", hiveRustfs.secretKey()); + properties.setProperty("bucket", hiveRustfs.bucket()); + properties.setProperty("s3endpoint", hiveRustfs.s3endpoint()); + hiveRustfs.region().ifPresent(region -> properties.setProperty("region", region)); + try (OutputStream output = Files.newOutputStream(path)) { + properties.store(output, "RustFS instance started by SparkTestsStartupAction"); + } + } + + @Override + public void close() { + if (hiveRustfs != null) { + hiveRustfs.close(); + hiveRustfs = null; + } + } +} From 80a251bfb2361bd534cf83dc8dbde79a77a246bb Mon Sep 17 00:00:00 2001 From: Robert Stupp Date: Fri, 12 Jun 2026 17:22:37 +0200 Subject: [PATCH 12/22] runtime-service/tests: unify Quarkus test profiles This change moves a bunch of Quarkus test profiles into the shared `Profiles` class. With that, exact duplicate profiles have been merged into one. The Iceberg catalog profiles have been merged. Each distinct Quarkus test profile requires the testing infrastructure to perform an expensive, time-consuming augmentation pass, hence this effort to unify test profiles. --- .../org/apache/polaris/service/Profiles.java | 245 ++++++++++++++++++ .../admin/PolarisAdminServiceAuthzTest.java | 3 +- .../service/admin/PolarisAuthzTestBase.java | 24 -- .../polaris/service/catalog/Profiles.java | 61 ----- ...isGenericTableCatalogHandlerAuthzTest.java | 3 +- ...arisGenericTableCatalogNoSqlInMemTest.java | 2 +- ...arisGenericTableCatalogRelationalTest.java | 2 +- .../AbstractLocalIcebergCatalogTest.java | 15 -- .../AbstractLocalIcebergCatalogViewTest.java | 14 - .../IcebergCatalogHandlerAuthzTest.java | 4 +- ...CatalogHandlerFineGrainedDisabledTest.java | 3 +- .../IcebergCatalogHandlerNoSqlAuthzTest.java | 18 +- .../LocalIcebergCatalogNoSqlInMemTest.java | 18 +- .../LocalIcebergCatalogRelationalTest.java | 3 +- ...LocalIcebergViewCatalogNoSqlInMemTest.java | 17 +- ...LocalIcebergViewCatalogRelationalTest.java | 3 +- .../policy/PolicyCatalogHandlerAuthzTest.java | 3 +- .../policy/PolicyCatalogNoSqlInMemTest.java | 2 +- .../policy/PolicyCatalogRelationalTest.java | 2 +- ...enceDistCacheInvalidationsIntegration.java | 7 +- ...arisEventListenerOldConfigurationTest.java | 15 +- .../events/PolarisEventListenersTest.java | 32 +-- ...moryBufferEventListenerBufferSizeTest.java | 18 +- ...moryBufferEventListenerBufferTimeTest.java | 18 +- ...oryBufferEventListenerIntegrationTest.java | 26 +- .../InMemoryBufferEventListenerTestBase.java | 20 -- .../RealmIdTagDisabledMetricsTest.java | 17 +- .../metrics/RealmIdTagEnabledMetricsTest.java | 26 +- .../UserPrincipalTagDisabledMetricsTest.java | 17 +- .../UserPrincipalTagEnabledMetricsTest.java | 24 +- 30 files changed, 293 insertions(+), 369 deletions(-) create mode 100644 runtime/service/src/test/java/org/apache/polaris/service/Profiles.java delete mode 100644 runtime/service/src/test/java/org/apache/polaris/service/catalog/Profiles.java diff --git a/runtime/service/src/test/java/org/apache/polaris/service/Profiles.java b/runtime/service/src/test/java/org/apache/polaris/service/Profiles.java new file mode 100644 index 00000000000..ef1d6cab2f2 --- /dev/null +++ b/runtime/service/src/test/java/org/apache/polaris/service/Profiles.java @@ -0,0 +1,245 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.polaris.service; + +import com.google.common.collect.ImmutableMap; +import io.quarkus.test.junit.QuarkusTestProfile; +import java.util.Map; +import java.util.Set; +import org.apache.polaris.service.admin.PolarisAuthzTestBase; +import org.apache.polaris.test.commons.NoSqlInMemoryProfile; + +public final class Profiles { + + private Profiles() {} + + public static final Map DEFAULT_PROFILE_CONFIG_OVERRIDES = + Map.of( + "polaris.features.\"ALLOW_SPECIFYING_FILE_IO_IMPL\"", + "true", + "polaris.features.\"ALLOW_INSECURE_STORAGE_TYPES\"", + "true", + "polaris.features.\"SUPPORTED_CATALOG_STORAGE_TYPES\"", + "[\"FILE\",\"S3\"]", + "polaris.event-listener.type", + "test", + "polaris.readiness.ignore-severe-issues", + "true"); + + public static class DefaultProfile implements QuarkusTestProfile { + @Override + public Map getConfigOverrides() { + return DEFAULT_PROFILE_CONFIG_OVERRIDES; + } + } + + public static class DefaultNoSqlProfile extends DefaultProfile { + @Override + public Map getConfigOverrides() { + return ImmutableMap.builder() + .putAll(super.getConfigOverrides()) + .putAll(NoSqlInMemoryProfile.NOSQL_PERSISTENCE) + .build(); + } + } + + public static class DefaultIcebergCatalogProfile extends DefaultProfile { + @Override + public Map getConfigOverrides() { + return ImmutableMap.builder() + .putAll(super.getConfigOverrides()) + .put("polaris.features.\"ALLOW_TABLE_LOCATION_OVERLAP\"", "true") + .put("polaris.features.\"LIST_PAGINATION_ENABLED\"", "true") + .put("polaris.features.\"ALLOW_WILDCARD_LOCATION\"", "true") + .put("polaris.features.\"SKIP_CREDENTIAL_SUBSCOPING_INDIRECTION\"", "true") + .put("polaris.behavior-changes.\"ALLOW_NAMESPACE_CUSTOM_LOCATION\"", "true") + .put("polaris.test.rootAugmentor.enabled", "true") + .build(); + } + } + + public static class NoSqlIcebergCatalogProfile extends DefaultIcebergCatalogProfile { + @Override + public Map getConfigOverrides() { + return ImmutableMap.builder() + .putAll(super.getConfigOverrides()) + .putAll(NoSqlInMemoryProfile.NOSQL_PERSISTENCE) + .build(); + } + } + + public static class PolarisAuthzBaseProfile extends DefaultProfile { + @Override + public Set> getEnabledAlternatives() { + return Set.of(PolarisAuthzTestBase.TestPolarisLocalCatalogFactory.class); + } + + @Override + public Map getConfigOverrides() { + return ImmutableMap.builder() + .putAll(super.getConfigOverrides()) + .put("polaris.features.\"ALLOW_EXTERNAL_METADATA_FILE_LOCATION\"", "true") + .put("polaris.features.\"ENABLE_GENERIC_TABLES\"", "true") + .put( + "polaris.features.\"ENFORCE_PRINCIPAL_CREDENTIAL_ROTATION_REQUIRED_CHECKING\"", + "true") + .put("polaris.features.\"DROP_WITH_PURGE_ENABLED\"", "true") + .put("polaris.behavior-changes.\"ALLOW_NAMESPACE_CUSTOM_LOCATION\"", "true") + .put("polaris.features.\"ENABLE_CATALOG_FEDERATION\"", "true") + .build(); + } + } + + public static class IcebergCatalogHandlerNoSqlAuthzProfile extends PolarisAuthzBaseProfile { + @Override + public Map getConfigOverrides() { + return ImmutableMap.builder() + .putAll(super.getConfigOverrides()) + .putAll(NoSqlInMemoryProfile.NOSQL_PERSISTENCE) + .build(); + } + } + + static final Map IN_MEMORY_BUFFER_EVENT_LISTENER_BASE_CONFIG = + ImmutableMap.builder() + .put("polaris.realm-context.realms", "test1,test2") + .put("polaris.persistence.type", "relational-jdbc") + .put("polaris.persistence.auto-bootstrap-types", "relational-jdbc") + .put("quarkus.datasource.db-kind", "h2") + .put( + "quarkus.datasource.jdbc.url", + "jdbc:h2:mem:test;DB_CLOSE_DELAY=-1;MODE=PostgreSQL;DATABASE_TO_LOWER=TRUE") + .put("polaris.event-listener.type", "persistence-in-memory-buffer") + .put( + "quarkus.fault-tolerance.\"org.apache.polaris.service.events.listeners.inmemory.InMemoryBufferEventListener/flush\".retry.max-retries", + "1") + .put( + "quarkus.fault-tolerance.\"org.apache.polaris.service.events.listeners.inmemory.InMemoryBufferEventListener/flush\".retry.delay", + "10") + .build(); + + public static class InMemoryBufferEventListenerBufferSizeProfile implements QuarkusTestProfile { + @Override + public Map getConfigOverrides() { + return ImmutableMap.builder() + .putAll(IN_MEMORY_BUFFER_EVENT_LISTENER_BASE_CONFIG) + .put("polaris.event-listener.persistence-in-memory-buffer.buffer-time", "60s") + .put("polaris.event-listener.persistence-in-memory-buffer.max-buffer-size", "10") + .build(); + } + } + + public static class InMemoryBufferEventListenerBufferTimeProfile implements QuarkusTestProfile { + @Override + public Map getConfigOverrides() { + return ImmutableMap.builder() + .putAll(IN_MEMORY_BUFFER_EVENT_LISTENER_BASE_CONFIG) + .put("polaris.event-listener.persistence-in-memory-buffer.buffer-time", "100ms") + .put("polaris.event-listener.persistence-in-memory-buffer.max-buffer-size", "1000") + .build(); + } + } + + public static class RealmIdTagEnabledMetricsProfile implements QuarkusTestProfile { + @Override + public Map getConfigOverrides() { + return Map.of( + "polaris.metrics.tags.environment", + "prod", + "polaris.realm-context.type", + "test", + "polaris.metrics.realm-id-tag.enable-in-api-metrics", + "true", + "polaris.metrics.realm-id-tag.enable-in-http-metrics", + "true", + "polaris.metrics.user-principal-tag.enable-in-api-metrics", + "false"); + } + } + + public static class UserPrincipalTagEnabledMetricsProfile implements QuarkusTestProfile { + @Override + public Map getConfigOverrides() { + return Map.of( + "polaris.metrics.tags.environment", + "prod", + "polaris.metrics.user-principal-tag.enable-in-api-metrics", + "true", + "polaris.metrics.realm-id-tag.enable-in-api-metrics", + "false", + "polaris.metrics.realm-id-tag.enable-in-http-metrics", + "false"); + } + } + + public static class TagsDisabledMetricsProfile implements QuarkusTestProfile { + @Override + public Map getConfigOverrides() { + return Map.of( + "polaris.metrics.tags.environment", "prod", "polaris.realm-context.type", "test"); + } + } + + public static class PolarisEventListenersTestProfile implements QuarkusTestProfile { + @Override + public Map getConfigOverrides() { + return ImmutableMap.builder() + .put( + "polaris.event-listener.types", + "after-send-listener,before-send-listener,consume-all-listener,consume-all-listener-2,consume-only-catalog-listener,consume-catalog-and-after-notification-listener") + .put( + "polaris.event-listener.after-send-listener.enabled-event-types", + "AFTER_SEND_NOTIFICATION") + .put( + "polaris.event-listener.before-send-listener.enabled-event-types", + "BEFORE_SEND_NOTIFICATION") + .put( + "polaris.event-listener.consume-only-catalog-listener.enabled-event-categories", + "CATALOG") + .put( + "polaris.event-listener.consume-catalog-and-after-notification-listener.enabled-event-categories", + "CATALOG") + .put( + "polaris.event-listener.consume-catalog-and-after-notification-listener.enabled-event-types", + "AFTER_SEND_NOTIFICATION") + .build(); + } + } + + public static class InMemoryBufferEventListenerIntegrationProfile implements QuarkusTestProfile { + @Override + public Map getConfigOverrides() { + return ImmutableMap.builder() + .put("polaris.persistence.type", "relational-jdbc") + .put("polaris.persistence.auto-bootstrap-types", "relational-jdbc") + .put("quarkus.datasource.db-kind", "h2") + .put("quarkus.otel.sdk.disabled", "false") + .put( + "quarkus.datasource.jdbc.url", + "jdbc:h2:mem:test;DB_CLOSE_DELAY=-1;MODE=PostgreSQL;DATABASE_TO_LOWER=TRUE") + .put("polaris.event-listener.type", "persistence-in-memory-buffer") + .put("polaris.event-listener.persistence-in-memory-buffer.buffer-time", "100ms") + .put("polaris.features.\"ALLOW_INSECURE_STORAGE_TYPES\"", "true") + .put("polaris.features.\"SUPPORTED_CATALOG_STORAGE_TYPES\"", "[\"FILE\",\"S3\"]") + .put("polaris.readiness.ignore-severe-issues", "true") + .build(); + } + } +} diff --git a/runtime/service/src/test/java/org/apache/polaris/service/admin/PolarisAdminServiceAuthzTest.java b/runtime/service/src/test/java/org/apache/polaris/service/admin/PolarisAdminServiceAuthzTest.java index eadd33aff7d..77866abbc9c 100644 --- a/runtime/service/src/test/java/org/apache/polaris/service/admin/PolarisAdminServiceAuthzTest.java +++ b/runtime/service/src/test/java/org/apache/polaris/service/admin/PolarisAdminServiceAuthzTest.java @@ -34,11 +34,12 @@ import org.apache.polaris.core.entity.PolarisPrivilege; import org.apache.polaris.core.entity.PrincipalEntity; import org.apache.polaris.core.entity.PrincipalRoleEntity; +import org.apache.polaris.service.Profiles; import org.junit.jupiter.api.DynamicNode; import org.junit.jupiter.api.TestFactory; @QuarkusTest -@TestProfile(PolarisAuthzTestBase.Profile.class) +@TestProfile(Profiles.PolarisAuthzBaseProfile.class) public class PolarisAdminServiceAuthzTest extends PolarisAuthzTestBase { private PolarisAdminService newTestAdminService() { return newTestAdminService(Set.of()); diff --git a/runtime/service/src/test/java/org/apache/polaris/service/admin/PolarisAuthzTestBase.java b/runtime/service/src/test/java/org/apache/polaris/service/admin/PolarisAuthzTestBase.java index 23dd67f428e..5a5a69015ee 100644 --- a/runtime/service/src/test/java/org/apache/polaris/service/admin/PolarisAuthzTestBase.java +++ b/runtime/service/src/test/java/org/apache/polaris/service/admin/PolarisAuthzTestBase.java @@ -74,7 +74,6 @@ import org.apache.polaris.core.secrets.UserSecretsManager; import org.apache.polaris.core.storage.cache.StorageCredentialCache; import org.apache.polaris.service.catalog.PolarisPassthroughResolutionView; -import org.apache.polaris.service.catalog.Profiles; import org.apache.polaris.service.catalog.generic.PolarisGenericTableCatalog; import org.apache.polaris.service.catalog.iceberg.LocalIcebergCatalog; import org.apache.polaris.service.catalog.io.FileIOFactory; @@ -100,29 +99,6 @@ /** Base class for shared test setup logic used by various Polaris authz-related tests. */ public abstract class PolarisAuthzTestBase { - public static class Profile extends Profiles.DefaultProfile { - - @Override - public Set> getEnabledAlternatives() { - return Set.of(TestPolarisLocalCatalogFactory.class); - } - - @Override - public Map getConfigOverrides() { - return ImmutableMap.builder() - .putAll(super.getConfigOverrides()) - .put("polaris.features.\"ALLOW_EXTERNAL_METADATA_FILE_LOCATION\"", "true") - .put("polaris.features.\"ENABLE_GENERIC_TABLES\"", "true") - .put( - "polaris.features.\"ENFORCE_PRINCIPAL_CREDENTIAL_ROTATION_REQUIRED_CHECKING\"", - "true") - .put("polaris.features.\"DROP_WITH_PURGE_ENABLED\"", "true") - .put("polaris.behavior-changes.\"ALLOW_NAMESPACE_CUSTOM_LOCATION\"", "true") - .put("polaris.features.\"ENABLE_CATALOG_FEDERATION\"", "true") - .build(); - } - } - protected static final String CATALOG_NAME = "polaris-catalog"; protected static final String FEDERATED_CATALOG_NAME = "federated-polaris-catalog"; protected static final String PRINCIPAL_NAME = "snowman"; diff --git a/runtime/service/src/test/java/org/apache/polaris/service/catalog/Profiles.java b/runtime/service/src/test/java/org/apache/polaris/service/catalog/Profiles.java deleted file mode 100644 index 9c5411037dd..00000000000 --- a/runtime/service/src/test/java/org/apache/polaris/service/catalog/Profiles.java +++ /dev/null @@ -1,61 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -package org.apache.polaris.service.catalog; - -import com.google.common.collect.ImmutableMap; -import io.quarkus.test.junit.QuarkusTestProfile; -import java.util.Map; - -public final class Profiles { - private Profiles() {} - - public static final Map NOSQL_IN_MEM = - ImmutableMap.builder() - .put("polaris.persistence.type", "nosql") - .put("polaris.persistence.nosql.backend", "InMemory") - .build(); - - public static class DefaultProfile implements QuarkusTestProfile { - @Override - public Map getConfigOverrides() { - return Map.of( - "polaris.features.\"ALLOW_SPECIFYING_FILE_IO_IMPL\"", - "true", - "polaris.features.\"ALLOW_INSECURE_STORAGE_TYPES\"", - "true", - "polaris.features.\"SUPPORTED_CATALOG_STORAGE_TYPES\"", - "[\"FILE\",\"S3\"]", - "polaris.event-listener.type", - "test", - "polaris.readiness.ignore-severe-issues", - "true"); - } - } - - public static class DefaultNoSqlProfile extends DefaultProfile { - @Override - public Map getConfigOverrides() { - return ImmutableMap.builder() - .putAll(super.getConfigOverrides()) - .putAll(NOSQL_IN_MEM) - .build(); - } - } -} diff --git a/runtime/service/src/test/java/org/apache/polaris/service/catalog/generic/PolarisGenericTableCatalogHandlerAuthzTest.java b/runtime/service/src/test/java/org/apache/polaris/service/catalog/generic/PolarisGenericTableCatalogHandlerAuthzTest.java index 0dceebe8c2a..c68d1b1765d 100644 --- a/runtime/service/src/test/java/org/apache/polaris/service/catalog/generic/PolarisGenericTableCatalogHandlerAuthzTest.java +++ b/runtime/service/src/test/java/org/apache/polaris/service/catalog/generic/PolarisGenericTableCatalogHandlerAuthzTest.java @@ -27,12 +27,13 @@ import org.apache.iceberg.catalog.TableIdentifier; import org.apache.polaris.core.auth.PolarisPrincipal; import org.apache.polaris.core.entity.PolarisPrivilege; +import org.apache.polaris.service.Profiles; import org.apache.polaris.service.admin.PolarisAuthzTestBase; import org.junit.jupiter.api.DynamicNode; import org.junit.jupiter.api.TestFactory; @QuarkusTest -@TestProfile(PolarisAuthzTestBase.Profile.class) +@TestProfile(Profiles.PolarisAuthzBaseProfile.class) public class PolarisGenericTableCatalogHandlerAuthzTest extends PolarisAuthzTestBase { @Inject GenericTableCatalogHandlerFactory genericTableCatalogHandlerFactory; diff --git a/runtime/service/src/test/java/org/apache/polaris/service/catalog/generic/PolarisGenericTableCatalogNoSqlInMemTest.java b/runtime/service/src/test/java/org/apache/polaris/service/catalog/generic/PolarisGenericTableCatalogNoSqlInMemTest.java index a7caa216f4a..ecc6b8a717d 100644 --- a/runtime/service/src/test/java/org/apache/polaris/service/catalog/generic/PolarisGenericTableCatalogNoSqlInMemTest.java +++ b/runtime/service/src/test/java/org/apache/polaris/service/catalog/generic/PolarisGenericTableCatalogNoSqlInMemTest.java @@ -24,7 +24,7 @@ import java.util.List; import org.apache.polaris.core.persistence.MetaStoreManagerFactory; import org.apache.polaris.core.persistence.bootstrap.RootCredentialsSet; -import org.apache.polaris.service.catalog.Profiles; +import org.apache.polaris.service.Profiles; @QuarkusTest @TestProfile(Profiles.DefaultNoSqlProfile.class) diff --git a/runtime/service/src/test/java/org/apache/polaris/service/catalog/generic/PolarisGenericTableCatalogRelationalTest.java b/runtime/service/src/test/java/org/apache/polaris/service/catalog/generic/PolarisGenericTableCatalogRelationalTest.java index 54aef1843a9..e69b9f125f4 100644 --- a/runtime/service/src/test/java/org/apache/polaris/service/catalog/generic/PolarisGenericTableCatalogRelationalTest.java +++ b/runtime/service/src/test/java/org/apache/polaris/service/catalog/generic/PolarisGenericTableCatalogRelationalTest.java @@ -20,7 +20,7 @@ import io.quarkus.test.junit.QuarkusTest; import io.quarkus.test.junit.TestProfile; -import org.apache.polaris.service.catalog.Profiles; +import org.apache.polaris.service.Profiles; @QuarkusTest @TestProfile(Profiles.DefaultProfile.class) diff --git a/runtime/service/src/test/java/org/apache/polaris/service/catalog/iceberg/AbstractLocalIcebergCatalogTest.java b/runtime/service/src/test/java/org/apache/polaris/service/catalog/iceberg/AbstractLocalIcebergCatalogTest.java index 70f59eec0ce..a7f36a943eb 100644 --- a/runtime/service/src/test/java/org/apache/polaris/service/catalog/iceberg/AbstractLocalIcebergCatalogTest.java +++ b/runtime/service/src/test/java/org/apache/polaris/service/catalog/iceberg/AbstractLocalIcebergCatalogTest.java @@ -135,7 +135,6 @@ import org.apache.polaris.core.storage.cache.StorageCredentialCache; import org.apache.polaris.service.admin.PolarisAdminService; import org.apache.polaris.service.catalog.PolarisPassthroughResolutionView; -import org.apache.polaris.service.catalog.Profiles; import org.apache.polaris.service.catalog.io.ExceptionMappingFileIO; import org.apache.polaris.service.catalog.io.FileIOFactory; import org.apache.polaris.service.catalog.io.MeasuredFileIOFactory; @@ -201,20 +200,6 @@ public abstract class AbstractLocalIcebergCatalogTest extends CatalogTests getConfigOverrides() { - return ImmutableMap.builder() - .putAll(super.getConfigOverrides()) - .put("polaris.features.\"ALLOW_TABLE_LOCATION_OVERLAP\"", "true") - .put("polaris.features.\"LIST_PAGINATION_ENABLED\"", "true") - .put("polaris.behavior-changes.\"ALLOW_NAMESPACE_CUSTOM_LOCATION\"", "true") - .put("polaris.test.rootAugmentor.enabled", "true") - .put("polaris.event-listener.types", "test") - .build(); - } - } - private static final String VIEW_QUERY = "select * from ns1.layer1_table"; public static final String CATALOG_NAME = "polaris-catalog"; diff --git a/runtime/service/src/test/java/org/apache/polaris/service/catalog/iceberg/AbstractLocalIcebergCatalogViewTest.java b/runtime/service/src/test/java/org/apache/polaris/service/catalog/iceberg/AbstractLocalIcebergCatalogViewTest.java index aef84fc8f95..15c417d33a9 100644 --- a/runtime/service/src/test/java/org/apache/polaris/service/catalog/iceberg/AbstractLocalIcebergCatalogViewTest.java +++ b/runtime/service/src/test/java/org/apache/polaris/service/catalog/iceberg/AbstractLocalIcebergCatalogViewTest.java @@ -55,7 +55,6 @@ import org.apache.polaris.core.storage.cache.StorageCredentialCache; import org.apache.polaris.service.admin.PolarisAdminService; import org.apache.polaris.service.catalog.PolarisPassthroughResolutionView; -import org.apache.polaris.service.catalog.Profiles; import org.apache.polaris.service.catalog.io.FileIOFactory; import org.apache.polaris.service.catalog.io.StorageAccessConfigProvider; import org.apache.polaris.service.config.ReservedProperties; @@ -85,19 +84,6 @@ public abstract class AbstractLocalIcebergCatalogViewTest Assumptions.setPreferredAssumptionException(PreferredAssumptionException.JUNIT5); } - public static class Profile extends Profiles.DefaultProfile { - @Override - public Map getConfigOverrides() { - return ImmutableMap.builder() - .putAll(super.getConfigOverrides()) - .put("polaris.features.\"ALLOW_WILDCARD_LOCATION\"", "true") - .put("polaris.features.\"SKIP_CREDENTIAL_SUBSCOPING_INDIRECTION\"", "true") - .put("polaris.features.\"LIST_PAGINATION_ENABLED\"", "true") - .put("polaris.event-listener.types", "test") - .build(); - } - } - public static final String CATALOG_NAME = "polaris-catalog"; public static Map VIEW_PREFIXES = diff --git a/runtime/service/src/test/java/org/apache/polaris/service/catalog/iceberg/IcebergCatalogHandlerAuthzTest.java b/runtime/service/src/test/java/org/apache/polaris/service/catalog/iceberg/IcebergCatalogHandlerAuthzTest.java index b0caedce4ba..da406c97fea 100644 --- a/runtime/service/src/test/java/org/apache/polaris/service/catalog/iceberg/IcebergCatalogHandlerAuthzTest.java +++ b/runtime/service/src/test/java/org/apache/polaris/service/catalog/iceberg/IcebergCatalogHandlerAuthzTest.java @@ -20,8 +20,8 @@ import io.quarkus.test.junit.QuarkusTest; import io.quarkus.test.junit.TestProfile; -import org.apache.polaris.service.admin.PolarisAuthzTestBase; +import org.apache.polaris.service.Profiles; @QuarkusTest -@TestProfile(PolarisAuthzTestBase.Profile.class) +@TestProfile(Profiles.PolarisAuthzBaseProfile.class) public class IcebergCatalogHandlerAuthzTest extends AbstractIcebergCatalogHandlerAuthzTest {} diff --git a/runtime/service/src/test/java/org/apache/polaris/service/catalog/iceberg/IcebergCatalogHandlerFineGrainedDisabledTest.java b/runtime/service/src/test/java/org/apache/polaris/service/catalog/iceberg/IcebergCatalogHandlerFineGrainedDisabledTest.java index 6560184fe6f..e9955c07960 100644 --- a/runtime/service/src/test/java/org/apache/polaris/service/catalog/iceberg/IcebergCatalogHandlerFineGrainedDisabledTest.java +++ b/runtime/service/src/test/java/org/apache/polaris/service/catalog/iceberg/IcebergCatalogHandlerFineGrainedDisabledTest.java @@ -31,6 +31,7 @@ import org.apache.iceberg.rest.requests.UpdateTableRequest; import org.apache.polaris.core.auth.PolarisPrincipal; import org.apache.polaris.core.entity.PolarisPrivilege; +import org.apache.polaris.service.Profiles; import org.apache.polaris.service.admin.PolarisAuthzTestBase; import org.junit.jupiter.api.DynamicNode; import org.junit.jupiter.api.TestFactory; @@ -50,7 +51,7 @@ private IcebergCatalogHandler newHandler() { return icebergCatalogHandlerFactory.createHandler(CATALOG_NAME, authenticatedPrincipal); } - public static class Profile extends PolarisAuthzTestBase.Profile { + public static class Profile extends Profiles.PolarisAuthzBaseProfile { @Override public Map getConfigOverrides() { return ImmutableMap.builder() diff --git a/runtime/service/src/test/java/org/apache/polaris/service/catalog/iceberg/IcebergCatalogHandlerNoSqlAuthzTest.java b/runtime/service/src/test/java/org/apache/polaris/service/catalog/iceberg/IcebergCatalogHandlerNoSqlAuthzTest.java index aad67ce272d..afead49f5eb 100644 --- a/runtime/service/src/test/java/org/apache/polaris/service/catalog/iceberg/IcebergCatalogHandlerNoSqlAuthzTest.java +++ b/runtime/service/src/test/java/org/apache/polaris/service/catalog/iceberg/IcebergCatalogHandlerNoSqlAuthzTest.java @@ -18,32 +18,18 @@ */ package org.apache.polaris.service.catalog.iceberg; -import static org.apache.polaris.service.catalog.Profiles.NOSQL_IN_MEM; - -import com.google.common.collect.ImmutableMap; import io.quarkus.test.junit.QuarkusTest; import io.quarkus.test.junit.TestProfile; import jakarta.inject.Inject; import java.util.List; -import java.util.Map; import org.apache.polaris.core.persistence.MetaStoreManagerFactory; import org.apache.polaris.core.persistence.bootstrap.RootCredentialsSet; -import org.apache.polaris.service.admin.PolarisAuthzTestBase; +import org.apache.polaris.service.Profiles; @QuarkusTest -@TestProfile(IcebergCatalogHandlerNoSqlAuthzTest.Profile.class) +@TestProfile(Profiles.IcebergCatalogHandlerNoSqlAuthzProfile.class) public class IcebergCatalogHandlerNoSqlAuthzTest extends AbstractIcebergCatalogHandlerAuthzTest { - public static class Profile extends PolarisAuthzTestBase.Profile { - @Override - public Map getConfigOverrides() { - return ImmutableMap.builder() - .putAll(super.getConfigOverrides()) - .putAll(NOSQL_IN_MEM) - .build(); - } - } - @Inject MetaStoreManagerFactory metaStoreManagerFactory; @Override diff --git a/runtime/service/src/test/java/org/apache/polaris/service/catalog/iceberg/LocalIcebergCatalogNoSqlInMemTest.java b/runtime/service/src/test/java/org/apache/polaris/service/catalog/iceberg/LocalIcebergCatalogNoSqlInMemTest.java index 1c0c0278bb1..c0c8473fe6b 100644 --- a/runtime/service/src/test/java/org/apache/polaris/service/catalog/iceberg/LocalIcebergCatalogNoSqlInMemTest.java +++ b/runtime/service/src/test/java/org/apache/polaris/service/catalog/iceberg/LocalIcebergCatalogNoSqlInMemTest.java @@ -18,29 +18,15 @@ */ package org.apache.polaris.service.catalog.iceberg; -import static org.apache.polaris.service.catalog.Profiles.NOSQL_IN_MEM; - -import com.google.common.collect.ImmutableMap; import io.quarkus.test.junit.QuarkusTest; import io.quarkus.test.junit.TestProfile; import java.util.List; -import java.util.Map; import org.apache.polaris.core.persistence.bootstrap.RootCredentialsSet; +import org.apache.polaris.service.Profiles; @QuarkusTest -@TestProfile(LocalIcebergCatalogNoSqlInMemTest.Profile.class) +@TestProfile(Profiles.NoSqlIcebergCatalogProfile.class) public class LocalIcebergCatalogNoSqlInMemTest extends AbstractLocalIcebergCatalogTest { - - public static class Profile extends AbstractLocalIcebergCatalogTest.Profile { - @Override - public Map getConfigOverrides() { - return ImmutableMap.builder() - .putAll(super.getConfigOverrides()) - .putAll(NOSQL_IN_MEM) - .build(); - } - } - @Override protected void bootstrapRealm(String realmName) { metaStoreManagerFactory diff --git a/runtime/service/src/test/java/org/apache/polaris/service/catalog/iceberg/LocalIcebergCatalogRelationalTest.java b/runtime/service/src/test/java/org/apache/polaris/service/catalog/iceberg/LocalIcebergCatalogRelationalTest.java index 26c2d86ce77..063153b606f 100644 --- a/runtime/service/src/test/java/org/apache/polaris/service/catalog/iceberg/LocalIcebergCatalogRelationalTest.java +++ b/runtime/service/src/test/java/org/apache/polaris/service/catalog/iceberg/LocalIcebergCatalogRelationalTest.java @@ -20,7 +20,8 @@ import io.quarkus.test.junit.QuarkusTest; import io.quarkus.test.junit.TestProfile; +import org.apache.polaris.service.Profiles; @QuarkusTest -@TestProfile(AbstractLocalIcebergCatalogTest.Profile.class) +@TestProfile(Profiles.DefaultIcebergCatalogProfile.class) public class LocalIcebergCatalogRelationalTest extends AbstractLocalIcebergCatalogTest {} diff --git a/runtime/service/src/test/java/org/apache/polaris/service/catalog/iceberg/LocalIcebergViewCatalogNoSqlInMemTest.java b/runtime/service/src/test/java/org/apache/polaris/service/catalog/iceberg/LocalIcebergViewCatalogNoSqlInMemTest.java index 623f68db1d5..658050ca496 100644 --- a/runtime/service/src/test/java/org/apache/polaris/service/catalog/iceberg/LocalIcebergViewCatalogNoSqlInMemTest.java +++ b/runtime/service/src/test/java/org/apache/polaris/service/catalog/iceberg/LocalIcebergViewCatalogNoSqlInMemTest.java @@ -18,33 +18,20 @@ */ package org.apache.polaris.service.catalog.iceberg; -import static org.apache.polaris.service.catalog.Profiles.NOSQL_IN_MEM; - -import com.google.common.collect.ImmutableMap; import io.quarkus.test.junit.QuarkusTest; import io.quarkus.test.junit.TestProfile; import jakarta.inject.Inject; import java.util.List; -import java.util.Map; import org.apache.polaris.core.persistence.MetaStoreManagerFactory; import org.apache.polaris.core.persistence.bootstrap.RootCredentialsSet; +import org.apache.polaris.service.Profiles; @QuarkusTest -@TestProfile(LocalIcebergViewCatalogNoSqlInMemTest.Profile.class) +@TestProfile(Profiles.NoSqlIcebergCatalogProfile.class) public class LocalIcebergViewCatalogNoSqlInMemTest extends AbstractLocalIcebergCatalogViewTest { @Inject MetaStoreManagerFactory metaStoreManagerFactory; - public static class Profile extends AbstractLocalIcebergCatalogViewTest.Profile { - @Override - public Map getConfigOverrides() { - return ImmutableMap.builder() - .putAll(super.getConfigOverrides()) - .putAll(NOSQL_IN_MEM) - .build(); - } - } - @Override protected void bootstrapRealm(String realmName) { metaStoreManagerFactory diff --git a/runtime/service/src/test/java/org/apache/polaris/service/catalog/iceberg/LocalIcebergViewCatalogRelationalTest.java b/runtime/service/src/test/java/org/apache/polaris/service/catalog/iceberg/LocalIcebergViewCatalogRelationalTest.java index f8c34b103f1..2ce70c9db16 100644 --- a/runtime/service/src/test/java/org/apache/polaris/service/catalog/iceberg/LocalIcebergViewCatalogRelationalTest.java +++ b/runtime/service/src/test/java/org/apache/polaris/service/catalog/iceberg/LocalIcebergViewCatalogRelationalTest.java @@ -20,7 +20,8 @@ import io.quarkus.test.junit.QuarkusTest; import io.quarkus.test.junit.TestProfile; +import org.apache.polaris.service.Profiles; @QuarkusTest -@TestProfile(AbstractLocalIcebergCatalogViewTest.Profile.class) +@TestProfile(Profiles.DefaultIcebergCatalogProfile.class) public class LocalIcebergViewCatalogRelationalTest extends AbstractLocalIcebergCatalogViewTest {} diff --git a/runtime/service/src/test/java/org/apache/polaris/service/catalog/policy/PolicyCatalogHandlerAuthzTest.java b/runtime/service/src/test/java/org/apache/polaris/service/catalog/policy/PolicyCatalogHandlerAuthzTest.java index e8d186900b7..5fd566febda 100644 --- a/runtime/service/src/test/java/org/apache/polaris/service/catalog/policy/PolicyCatalogHandlerAuthzTest.java +++ b/runtime/service/src/test/java/org/apache/polaris/service/catalog/policy/PolicyCatalogHandlerAuthzTest.java @@ -28,6 +28,7 @@ import org.apache.polaris.core.catalog.PolarisCatalogHelpers; import org.apache.polaris.core.entity.PolarisPrivilege; import org.apache.polaris.core.policy.PredefinedPolicyTypes; +import org.apache.polaris.service.Profiles; import org.apache.polaris.service.admin.PolarisAuthzTestBase; import org.apache.polaris.service.types.AttachPolicyRequest; import org.apache.polaris.service.types.CreatePolicyRequest; @@ -39,7 +40,7 @@ import org.junit.jupiter.api.TestFactory; @QuarkusTest -@TestProfile(PolarisAuthzTestBase.Profile.class) +@TestProfile(Profiles.PolarisAuthzBaseProfile.class) public class PolicyCatalogHandlerAuthzTest extends PolarisAuthzTestBase { @Inject PolicyCatalogHandlerFactory policyCatalogHandlerFactory; diff --git a/runtime/service/src/test/java/org/apache/polaris/service/catalog/policy/PolicyCatalogNoSqlInMemTest.java b/runtime/service/src/test/java/org/apache/polaris/service/catalog/policy/PolicyCatalogNoSqlInMemTest.java index 521769e4bfa..e5c9d67b3e3 100644 --- a/runtime/service/src/test/java/org/apache/polaris/service/catalog/policy/PolicyCatalogNoSqlInMemTest.java +++ b/runtime/service/src/test/java/org/apache/polaris/service/catalog/policy/PolicyCatalogNoSqlInMemTest.java @@ -24,7 +24,7 @@ import java.util.List; import org.apache.polaris.core.persistence.MetaStoreManagerFactory; import org.apache.polaris.core.persistence.bootstrap.RootCredentialsSet; -import org.apache.polaris.service.catalog.Profiles; +import org.apache.polaris.service.Profiles; @QuarkusTest @TestProfile(Profiles.DefaultNoSqlProfile.class) diff --git a/runtime/service/src/test/java/org/apache/polaris/service/catalog/policy/PolicyCatalogRelationalTest.java b/runtime/service/src/test/java/org/apache/polaris/service/catalog/policy/PolicyCatalogRelationalTest.java index fd812262b9a..35b5361e2cc 100644 --- a/runtime/service/src/test/java/org/apache/polaris/service/catalog/policy/PolicyCatalogRelationalTest.java +++ b/runtime/service/src/test/java/org/apache/polaris/service/catalog/policy/PolicyCatalogRelationalTest.java @@ -20,7 +20,7 @@ import io.quarkus.test.junit.QuarkusTest; import io.quarkus.test.junit.TestProfile; -import org.apache.polaris.service.catalog.Profiles; +import org.apache.polaris.service.Profiles; @QuarkusTest @TestProfile(Profiles.DefaultProfile.class) diff --git a/runtime/service/src/test/java/org/apache/polaris/service/distcache/TestPersistenceDistCacheInvalidationsIntegration.java b/runtime/service/src/test/java/org/apache/polaris/service/distcache/TestPersistenceDistCacheInvalidationsIntegration.java index 4ba068027e8..c46052af054 100644 --- a/runtime/service/src/test/java/org/apache/polaris/service/distcache/TestPersistenceDistCacheInvalidationsIntegration.java +++ b/runtime/service/src/test/java/org/apache/polaris/service/distcache/TestPersistenceDistCacheInvalidationsIntegration.java @@ -49,7 +49,7 @@ import org.apache.polaris.persistence.nosql.api.cache.CacheInvalidations; import org.apache.polaris.persistence.nosql.api.cache.CacheInvalidations.CacheInvalidation; import org.apache.polaris.persistence.nosql.api.obj.SimpleTestObj; -import org.apache.polaris.service.catalog.iceberg.AbstractLocalIcebergCatalogTest; +import org.apache.polaris.service.Profiles; import org.assertj.core.api.SoftAssertions; import org.eclipse.microprofile.config.ConfigProvider; import org.junit.jupiter.api.AfterEach; @@ -69,7 +69,7 @@ @SuppressWarnings("CdiInjectionPointsInspection") public class TestPersistenceDistCacheInvalidationsIntegration { - public static class Profile extends AbstractLocalIcebergCatalogTest.Profile { + public static class Profile extends Profiles.NoSqlIcebergCatalogProfile { @Override public Map getConfigOverrides() { return ImmutableMap.builder() @@ -77,9 +77,6 @@ public Map getConfigOverrides() { .put("quarkus.management.port", "0") .put("quarkus.management.host", "127.0.0.1") .put("quarkus.management.enabled", "true") - .put("polaris.persistence.type", "nosql") - .put("polaris.persistence.auto-bootstrap-types", "nosql") - .put("polaris.persistence.nosql.backend", "InMemory") .put( "polaris.persistence.distributed-cache-invalidations.valid-tokens", "token1," + TOKEN) .put( diff --git a/runtime/service/src/test/java/org/apache/polaris/service/events/PolarisEventListenerOldConfigurationTest.java b/runtime/service/src/test/java/org/apache/polaris/service/events/PolarisEventListenerOldConfigurationTest.java index 5f98d2c2ff8..a95d39603ba 100644 --- a/runtime/service/src/test/java/org/apache/polaris/service/events/PolarisEventListenerOldConfigurationTest.java +++ b/runtime/service/src/test/java/org/apache/polaris/service/events/PolarisEventListenerOldConfigurationTest.java @@ -22,28 +22,17 @@ import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.assertTrue; -import com.google.common.collect.ImmutableMap; import io.quarkus.test.junit.QuarkusTest; -import io.quarkus.test.junit.QuarkusTestProfile; import io.quarkus.test.junit.TestProfile; import jakarta.inject.Inject; -import java.util.Map; import java.util.Set; +import org.apache.polaris.service.Profiles; import org.junit.jupiter.api.Test; @QuarkusTest -@TestProfile(PolarisEventListenerOldConfigurationTest.OldConfigEventListenerProfile.class) +@TestProfile(Profiles.DefaultProfile.class) public class PolarisEventListenerOldConfigurationTest { - public static class OldConfigEventListenerProfile implements QuarkusTestProfile { - @Override - public Map getConfigOverrides() { - return ImmutableMap.builder() - .put("polaris.event-listener.type", "test") - .build(); - } - } - @Inject PolarisEventListenerConfiguration polarisEventListenerConfiguration; @SuppressWarnings("removal") diff --git a/runtime/service/src/test/java/org/apache/polaris/service/events/PolarisEventListenersTest.java b/runtime/service/src/test/java/org/apache/polaris/service/events/PolarisEventListenersTest.java index 39553e9f8b6..f058ba2bf09 100644 --- a/runtime/service/src/test/java/org/apache/polaris/service/events/PolarisEventListenersTest.java +++ b/runtime/service/src/test/java/org/apache/polaris/service/events/PolarisEventListenersTest.java @@ -24,9 +24,7 @@ import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.assertTrue; -import com.google.common.collect.ImmutableMap; import io.quarkus.test.junit.QuarkusTest; -import io.quarkus.test.junit.QuarkusTestProfile; import io.quarkus.test.junit.TestProfile; import io.smallrye.common.annotation.Identifier; import io.vertx.core.Context; @@ -34,18 +32,18 @@ import jakarta.inject.Singleton; import java.util.HashSet; import java.util.List; -import java.util.Map; import java.util.Set; import java.util.concurrent.CopyOnWriteArrayList; import java.util.concurrent.atomic.AtomicInteger; import java.util.function.Predicate; import java.util.stream.Collectors; import java.util.stream.Stream; +import org.apache.polaris.service.Profiles; import org.apache.polaris.service.events.listeners.PolarisEventListener; import org.junit.jupiter.api.Test; @QuarkusTest -@TestProfile(PolarisEventListenersTest.PolarisEventListenersTestProfile.class) +@TestProfile(Profiles.PolarisEventListenersTestProfile.class) public class PolarisEventListenersTest { static final Set CATALOG_EVENT_TYPES = PolarisEventType.typesOfCategory(PolarisEventType.Category.CATALOG); @@ -127,32 +125,6 @@ public static class ConsumeCatalogAndNotificationEventsListener extends Filterin } } - public static class PolarisEventListenersTestProfile implements QuarkusTestProfile { - @Override - public Map getConfigOverrides() { - return ImmutableMap.builder() - .put( - "polaris.event-listener.types", - "after-send-listener,before-send-listener,consume-all-listener,consume-all-listener-2,consume-only-catalog-listener,consume-catalog-and-after-notification-listener") - .put( - "polaris.event-listener.after-send-listener.enabled-event-types", - "AFTER_SEND_NOTIFICATION") - .put( - "polaris.event-listener.before-send-listener.enabled-event-types", - "BEFORE_SEND_NOTIFICATION") - .put( - "polaris.event-listener.consume-only-catalog-listener.enabled-event-categories", - "CATALOG") - .put( - "polaris.event-listener.consume-catalog-and-after-notification-listener.enabled-event-categories", - "CATALOG") - .put( - "polaris.event-listener.consume-catalog-and-after-notification-listener.enabled-event-types", - "AFTER_SEND_NOTIFICATION") - .build(); - } - } - @Inject PolarisEventDispatcher eventDispatcher; @Inject diff --git a/runtime/service/src/test/java/org/apache/polaris/service/events/listeners/inmemory/InMemoryBufferEventListenerBufferSizeTest.java b/runtime/service/src/test/java/org/apache/polaris/service/events/listeners/inmemory/InMemoryBufferEventListenerBufferSizeTest.java index ca6bcbdd84a..ad45042be00 100644 --- a/runtime/service/src/test/java/org/apache/polaris/service/events/listeners/inmemory/InMemoryBufferEventListenerBufferSizeTest.java +++ b/runtime/service/src/test/java/org/apache/polaris/service/events/listeners/inmemory/InMemoryBufferEventListenerBufferSizeTest.java @@ -25,35 +25,21 @@ import static org.mockito.Mockito.doReturn; import static org.mockito.Mockito.doThrow; -import com.google.common.collect.ImmutableMap; import io.quarkus.test.junit.QuarkusTest; -import io.quarkus.test.junit.QuarkusTestProfile; import io.quarkus.test.junit.TestProfile; import io.smallrye.mutiny.subscription.BackPressureFailure; import java.time.Duration; -import java.util.Map; import org.apache.polaris.core.persistence.PolarisMetaStoreManager; +import org.apache.polaris.service.Profiles; import org.junit.jupiter.api.Test; import org.junit.jupiter.api.TestInstance; import org.mockito.Mockito; @QuarkusTest @TestInstance(TestInstance.Lifecycle.PER_CLASS) -@TestProfile(InMemoryBufferEventListenerBufferSizeTest.Profile.class) +@TestProfile(Profiles.InMemoryBufferEventListenerBufferSizeProfile.class) class InMemoryBufferEventListenerBufferSizeTest extends InMemoryBufferEventListenerTestBase { - public static class Profile implements QuarkusTestProfile { - - @Override - public Map getConfigOverrides() { - return ImmutableMap.builder() - .putAll(BASE_CONFIG) - .put("polaris.event-listener.persistence-in-memory-buffer.buffer-time", "60s") - .put("polaris.event-listener.persistence-in-memory-buffer.max-buffer-size", "10") - .build(); - } - } - @Test void testFlushOnSize() { sendAsync("test1", 10); diff --git a/runtime/service/src/test/java/org/apache/polaris/service/events/listeners/inmemory/InMemoryBufferEventListenerBufferTimeTest.java b/runtime/service/src/test/java/org/apache/polaris/service/events/listeners/inmemory/InMemoryBufferEventListenerBufferTimeTest.java index 3a6c7210e33..83681f50f25 100644 --- a/runtime/service/src/test/java/org/apache/polaris/service/events/listeners/inmemory/InMemoryBufferEventListenerBufferTimeTest.java +++ b/runtime/service/src/test/java/org/apache/polaris/service/events/listeners/inmemory/InMemoryBufferEventListenerBufferTimeTest.java @@ -19,31 +19,17 @@ package org.apache.polaris.service.events.listeners.inmemory; -import com.google.common.collect.ImmutableMap; import io.quarkus.test.junit.QuarkusTest; -import io.quarkus.test.junit.QuarkusTestProfile; import io.quarkus.test.junit.TestProfile; -import java.util.Map; +import org.apache.polaris.service.Profiles; import org.junit.jupiter.api.Test; import org.junit.jupiter.api.TestInstance; @QuarkusTest @TestInstance(TestInstance.Lifecycle.PER_CLASS) -@TestProfile(InMemoryBufferEventListenerBufferTimeTest.Profile.class) +@TestProfile(Profiles.InMemoryBufferEventListenerBufferTimeProfile.class) class InMemoryBufferEventListenerBufferTimeTest extends InMemoryBufferEventListenerTestBase { - public static class Profile implements QuarkusTestProfile { - - @Override - public Map getConfigOverrides() { - return ImmutableMap.builder() - .putAll(BASE_CONFIG) - .put("polaris.event-listener.persistence-in-memory-buffer.buffer-time", "100ms") - .put("polaris.event-listener.persistence-in-memory-buffer.max-buffer-size", "1000") - .build(); - } - } - @Test void testFlushOnTimeout() { sendAsync("test1", 5); diff --git a/runtime/service/src/test/java/org/apache/polaris/service/events/listeners/inmemory/InMemoryBufferEventListenerIntegrationTest.java b/runtime/service/src/test/java/org/apache/polaris/service/events/listeners/inmemory/InMemoryBufferEventListenerIntegrationTest.java index bc90a47c6f6..735ba69f1f3 100644 --- a/runtime/service/src/test/java/org/apache/polaris/service/events/listeners/inmemory/InMemoryBufferEventListenerIntegrationTest.java +++ b/runtime/service/src/test/java/org/apache/polaris/service/events/listeners/inmemory/InMemoryBufferEventListenerIntegrationTest.java @@ -27,7 +27,6 @@ import com.google.common.collect.ImmutableList; import com.google.common.collect.ImmutableMap; import io.quarkus.test.junit.QuarkusTest; -import io.quarkus.test.junit.QuarkusTestProfile; import io.quarkus.test.junit.TestProfile; import jakarta.enterprise.inject.Instance; import jakarta.inject.Inject; @@ -41,7 +40,6 @@ import java.sql.Statement; import java.time.Duration; import java.util.List; -import java.util.Map; import javax.sql.DataSource; import org.apache.iceberg.PartitionSpec; import org.apache.iceberg.Schema; @@ -58,6 +56,7 @@ import org.apache.polaris.core.admin.model.PolarisCatalog; import org.apache.polaris.core.admin.model.StorageConfigInfo; import org.apache.polaris.core.entity.PolarisEvent; +import org.apache.polaris.service.Profiles; import org.apache.polaris.service.it.env.ClientPrincipal; import org.apache.polaris.service.it.env.IntegrationTestsHelper; import org.apache.polaris.service.it.env.PolarisApiEndpoints; @@ -72,31 +71,10 @@ @QuarkusTest @TestInstance(TestInstance.Lifecycle.PER_CLASS) -@TestProfile(InMemoryBufferEventListenerIntegrationTest.Profile.class) +@TestProfile(Profiles.InMemoryBufferEventListenerIntegrationProfile.class) @ExtendWith(PolarisIntegrationTestExtension.class) class InMemoryBufferEventListenerIntegrationTest { - public static class Profile implements QuarkusTestProfile { - - @Override - public Map getConfigOverrides() { - return ImmutableMap.builder() - .put("polaris.persistence.type", "relational-jdbc") - .put("polaris.persistence.auto-bootstrap-types", "relational-jdbc") - .put("quarkus.datasource.db-kind", "h2") - .put("quarkus.otel.sdk.disabled", "false") - .put( - "quarkus.datasource.jdbc.url", - "jdbc:h2:mem:test;DB_CLOSE_DELAY=-1;MODE=PostgreSQL;DATABASE_TO_LOWER=TRUE") - .put("polaris.event-listener.type", "persistence-in-memory-buffer") - .put("polaris.event-listener.persistence-in-memory-buffer.buffer-time", "100ms") - .put("polaris.features.\"ALLOW_INSECURE_STORAGE_TYPES\"", "true") - .put("polaris.features.\"SUPPORTED_CATALOG_STORAGE_TYPES\"", "[\"FILE\",\"S3\"]") - .put("polaris.readiness.ignore-severe-issues", "true") - .build(); - } - } - private RestApi managementApi; private PolarisApiEndpoints endpoints; private PolarisClient client; diff --git a/runtime/service/src/test/java/org/apache/polaris/service/events/listeners/inmemory/InMemoryBufferEventListenerTestBase.java b/runtime/service/src/test/java/org/apache/polaris/service/events/listeners/inmemory/InMemoryBufferEventListenerTestBase.java index a1d92485196..7d943461b42 100644 --- a/runtime/service/src/test/java/org/apache/polaris/service/events/listeners/inmemory/InMemoryBufferEventListenerTestBase.java +++ b/runtime/service/src/test/java/org/apache/polaris/service/events/listeners/inmemory/InMemoryBufferEventListenerTestBase.java @@ -24,7 +24,6 @@ import static org.awaitility.Awaitility.await; import static org.mockito.Mockito.reset; -import com.google.common.collect.ImmutableMap; import io.netty.channel.EventLoopGroup; import io.quarkus.netty.MainEventLoopGroup; import io.quarkus.test.junit.mockito.InjectSpy; @@ -35,7 +34,6 @@ import java.sql.ResultSet; import java.sql.Statement; import java.time.Duration; -import java.util.Map; import java.util.UUID; import javax.sql.DataSource; import org.apache.polaris.core.entity.PolarisEvent; @@ -45,24 +43,6 @@ abstract class InMemoryBufferEventListenerTestBase { - static final Map BASE_CONFIG = - ImmutableMap.builder() - .put("polaris.realm-context.realms", "test1,test2") - .put("polaris.persistence.type", "relational-jdbc") - .put("polaris.persistence.auto-bootstrap-types", "relational-jdbc") - .put("quarkus.datasource.db-kind", "h2") - .put( - "quarkus.datasource.jdbc.url", - "jdbc:h2:mem:test;DB_CLOSE_DELAY=-1;MODE=PostgreSQL;DATABASE_TO_LOWER=TRUE") - .put("polaris.event-listener.type", "persistence-in-memory-buffer") - .put( - "quarkus.fault-tolerance.\"org.apache.polaris.service.events.listeners.inmemory.InMemoryBufferEventListener/flush\".retry.max-retries", - "1") - .put( - "quarkus.fault-tolerance.\"org.apache.polaris.service.events.listeners.inmemory.InMemoryBufferEventListener/flush\".retry.delay", - "10") - .build(); - @Inject @Identifier("persistence-in-memory-buffer") Instance producerInstance; diff --git a/runtime/service/src/test/java/org/apache/polaris/service/metrics/RealmIdTagDisabledMetricsTest.java b/runtime/service/src/test/java/org/apache/polaris/service/metrics/RealmIdTagDisabledMetricsTest.java index 68fbc5180f1..e60c6b22b20 100644 --- a/runtime/service/src/test/java/org/apache/polaris/service/metrics/RealmIdTagDisabledMetricsTest.java +++ b/runtime/service/src/test/java/org/apache/polaris/service/metrics/RealmIdTagDisabledMetricsTest.java @@ -19,20 +19,9 @@ package org.apache.polaris.service.metrics; import io.quarkus.test.junit.QuarkusTest; -import io.quarkus.test.junit.QuarkusTestProfile; import io.quarkus.test.junit.TestProfile; -import java.util.Map; +import org.apache.polaris.service.Profiles; @QuarkusTest -@TestProfile(RealmIdTagDisabledMetricsTest.Profile.class) -public class RealmIdTagDisabledMetricsTest extends MetricsTestBase { - - public static class Profile implements QuarkusTestProfile { - - @Override - public Map getConfigOverrides() { - return Map.of( - "polaris.metrics.tags.environment", "prod", "polaris.realm-context.type", "test"); - } - } -} +@TestProfile(Profiles.TagsDisabledMetricsProfile.class) +public class RealmIdTagDisabledMetricsTest extends MetricsTestBase {} diff --git a/runtime/service/src/test/java/org/apache/polaris/service/metrics/RealmIdTagEnabledMetricsTest.java b/runtime/service/src/test/java/org/apache/polaris/service/metrics/RealmIdTagEnabledMetricsTest.java index 86ddf1fb99a..9b37b29f7a9 100644 --- a/runtime/service/src/test/java/org/apache/polaris/service/metrics/RealmIdTagEnabledMetricsTest.java +++ b/runtime/service/src/test/java/org/apache/polaris/service/metrics/RealmIdTagEnabledMetricsTest.java @@ -19,29 +19,9 @@ package org.apache.polaris.service.metrics; import io.quarkus.test.junit.QuarkusTest; -import io.quarkus.test.junit.QuarkusTestProfile; import io.quarkus.test.junit.TestProfile; -import java.util.Map; +import org.apache.polaris.service.Profiles; @QuarkusTest -@TestProfile(RealmIdTagEnabledMetricsTest.Profile.class) -public class RealmIdTagEnabledMetricsTest extends MetricsTestBase { - - public static class Profile implements QuarkusTestProfile { - - @Override - public Map getConfigOverrides() { - return Map.of( - "polaris.metrics.tags.environment", - "prod", - "polaris.realm-context.type", - "test", - "polaris.metrics.realm-id-tag.enable-in-api-metrics", - "true", - "polaris.metrics.realm-id-tag.enable-in-http-metrics", - "true", - "polaris.metrics.user-principal-tag.enable-in-api-metrics", - "false"); - } - } -} +@TestProfile(Profiles.RealmIdTagEnabledMetricsProfile.class) +public class RealmIdTagEnabledMetricsTest extends MetricsTestBase {} diff --git a/runtime/service/src/test/java/org/apache/polaris/service/metrics/UserPrincipalTagDisabledMetricsTest.java b/runtime/service/src/test/java/org/apache/polaris/service/metrics/UserPrincipalTagDisabledMetricsTest.java index 9f45bb51c48..119572cf539 100644 --- a/runtime/service/src/test/java/org/apache/polaris/service/metrics/UserPrincipalTagDisabledMetricsTest.java +++ b/runtime/service/src/test/java/org/apache/polaris/service/metrics/UserPrincipalTagDisabledMetricsTest.java @@ -19,20 +19,9 @@ package org.apache.polaris.service.metrics; import io.quarkus.test.junit.QuarkusTest; -import io.quarkus.test.junit.QuarkusTestProfile; import io.quarkus.test.junit.TestProfile; -import java.util.Map; +import org.apache.polaris.service.Profiles; @QuarkusTest -@TestProfile(UserPrincipalTagDisabledMetricsTest.Profile.class) -public class UserPrincipalTagDisabledMetricsTest extends MetricsTestBase { - - public static class Profile implements QuarkusTestProfile { - - @Override - public Map getConfigOverrides() { - return Map.of( - "polaris.metrics.tags.environment", "prod", "polaris.realm-context.type", "test"); - } - } -} +@TestProfile(Profiles.TagsDisabledMetricsProfile.class) +public class UserPrincipalTagDisabledMetricsTest extends MetricsTestBase {} diff --git a/runtime/service/src/test/java/org/apache/polaris/service/metrics/UserPrincipalTagEnabledMetricsTest.java b/runtime/service/src/test/java/org/apache/polaris/service/metrics/UserPrincipalTagEnabledMetricsTest.java index b7e706fea01..cd743a4b245 100644 --- a/runtime/service/src/test/java/org/apache/polaris/service/metrics/UserPrincipalTagEnabledMetricsTest.java +++ b/runtime/service/src/test/java/org/apache/polaris/service/metrics/UserPrincipalTagEnabledMetricsTest.java @@ -19,27 +19,9 @@ package org.apache.polaris.service.metrics; import io.quarkus.test.junit.QuarkusTest; -import io.quarkus.test.junit.QuarkusTestProfile; import io.quarkus.test.junit.TestProfile; -import java.util.Map; +import org.apache.polaris.service.Profiles; @QuarkusTest -@TestProfile(UserPrincipalTagEnabledMetricsTest.Profile.class) -public class UserPrincipalTagEnabledMetricsTest extends MetricsTestBase { - - public static class Profile implements QuarkusTestProfile { - - @Override - public Map getConfigOverrides() { - return Map.of( - "polaris.metrics.tags.environment", - "prod", - "polaris.metrics.user-principal-tag.enable-in-api-metrics", - "true", - "polaris.metrics.realm-id-tag.enable-in-api-metrics", - "false", - "polaris.metrics.realm-id-tag.enable-in-http-metrics", - "false"); - } - } -} +@TestProfile(Profiles.UserPrincipalTagEnabledMetricsProfile.class) +public class UserPrincipalTagEnabledMetricsTest extends MetricsTestBase {} From 626c8b12622bd957a31a13eb7dfb59f245db66f6 Mon Sep 17 00:00:00 2001 From: Robert Stupp Date: Fri, 12 Jun 2026 17:59:19 +0200 Subject: [PATCH 13/22] polaris-admin/test: Centralize Quarkus test profiles --- runtime/admin/build.gradle.kts | 1 + .../NoSqlInMemoryBootstrapCommandTest.java | 3 +- .../admintool/nosql/NoSqlInMemoryProfile.java | 30 ---- .../nosql/NoSqlInMemoryPurgeCommandTest.java | 18 +-- .../nosql/NoSqlMongoBootstrapCommandTest.java | 3 +- .../NoSqlMongoMaintenanceCommandTest.java | 3 +- .../admintool/nosql/NoSqlMongoProfile.java | 37 ----- .../nosql/NoSqlMongoPurgeCommandTest.java | 18 +-- .../jdbc/CockroachJdbcAdminProfile.java | 47 ------ .../CockroachJdbcBootstrapCommandTest.java | 3 +- .../jdbc/CockroachJdbcPurgeCommandTest.java | 17 +-- .../jdbc/RelationalJdbcAdminProfile.java | 45 ------ .../RelationalJdbcBootstrapCommandTest.java | 3 +- .../jdbc/RelationalJdbcPurgeCommandTest.java | 17 +-- .../polaris/admintool/AdminProfiles.java | 138 ++++++++++++++++++ 15 files changed, 161 insertions(+), 222 deletions(-) delete mode 100644 runtime/admin/src/test/java/org/apache/polaris/admintool/nosql/NoSqlInMemoryProfile.java delete mode 100644 runtime/admin/src/test/java/org/apache/polaris/admintool/nosql/NoSqlMongoProfile.java delete mode 100644 runtime/admin/src/test/java/org/apache/polaris/admintool/relational/jdbc/CockroachJdbcAdminProfile.java delete mode 100644 runtime/admin/src/test/java/org/apache/polaris/admintool/relational/jdbc/RelationalJdbcAdminProfile.java create mode 100644 runtime/admin/src/testFixtures/java/org/apache/polaris/admintool/AdminProfiles.java diff --git a/runtime/admin/build.gradle.kts b/runtime/admin/build.gradle.kts index e6bdeeb5bb3..f751e8c7000 100644 --- a/runtime/admin/build.gradle.kts +++ b/runtime/admin/build.gradle.kts @@ -58,6 +58,7 @@ dependencies { testImplementation(project(":polaris-runtime-test-common")) testFixturesApi(project(":polaris-core")) + testFixturesImplementation(project(":polaris-runtime-test-common")) testFixturesApi(enforcedPlatform(libs.quarkus.bom)) testFixturesApi("io.quarkus:quarkus-junit") diff --git a/runtime/admin/src/test/java/org/apache/polaris/admintool/nosql/NoSqlInMemoryBootstrapCommandTest.java b/runtime/admin/src/test/java/org/apache/polaris/admintool/nosql/NoSqlInMemoryBootstrapCommandTest.java index 5f9d6f6c1f3..50a8649ee5a 100644 --- a/runtime/admin/src/test/java/org/apache/polaris/admintool/nosql/NoSqlInMemoryBootstrapCommandTest.java +++ b/runtime/admin/src/test/java/org/apache/polaris/admintool/nosql/NoSqlInMemoryBootstrapCommandTest.java @@ -19,7 +19,8 @@ package org.apache.polaris.admintool.nosql; import io.quarkus.test.junit.TestProfile; +import org.apache.polaris.admintool.AdminProfiles; import org.apache.polaris.admintool.BootstrapCommandTestBase; -@TestProfile(NoSqlInMemoryProfile.class) +@TestProfile(AdminProfiles.NoSqlInMemory.class) class NoSqlInMemoryBootstrapCommandTest extends BootstrapCommandTestBase {} diff --git a/runtime/admin/src/test/java/org/apache/polaris/admintool/nosql/NoSqlInMemoryProfile.java b/runtime/admin/src/test/java/org/apache/polaris/admintool/nosql/NoSqlInMemoryProfile.java deleted file mode 100644 index 024631403da..00000000000 --- a/runtime/admin/src/test/java/org/apache/polaris/admintool/nosql/NoSqlInMemoryProfile.java +++ /dev/null @@ -1,30 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ -package org.apache.polaris.admintool.nosql; - -import io.quarkus.test.junit.QuarkusTestProfile; -import java.util.Map; - -public class NoSqlInMemoryProfile implements QuarkusTestProfile { - @Override - public Map getConfigOverrides() { - return Map.of( - "polaris.persistence.type", "nosql", "polaris.persistence.nosql.backend", "InMemory"); - } -} diff --git a/runtime/admin/src/test/java/org/apache/polaris/admintool/nosql/NoSqlInMemoryPurgeCommandTest.java b/runtime/admin/src/test/java/org/apache/polaris/admintool/nosql/NoSqlInMemoryPurgeCommandTest.java index ecfacf5c319..b3fc169514a 100644 --- a/runtime/admin/src/test/java/org/apache/polaris/admintool/nosql/NoSqlInMemoryPurgeCommandTest.java +++ b/runtime/admin/src/test/java/org/apache/polaris/admintool/nosql/NoSqlInMemoryPurgeCommandTest.java @@ -18,21 +18,9 @@ */ package org.apache.polaris.admintool.nosql; -import com.google.common.collect.ImmutableMap; import io.quarkus.test.junit.TestProfile; -import java.util.Map; +import org.apache.polaris.admintool.AdminProfiles; import org.apache.polaris.admintool.PurgeCommandTestBase; -@TestProfile(NoSqlInMemoryPurgeCommandTest.Profile.class) -class NoSqlInMemoryPurgeCommandTest extends PurgeCommandTestBase { - - public static class Profile extends NoSqlInMemoryProfile { - @Override - public Map getConfigOverrides() { - return ImmutableMap.builder() - .putAll(super.getConfigOverrides()) - .put("pre-bootstrap", "true") - .build(); - } - } -} +@TestProfile(AdminProfiles.PreBootstrappedNoSqlInMemory.class) +class NoSqlInMemoryPurgeCommandTest extends PurgeCommandTestBase {} diff --git a/runtime/admin/src/test/java/org/apache/polaris/admintool/nosql/NoSqlMongoBootstrapCommandTest.java b/runtime/admin/src/test/java/org/apache/polaris/admintool/nosql/NoSqlMongoBootstrapCommandTest.java index 22aa507b3d7..14b335f3814 100644 --- a/runtime/admin/src/test/java/org/apache/polaris/admintool/nosql/NoSqlMongoBootstrapCommandTest.java +++ b/runtime/admin/src/test/java/org/apache/polaris/admintool/nosql/NoSqlMongoBootstrapCommandTest.java @@ -19,7 +19,8 @@ package org.apache.polaris.admintool.nosql; import io.quarkus.test.junit.TestProfile; +import org.apache.polaris.admintool.AdminProfiles; import org.apache.polaris.admintool.BootstrapCommandTestBase; -@TestProfile(NoSqlMongoProfile.class) +@TestProfile(AdminProfiles.NoSqlMongo.class) class NoSqlMongoBootstrapCommandTest extends BootstrapCommandTestBase {} diff --git a/runtime/admin/src/test/java/org/apache/polaris/admintool/nosql/NoSqlMongoMaintenanceCommandTest.java b/runtime/admin/src/test/java/org/apache/polaris/admintool/nosql/NoSqlMongoMaintenanceCommandTest.java index 595160853de..beacbe5e915 100644 --- a/runtime/admin/src/test/java/org/apache/polaris/admintool/nosql/NoSqlMongoMaintenanceCommandTest.java +++ b/runtime/admin/src/test/java/org/apache/polaris/admintool/nosql/NoSqlMongoMaintenanceCommandTest.java @@ -24,9 +24,10 @@ import io.quarkus.test.junit.main.Launch; import io.quarkus.test.junit.main.LaunchResult; import io.quarkus.test.junit.main.QuarkusMainTest; +import org.apache.polaris.admintool.AdminProfiles; import org.junit.jupiter.api.Test; -@TestProfile(NoSqlMongoProfile.class) +@TestProfile(AdminProfiles.NoSqlMongo.class) @QuarkusMainTest class NoSqlMongoMaintenanceCommandTest { @Test diff --git a/runtime/admin/src/test/java/org/apache/polaris/admintool/nosql/NoSqlMongoProfile.java b/runtime/admin/src/test/java/org/apache/polaris/admintool/nosql/NoSqlMongoProfile.java deleted file mode 100644 index 4b6f9959d11..00000000000 --- a/runtime/admin/src/test/java/org/apache/polaris/admintool/nosql/NoSqlMongoProfile.java +++ /dev/null @@ -1,37 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ -package org.apache.polaris.admintool.nosql; - -import io.quarkus.test.junit.QuarkusTestProfile; -import java.util.List; -import java.util.Map; -import org.apache.polaris.admintool.MongoTestResourceLifecycleManager; - -public class NoSqlMongoProfile implements QuarkusTestProfile { - @Override - public Map getConfigOverrides() { - return Map.of( - "polaris.persistence.type", "nosql", "polaris.persistence.nosql.backend", "MongoDb"); - } - - @Override - public List testResources() { - return List.of(new TestResourceEntry(MongoTestResourceLifecycleManager.class)); - } -} diff --git a/runtime/admin/src/test/java/org/apache/polaris/admintool/nosql/NoSqlMongoPurgeCommandTest.java b/runtime/admin/src/test/java/org/apache/polaris/admintool/nosql/NoSqlMongoPurgeCommandTest.java index 81deb680cc1..f9cb7425989 100644 --- a/runtime/admin/src/test/java/org/apache/polaris/admintool/nosql/NoSqlMongoPurgeCommandTest.java +++ b/runtime/admin/src/test/java/org/apache/polaris/admintool/nosql/NoSqlMongoPurgeCommandTest.java @@ -18,21 +18,9 @@ */ package org.apache.polaris.admintool.nosql; -import com.google.common.collect.ImmutableMap; import io.quarkus.test.junit.TestProfile; -import java.util.Map; +import org.apache.polaris.admintool.AdminProfiles; import org.apache.polaris.admintool.PurgeCommandTestBase; -@TestProfile(NoSqlMongoPurgeCommandTest.Profile.class) -class NoSqlMongoPurgeCommandTest extends PurgeCommandTestBase { - - public static class Profile extends NoSqlMongoProfile { - @Override - public Map getConfigOverrides() { - return ImmutableMap.builder() - .putAll(super.getConfigOverrides()) - .put("pre-bootstrap", "true") - .build(); - } - } -} +@TestProfile(AdminProfiles.PreBootstrappedNoSqlMongo.class) +class NoSqlMongoPurgeCommandTest extends PurgeCommandTestBase {} diff --git a/runtime/admin/src/test/java/org/apache/polaris/admintool/relational/jdbc/CockroachJdbcAdminProfile.java b/runtime/admin/src/test/java/org/apache/polaris/admintool/relational/jdbc/CockroachJdbcAdminProfile.java deleted file mode 100644 index 2b2e1d4d970..00000000000 --- a/runtime/admin/src/test/java/org/apache/polaris/admintool/relational/jdbc/CockroachJdbcAdminProfile.java +++ /dev/null @@ -1,47 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ -package org.apache.polaris.admintool.relational.jdbc; - -import java.util.List; -import java.util.Map; -import org.apache.polaris.test.commons.CockroachRelationalJdbcLifeCycleManagement; -import org.apache.polaris.test.commons.RelationalJdbcProfile; - -public class CockroachJdbcAdminProfile extends RelationalJdbcProfile { - @Override - public Map getConfigOverrides() { - return Map.of( - "polaris.persistence.type", - "relational-jdbc", - "polaris.persistence.relational.jdbc.database-type", - "cockroachdb", - // These two options are required to "trigger" the enablement of JDBC data sources in - // admin-tool tests, but do not harm other tests. - "quarkus.datasource.active", - "true", - "quarkus.datasource.db-kind", - "postgresql"); - } - - @Override - public List testResources() { - return List.of( - new TestResourceEntry(CockroachRelationalJdbcLifeCycleManagement.class, Map.of())); - } -} diff --git a/runtime/admin/src/test/java/org/apache/polaris/admintool/relational/jdbc/CockroachJdbcBootstrapCommandTest.java b/runtime/admin/src/test/java/org/apache/polaris/admintool/relational/jdbc/CockroachJdbcBootstrapCommandTest.java index c9fd056d6fa..728e7dd9d01 100644 --- a/runtime/admin/src/test/java/org/apache/polaris/admintool/relational/jdbc/CockroachJdbcBootstrapCommandTest.java +++ b/runtime/admin/src/test/java/org/apache/polaris/admintool/relational/jdbc/CockroachJdbcBootstrapCommandTest.java @@ -23,9 +23,10 @@ import io.quarkus.test.junit.TestProfile; import io.quarkus.test.junit.main.LaunchResult; import io.quarkus.test.junit.main.QuarkusMainLauncher; +import org.apache.polaris.admintool.AdminProfiles; import org.junit.jupiter.api.Test; -@TestProfile(CockroachJdbcAdminProfile.class) +@TestProfile(AdminProfiles.CockroachJdbc.class) public class CockroachJdbcBootstrapCommandTest extends RelationalJdbcBootstrapCommandTest { @Override diff --git a/runtime/admin/src/test/java/org/apache/polaris/admintool/relational/jdbc/CockroachJdbcPurgeCommandTest.java b/runtime/admin/src/test/java/org/apache/polaris/admintool/relational/jdbc/CockroachJdbcPurgeCommandTest.java index 550eb779f46..0c08ca7a847 100644 --- a/runtime/admin/src/test/java/org/apache/polaris/admintool/relational/jdbc/CockroachJdbcPurgeCommandTest.java +++ b/runtime/admin/src/test/java/org/apache/polaris/admintool/relational/jdbc/CockroachJdbcPurgeCommandTest.java @@ -18,20 +18,9 @@ */ package org.apache.polaris.admintool.relational.jdbc; -import com.google.common.collect.ImmutableMap; import io.quarkus.test.junit.TestProfile; -import java.util.Map; +import org.apache.polaris.admintool.AdminProfiles; import org.apache.polaris.admintool.PurgeCommandTestBase; -@TestProfile(CockroachJdbcPurgeCommandTest.Profile.class) -public class CockroachJdbcPurgeCommandTest extends PurgeCommandTestBase { - public static class Profile extends CockroachJdbcAdminProfile { - @Override - public Map getConfigOverrides() { - return ImmutableMap.builder() - .putAll(super.getConfigOverrides()) - .put("pre-bootstrap", "true") - .build(); - } - } -} +@TestProfile(AdminProfiles.PreBootstrappedCockroachJdbc.class) +public class CockroachJdbcPurgeCommandTest extends PurgeCommandTestBase {} diff --git a/runtime/admin/src/test/java/org/apache/polaris/admintool/relational/jdbc/RelationalJdbcAdminProfile.java b/runtime/admin/src/test/java/org/apache/polaris/admintool/relational/jdbc/RelationalJdbcAdminProfile.java deleted file mode 100644 index 07d2e29d364..00000000000 --- a/runtime/admin/src/test/java/org/apache/polaris/admintool/relational/jdbc/RelationalJdbcAdminProfile.java +++ /dev/null @@ -1,45 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ -package org.apache.polaris.admintool.relational.jdbc; - -import java.util.List; -import java.util.Map; -import org.apache.polaris.test.commons.PostgresRelationalJdbcLifeCycleManagement; -import org.apache.polaris.test.commons.RelationalJdbcProfile; - -public class RelationalJdbcAdminProfile extends RelationalJdbcProfile { - @Override - public Map getConfigOverrides() { - return Map.of( - "polaris.persistence.type", - "relational-jdbc", - // These two options are required to "trigger" the enablement of JDBC data sources in - // admin-tool tests, but do not harm other tests. - "quarkus.datasource.active", - "true", - "quarkus.datasource.db-kind", - "postgresql"); - } - - @Override - public List testResources() { - return List.of( - new TestResourceEntry(PostgresRelationalJdbcLifeCycleManagement.class, Map.of())); - } -} diff --git a/runtime/admin/src/test/java/org/apache/polaris/admintool/relational/jdbc/RelationalJdbcBootstrapCommandTest.java b/runtime/admin/src/test/java/org/apache/polaris/admintool/relational/jdbc/RelationalJdbcBootstrapCommandTest.java index 5feb36d660e..4824f61163c 100644 --- a/runtime/admin/src/test/java/org/apache/polaris/admintool/relational/jdbc/RelationalJdbcBootstrapCommandTest.java +++ b/runtime/admin/src/test/java/org/apache/polaris/admintool/relational/jdbc/RelationalJdbcBootstrapCommandTest.java @@ -23,10 +23,11 @@ import io.quarkus.test.junit.TestProfile; import io.quarkus.test.junit.main.LaunchResult; import io.quarkus.test.junit.main.QuarkusMainLauncher; +import org.apache.polaris.admintool.AdminProfiles; import org.apache.polaris.admintool.BootstrapCommandTestBase; import org.junit.jupiter.api.Test; -@TestProfile(RelationalJdbcAdminProfile.class) +@TestProfile(AdminProfiles.RelationalJdbc.class) public class RelationalJdbcBootstrapCommandTest extends BootstrapCommandTestBase { @Test diff --git a/runtime/admin/src/test/java/org/apache/polaris/admintool/relational/jdbc/RelationalJdbcPurgeCommandTest.java b/runtime/admin/src/test/java/org/apache/polaris/admintool/relational/jdbc/RelationalJdbcPurgeCommandTest.java index 096708643cb..a8541c1b2e5 100644 --- a/runtime/admin/src/test/java/org/apache/polaris/admintool/relational/jdbc/RelationalJdbcPurgeCommandTest.java +++ b/runtime/admin/src/test/java/org/apache/polaris/admintool/relational/jdbc/RelationalJdbcPurgeCommandTest.java @@ -18,20 +18,9 @@ */ package org.apache.polaris.admintool.relational.jdbc; -import com.google.common.collect.ImmutableMap; import io.quarkus.test.junit.TestProfile; -import java.util.Map; +import org.apache.polaris.admintool.AdminProfiles; import org.apache.polaris.admintool.PurgeCommandTestBase; -@TestProfile(RelationalJdbcPurgeCommandTest.Profile.class) -public class RelationalJdbcPurgeCommandTest extends PurgeCommandTestBase { - public static class Profile extends RelationalJdbcAdminProfile { - @Override - public Map getConfigOverrides() { - return ImmutableMap.builder() - .putAll(super.getConfigOverrides()) - .put("pre-bootstrap", "true") - .build(); - } - } -} +@TestProfile(AdminProfiles.PreBootstrappedRelationalJdbc.class) +public class RelationalJdbcPurgeCommandTest extends PurgeCommandTestBase {} diff --git a/runtime/admin/src/testFixtures/java/org/apache/polaris/admintool/AdminProfiles.java b/runtime/admin/src/testFixtures/java/org/apache/polaris/admintool/AdminProfiles.java new file mode 100644 index 00000000000..82c6380b16f --- /dev/null +++ b/runtime/admin/src/testFixtures/java/org/apache/polaris/admintool/AdminProfiles.java @@ -0,0 +1,138 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.polaris.admintool; + +import com.google.common.collect.ImmutableMap; +import io.quarkus.test.junit.QuarkusTestProfile; +import java.util.List; +import java.util.Map; +import org.apache.polaris.test.commons.CockroachRelationalJdbcLifeCycleManagement; +import org.apache.polaris.test.commons.PostgresRelationalJdbcLifeCycleManagement; + +public final class AdminProfiles { + + private static final Map PRE_BOOTSTRAP_CONFIG = Map.of("pre-bootstrap", "true"); + + private AdminProfiles() {} + + public static class NoSqlInMemory implements QuarkusTestProfile { + @Override + public Map getConfigOverrides() { + return Map.of( + "polaris.persistence.type", "nosql", "polaris.persistence.nosql.backend", "InMemory"); + } + } + + public static class PreBootstrappedNoSqlInMemory extends NoSqlInMemory { + @Override + public Map getConfigOverrides() { + return ImmutableMap.builder() + .putAll(super.getConfigOverrides()) + .putAll(PRE_BOOTSTRAP_CONFIG) + .build(); + } + } + + public static class NoSqlMongo implements QuarkusTestProfile { + @Override + public Map getConfigOverrides() { + return Map.of( + "polaris.persistence.type", "nosql", "polaris.persistence.nosql.backend", "MongoDb"); + } + + @Override + public List testResources() { + return List.of(new TestResourceEntry(MongoTestResourceLifecycleManager.class)); + } + } + + public static class PreBootstrappedNoSqlMongo extends NoSqlMongo { + @Override + public Map getConfigOverrides() { + return ImmutableMap.builder() + .putAll(super.getConfigOverrides()) + .putAll(PRE_BOOTSTRAP_CONFIG) + .build(); + } + } + + public static class RelationalJdbc implements QuarkusTestProfile { + @Override + public Map getConfigOverrides() { + return Map.of( + "polaris.persistence.type", + "relational-jdbc", + // These two options are required to "trigger" the enablement of JDBC data sources in + // admin-tool tests, but do not harm other tests. + "quarkus.datasource.active", + "true", + "quarkus.datasource.db-kind", + "postgresql"); + } + + @Override + public List testResources() { + return List.of( + new TestResourceEntry(PostgresRelationalJdbcLifeCycleManagement.class, Map.of())); + } + } + + public static class PreBootstrappedRelationalJdbc extends RelationalJdbc { + @Override + public Map getConfigOverrides() { + return ImmutableMap.builder() + .putAll(super.getConfigOverrides()) + .putAll(PRE_BOOTSTRAP_CONFIG) + .build(); + } + } + + public static class CockroachJdbc implements QuarkusTestProfile { + @Override + public Map getConfigOverrides() { + return Map.of( + "polaris.persistence.type", + "relational-jdbc", + "polaris.persistence.relational.jdbc.database-type", + "cockroachdb", + // These two options are required to "trigger" the enablement of JDBC data sources in + // admin-tool tests, but do not harm other tests. + "quarkus.datasource.active", + "true", + "quarkus.datasource.db-kind", + "postgresql"); + } + + @Override + public List testResources() { + return List.of( + new TestResourceEntry(CockroachRelationalJdbcLifeCycleManagement.class, Map.of())); + } + } + + public static class PreBootstrappedCockroachJdbc extends CockroachJdbc { + @Override + public Map getConfigOverrides() { + return ImmutableMap.builder() + .putAll(super.getConfigOverrides()) + .putAll(PRE_BOOTSTRAP_CONFIG) + .build(); + } + } +} From 1c65fb56a1112b9ab09425e9ecd0817ccb4a40cc Mon Sep 17 00:00:00 2001 From: Robert Stupp Date: Fri, 12 Jun 2026 19:15:02 +0200 Subject: [PATCH 14/22] polaris-admin/tests: consolidate to 1 test profile per backend Removed the four PreBootstrapped* admin profiles and the pre-bootstrap config flag from AdminProfiles. * Switched purge tests to the regular backend profiles. * Kept purge setup inside PurgeCommandTestBase with an unconditional test startup observer, using purge-specific realms: `purge-test-realm1`, `purge-test-realm2`, `purge-test-missing-realm`. --- .../admintool/PurgeCommandTestBase.java | 23 +++++----- .../nosql/NoSqlInMemoryPurgeCommandTest.java | 2 +- .../nosql/NoSqlMongoPurgeCommandTest.java | 2 +- .../CockroachJdbcBootstrapCommandTest.java | 20 +-------- .../jdbc/CockroachJdbcPurgeCommandTest.java | 2 +- .../RelationalJdbcBootstrapCommandTest.java | 12 +----- .../jdbc/RelationalJdbcPurgeCommandTest.java | 2 +- .../polaris/admintool/AdminProfiles.java | 43 ------------------- 8 files changed, 18 insertions(+), 88 deletions(-) diff --git a/runtime/admin/src/test/java/org/apache/polaris/admintool/PurgeCommandTestBase.java b/runtime/admin/src/test/java/org/apache/polaris/admintool/PurgeCommandTestBase.java index 758f41d68db..22cf62ab240 100644 --- a/runtime/admin/src/test/java/org/apache/polaris/admintool/PurgeCommandTestBase.java +++ b/runtime/admin/src/test/java/org/apache/polaris/admintool/PurgeCommandTestBase.java @@ -29,13 +29,16 @@ import org.apache.polaris.core.persistence.MetaStoreManagerFactory; import org.apache.polaris.core.persistence.bootstrap.RootCredentialsSet; import org.assertj.core.api.SoftAssertions; -import org.eclipse.microprofile.config.inject.ConfigProperty; import org.junit.jupiter.api.AfterEach; import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; @QuarkusMainTest public abstract class PurgeCommandTestBase { + private static final String REALM1 = "purge-test-realm1"; + private static final String REALM2 = "purge-test-realm2"; + private static final String MISSING_REALM = "purge-test-missing-realm"; + protected SoftAssertions soft; @BeforeEach @@ -48,30 +51,26 @@ void after() { soft.assertAll(); } - void preBootstrap( - @Observes StartupEvent event, - @ConfigProperty(name = "pre-bootstrap", defaultValue = "false") boolean preBootstrap, - MetaStoreManagerFactory metaStoreManagerFactory) { - if (preBootstrap) { - metaStoreManagerFactory.bootstrapRealms( - List.of("realm1", "realm2"), RootCredentialsSet.EMPTY); - } + void preBootstrap(@Observes StartupEvent event, MetaStoreManagerFactory metaStoreManagerFactory) { + metaStoreManagerFactory.bootstrapRealms(List.of(REALM1, REALM2), RootCredentialsSet.EMPTY); } @Test - @Launch(value = {"purge", "-r", "realm1", "-r", "realm2"}) + @Launch(value = {"purge", "-r", REALM1, "-r", REALM2}) public void testPurge(LaunchResult result) { assertThat(result.getOutput()).contains("Purge completed successfully."); } @Test @Launch( - value = {"purge", "-r", "realm3"}, + value = {"purge", "-r", MISSING_REALM}, exitCode = BaseCommand.EXIT_CODE_PURGE_ERROR) public void testPurgeFailure(LaunchResult result) { soft.assertThat(result.getOutput()) .contains( - "Realm realm3 is not bootstrapped, could not load root principal. Please run Bootstrap command."); + "Realm " + + MISSING_REALM + + " is not bootstrapped, could not load root principal. Please run Bootstrap command."); soft.assertThat(result.getErrorOutput()).contains("Purge encountered errors during operation."); } } diff --git a/runtime/admin/src/test/java/org/apache/polaris/admintool/nosql/NoSqlInMemoryPurgeCommandTest.java b/runtime/admin/src/test/java/org/apache/polaris/admintool/nosql/NoSqlInMemoryPurgeCommandTest.java index b3fc169514a..e338477066b 100644 --- a/runtime/admin/src/test/java/org/apache/polaris/admintool/nosql/NoSqlInMemoryPurgeCommandTest.java +++ b/runtime/admin/src/test/java/org/apache/polaris/admintool/nosql/NoSqlInMemoryPurgeCommandTest.java @@ -22,5 +22,5 @@ import org.apache.polaris.admintool.AdminProfiles; import org.apache.polaris.admintool.PurgeCommandTestBase; -@TestProfile(AdminProfiles.PreBootstrappedNoSqlInMemory.class) +@TestProfile(AdminProfiles.NoSqlInMemory.class) class NoSqlInMemoryPurgeCommandTest extends PurgeCommandTestBase {} diff --git a/runtime/admin/src/test/java/org/apache/polaris/admintool/nosql/NoSqlMongoPurgeCommandTest.java b/runtime/admin/src/test/java/org/apache/polaris/admintool/nosql/NoSqlMongoPurgeCommandTest.java index f9cb7425989..124849b9a5d 100644 --- a/runtime/admin/src/test/java/org/apache/polaris/admintool/nosql/NoSqlMongoPurgeCommandTest.java +++ b/runtime/admin/src/test/java/org/apache/polaris/admintool/nosql/NoSqlMongoPurgeCommandTest.java @@ -22,5 +22,5 @@ import org.apache.polaris.admintool.AdminProfiles; import org.apache.polaris.admintool.PurgeCommandTestBase; -@TestProfile(AdminProfiles.PreBootstrappedNoSqlMongo.class) +@TestProfile(AdminProfiles.NoSqlMongo.class) class NoSqlMongoPurgeCommandTest extends PurgeCommandTestBase {} diff --git a/runtime/admin/src/test/java/org/apache/polaris/admintool/relational/jdbc/CockroachJdbcBootstrapCommandTest.java b/runtime/admin/src/test/java/org/apache/polaris/admintool/relational/jdbc/CockroachJdbcBootstrapCommandTest.java index 728e7dd9d01..2e030d70986 100644 --- a/runtime/admin/src/test/java/org/apache/polaris/admintool/relational/jdbc/CockroachJdbcBootstrapCommandTest.java +++ b/runtime/admin/src/test/java/org/apache/polaris/admintool/relational/jdbc/CockroachJdbcBootstrapCommandTest.java @@ -18,26 +18,8 @@ */ package org.apache.polaris.admintool.relational.jdbc; -import static org.assertj.core.api.Assertions.assertThat; - import io.quarkus.test.junit.TestProfile; -import io.quarkus.test.junit.main.LaunchResult; -import io.quarkus.test.junit.main.QuarkusMainLauncher; import org.apache.polaris.admintool.AdminProfiles; -import org.junit.jupiter.api.Test; @TestProfile(AdminProfiles.CockroachJdbc.class) -public class CockroachJdbcBootstrapCommandTest extends RelationalJdbcBootstrapCommandTest { - - @Override - @Test - public void testBootstrapFailsWhenAddingRealmWithDifferentSchemaVersion( - QuarkusMainLauncher launcher) { - // CockroachDB only has schema v4 (no v1, v2 or v3 schemas exist). - // Override to bootstrap with v4, which is the only version available for CockroachDB. - LaunchResult result1 = - launcher.launch("bootstrap", "-v", "4", "-r", "realm1", "-c", "realm1,root,s3cr3t"); - assertThat(result1.exitCode()).isEqualTo(0); - assertThat(result1.getOutput()).contains("Bootstrap completed successfully."); - } -} +public class CockroachJdbcBootstrapCommandTest extends RelationalJdbcBootstrapCommandTest {} diff --git a/runtime/admin/src/test/java/org/apache/polaris/admintool/relational/jdbc/CockroachJdbcPurgeCommandTest.java b/runtime/admin/src/test/java/org/apache/polaris/admintool/relational/jdbc/CockroachJdbcPurgeCommandTest.java index 0c08ca7a847..c3556ec0b94 100644 --- a/runtime/admin/src/test/java/org/apache/polaris/admintool/relational/jdbc/CockroachJdbcPurgeCommandTest.java +++ b/runtime/admin/src/test/java/org/apache/polaris/admintool/relational/jdbc/CockroachJdbcPurgeCommandTest.java @@ -22,5 +22,5 @@ import org.apache.polaris.admintool.AdminProfiles; import org.apache.polaris.admintool.PurgeCommandTestBase; -@TestProfile(AdminProfiles.PreBootstrappedCockroachJdbc.class) +@TestProfile(AdminProfiles.CockroachJdbc.class) public class CockroachJdbcPurgeCommandTest extends PurgeCommandTestBase {} diff --git a/runtime/admin/src/test/java/org/apache/polaris/admintool/relational/jdbc/RelationalJdbcBootstrapCommandTest.java b/runtime/admin/src/test/java/org/apache/polaris/admintool/relational/jdbc/RelationalJdbcBootstrapCommandTest.java index 4824f61163c..68f81073b3f 100644 --- a/runtime/admin/src/test/java/org/apache/polaris/admintool/relational/jdbc/RelationalJdbcBootstrapCommandTest.java +++ b/runtime/admin/src/test/java/org/apache/polaris/admintool/relational/jdbc/RelationalJdbcBootstrapCommandTest.java @@ -31,19 +31,11 @@ public class RelationalJdbcBootstrapCommandTest extends BootstrapCommandTestBase { @Test - public void testBootstrapFailsWhenAddingRealmWithDifferentSchemaVersion( - QuarkusMainLauncher launcher) { - // First, bootstrap the schema to version 1 + public void testBootstrapWithExplicitSchemaVersion(QuarkusMainLauncher launcher) { LaunchResult result1 = - launcher.launch("bootstrap", "-v", "1", "-r", "realm1", "-c", "realm1,root,s3cr3t"); + launcher.launch("bootstrap", "-v", "4", "-r", "realm1", "-c", "realm1,root,s3cr3t"); assertThat(result1.exitCode()).isEqualTo(0); assertThat(result1.getOutput()).contains("Bootstrap completed successfully."); - - // TODO: enable this once we enable postgres container reuse in the same test. - // LaunchResult result2 = launcher.launch("bootstrap", "-v", "2", "-r", "realm2", "-c", - // "realm2,root,s3cr3t"); - // assertThat(result2.exitCode()).isEqualTo(EXIT_CODE_BOOTSTRAP_ERROR); - // assertThat(result2.getOutput()).contains("Cannot bootstrap due to schema version mismatch."); } @Test diff --git a/runtime/admin/src/test/java/org/apache/polaris/admintool/relational/jdbc/RelationalJdbcPurgeCommandTest.java b/runtime/admin/src/test/java/org/apache/polaris/admintool/relational/jdbc/RelationalJdbcPurgeCommandTest.java index a8541c1b2e5..7fabdab8bc5 100644 --- a/runtime/admin/src/test/java/org/apache/polaris/admintool/relational/jdbc/RelationalJdbcPurgeCommandTest.java +++ b/runtime/admin/src/test/java/org/apache/polaris/admintool/relational/jdbc/RelationalJdbcPurgeCommandTest.java @@ -22,5 +22,5 @@ import org.apache.polaris.admintool.AdminProfiles; import org.apache.polaris.admintool.PurgeCommandTestBase; -@TestProfile(AdminProfiles.PreBootstrappedRelationalJdbc.class) +@TestProfile(AdminProfiles.RelationalJdbc.class) public class RelationalJdbcPurgeCommandTest extends PurgeCommandTestBase {} diff --git a/runtime/admin/src/testFixtures/java/org/apache/polaris/admintool/AdminProfiles.java b/runtime/admin/src/testFixtures/java/org/apache/polaris/admintool/AdminProfiles.java index 82c6380b16f..3abe2b906cc 100644 --- a/runtime/admin/src/testFixtures/java/org/apache/polaris/admintool/AdminProfiles.java +++ b/runtime/admin/src/testFixtures/java/org/apache/polaris/admintool/AdminProfiles.java @@ -18,7 +18,6 @@ */ package org.apache.polaris.admintool; -import com.google.common.collect.ImmutableMap; import io.quarkus.test.junit.QuarkusTestProfile; import java.util.List; import java.util.Map; @@ -27,8 +26,6 @@ public final class AdminProfiles { - private static final Map PRE_BOOTSTRAP_CONFIG = Map.of("pre-bootstrap", "true"); - private AdminProfiles() {} public static class NoSqlInMemory implements QuarkusTestProfile { @@ -39,16 +36,6 @@ public Map getConfigOverrides() { } } - public static class PreBootstrappedNoSqlInMemory extends NoSqlInMemory { - @Override - public Map getConfigOverrides() { - return ImmutableMap.builder() - .putAll(super.getConfigOverrides()) - .putAll(PRE_BOOTSTRAP_CONFIG) - .build(); - } - } - public static class NoSqlMongo implements QuarkusTestProfile { @Override public Map getConfigOverrides() { @@ -62,16 +49,6 @@ public List testResources() { } } - public static class PreBootstrappedNoSqlMongo extends NoSqlMongo { - @Override - public Map getConfigOverrides() { - return ImmutableMap.builder() - .putAll(super.getConfigOverrides()) - .putAll(PRE_BOOTSTRAP_CONFIG) - .build(); - } - } - public static class RelationalJdbc implements QuarkusTestProfile { @Override public Map getConfigOverrides() { @@ -93,16 +70,6 @@ public List testResources() { } } - public static class PreBootstrappedRelationalJdbc extends RelationalJdbc { - @Override - public Map getConfigOverrides() { - return ImmutableMap.builder() - .putAll(super.getConfigOverrides()) - .putAll(PRE_BOOTSTRAP_CONFIG) - .build(); - } - } - public static class CockroachJdbc implements QuarkusTestProfile { @Override public Map getConfigOverrides() { @@ -125,14 +92,4 @@ public List testResources() { new TestResourceEntry(CockroachRelationalJdbcLifeCycleManagement.class, Map.of())); } } - - public static class PreBootstrappedCockroachJdbc extends CockroachJdbc { - @Override - public Map getConfigOverrides() { - return ImmutableMap.builder() - .putAll(super.getConfigOverrides()) - .putAll(PRE_BOOTSTRAP_CONFIG) - .build(); - } - } } From 77951089c4df13969d14aa2b380fe36822571170 Mon Sep 17 00:00:00 2001 From: Robert Stupp Date: Fri, 12 Jun 2026 19:38:45 +0200 Subject: [PATCH 15/22] polaris-admin/test: migrate some tests to pure unit tests This change moves a few test cases to `PurgeCommandTest` and the new `BootstrapCommandTest`. Each `@QuarkusMainTest`/`@Launch` test case starts a new Quarkus command tool with fresh test resources. This means that each of these test cases also spawns new Postgres/Cockroach/Mongo container instances, which is time consuming. Runtime of `:polaris-admin:test` on my machine dropped from ~7 minutes to ~4 minutes. --- .../admintool/BootstrapCommandTest.java | 95 +++++++++++++++++++ .../admintool/BootstrapCommandTestBase.java | 61 ------------ .../polaris/admintool/PurgeCommandTest.java | 47 ++++++++- .../admintool/PurgeCommandTestBase.java | 29 ------ 4 files changed, 137 insertions(+), 95 deletions(-) create mode 100644 runtime/admin/src/test/java/org/apache/polaris/admintool/BootstrapCommandTest.java diff --git a/runtime/admin/src/test/java/org/apache/polaris/admintool/BootstrapCommandTest.java b/runtime/admin/src/test/java/org/apache/polaris/admintool/BootstrapCommandTest.java new file mode 100644 index 00000000000..180ed1bb3ed --- /dev/null +++ b/runtime/admin/src/test/java/org/apache/polaris/admintool/BootstrapCommandTest.java @@ -0,0 +1,95 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.polaris.admintool; + +import static org.assertj.core.api.Assertions.assertThat; + +import java.io.PrintWriter; +import java.io.StringWriter; +import org.junit.jupiter.api.Test; +import picocli.CommandLine; + +class BootstrapCommandTest { + + @Test + void testBootstrapInvalidCredentials() { + CommandLine commandLine = new CommandLine(new BootstrapCommand()); + StringWriter err = new StringWriter(); + commandLine.setErr(new PrintWriter(err)); + + int exitCode = commandLine.execute("-r", "realm1", "-c", "invalid syntax"); + + assertThat(exitCode).isEqualTo(BaseCommand.EXIT_CODE_BOOTSTRAP_ERROR); + assertThat(err.toString()) + .contains("Invalid credentials format: invalid syntax") + .contains("Bootstrap encountered errors during operation."); + } + + @Test + void testBootstrapInvalidArguments() { + CommandLine commandLine = new CommandLine(new BootstrapCommand()); + StringWriter err = new StringWriter(); + commandLine.setErr(new PrintWriter(err)); + + int exitCode = commandLine.execute("-r", "realm1", "-f", "/irrelevant"); + + assertThat(exitCode).isEqualTo(BaseCommand.EXIT_CODE_USAGE); + assertThat(err.toString()) + .contains( + "Error: [-r= [-r=]... [-c=]... [-p]] and [[-f=]] are mutually exclusive (specify only one)"); + } + + @Test + void testBootstrapFromInvalidFile() { + CommandLine commandLine = new CommandLine(new BootstrapCommand()); + StringWriter err = new StringWriter(); + commandLine.setErr(new PrintWriter(err)); + + int exitCode = commandLine.execute("-f", "/non/existing/file"); + + assertThat(exitCode).isEqualTo(BaseCommand.EXIT_CODE_BOOTSTRAP_ERROR); + assertThat(err.toString()) + .contains("Failed to read credentials from file:///non/existing/file") + .contains("Bootstrap encountered errors during operation."); + } + + @Test + void testNoPrintCredentialsSystemGenerated() { + CommandLine commandLine = new CommandLine(new BootstrapCommand()); + StringWriter err = new StringWriter(); + commandLine.setErr(new PrintWriter(err)); + + int exitCode = commandLine.execute("-r", "realm1"); + + assertThat(exitCode).isEqualTo(BaseCommand.EXIT_CODE_BOOTSTRAP_ERROR); + assertThat(err.toString()).contains("--credentials").contains("--print-credentials"); + } + + @Test + void testBootstrapInvalidArg() { + CommandLine commandLine = new CommandLine(new BootstrapCommand()); + StringWriter err = new StringWriter(); + commandLine.setErr(new PrintWriter(err)); + + int exitCode = commandLine.execute("-r", "realm1", "--not-real-arg"); + + assertThat(exitCode).isEqualTo(BaseCommand.EXIT_CODE_USAGE); + assertThat(err.toString()).contains("Unknown option: '--not-real-arg'").contains("Usage:"); + } +} diff --git a/runtime/admin/src/test/java/org/apache/polaris/admintool/BootstrapCommandTestBase.java b/runtime/admin/src/test/java/org/apache/polaris/admintool/BootstrapCommandTestBase.java index 9286c53b727..45a401c4f6c 100644 --- a/runtime/admin/src/test/java/org/apache/polaris/admintool/BootstrapCommandTestBase.java +++ b/runtime/admin/src/test/java/org/apache/polaris/admintool/BootstrapCommandTestBase.java @@ -18,8 +18,6 @@ */ package org.apache.polaris.admintool; -import static org.apache.polaris.admintool.BaseCommand.EXIT_CODE_BOOTSTRAP_ERROR; -import static org.apache.polaris.admintool.BaseCommand.EXIT_CODE_USAGE; import static org.assertj.core.api.Assertions.assertThat; import io.quarkus.test.junit.main.Launch; @@ -68,32 +66,6 @@ public void testBootstrapFromCommandLineArguments(LaunchResult result) { .contains("Bootstrap completed successfully."); } - @Test - @Launch( - value = { - "bootstrap", - "-r", - "realm1", - "-c", - "invalid syntax", - }, - exitCode = EXIT_CODE_BOOTSTRAP_ERROR) - public void testBootstrapInvalidCredentials(LaunchResult result) { - assertThat(result.getErrorOutput()) - .contains("Invalid credentials format: invalid syntax") - .contains("Bootstrap encountered errors during operation."); - } - - @Test - @Launch( - value = {"bootstrap", "-r", "realm1", "-f", "/irrelevant"}, - exitCode = EXIT_CODE_USAGE) - public void testBootstrapInvalidArguments(LaunchResult result) { - assertThat(result.getErrorOutput()) - .contains( - "Error: [-r= [-r=]... [-c=]... [-p]] and [[-f=]] are mutually exclusive (specify only one)"); - } - @Test public void testBootstrapFromValidJsonFile(QuarkusMainLauncher launcher) { LaunchResult result = launcher.launch("bootstrap", "-f", json.toString()); @@ -114,15 +86,6 @@ public void testBootstrapFromValidYamlFile(QuarkusMainLauncher launcher) { .contains("Bootstrap completed successfully."); } - @Test - public void testBootstrapFromInvalidFile(QuarkusMainLauncher launcher) { - LaunchResult result = launcher.launch("bootstrap", "-f", "/non/existing/file"); - assertThat(result.exitCode()).isEqualTo(EXIT_CODE_BOOTSTRAP_ERROR); - assertThat(result.getErrorOutput()) - .contains("Failed to read credentials from file:///non/existing/file") - .contains("Bootstrap encountered errors during operation."); - } - @Test @Launch( value = {"bootstrap", "-r", "realm1", "-c", "realm1,client1d,s3cr3t", "--print-credentials"}) @@ -138,30 +101,6 @@ public void testPrintCredentialsSystemGenerated(LaunchResult result) { assertThat(result.getOutput()).contains("realm: realm1 root principal credentials: "); } - @Test - @Launch( - value = {"bootstrap", "-r", "realm1"}, - exitCode = EXIT_CODE_BOOTSTRAP_ERROR) - public void testNoPrintCredentialsSystemGenerated(LaunchResult result) { - assertThat(result.getErrorOutput()).contains("--credentials"); - assertThat(result.getErrorOutput()).contains("--print-credentials"); - } - - @Test - @Launch( - value = { - "bootstrap", - "-r", - "realm1", - "--not-real-arg", - }, - exitCode = EXIT_CODE_USAGE) - public void testBootstrapInvalidArg(LaunchResult result) { - assertThat(result.getErrorOutput()) - .contains("Unknown option: '--not-real-arg'") - .contains("Usage:"); - } - private static Path copyResource(Path temp, String resource) throws IOException { URL source = Objects.requireNonNull(BootstrapCommandTestBase.class.getResource(resource)); Path dest = temp.resolve(resource); diff --git a/runtime/admin/src/test/java/org/apache/polaris/admintool/PurgeCommandTest.java b/runtime/admin/src/test/java/org/apache/polaris/admintool/PurgeCommandTest.java index e2cf48972df..a9cd643b6fb 100644 --- a/runtime/admin/src/test/java/org/apache/polaris/admintool/PurgeCommandTest.java +++ b/runtime/admin/src/test/java/org/apache/polaris/admintool/PurgeCommandTest.java @@ -38,6 +38,33 @@ class PurgeCommandTest { + @Test + void testPurgeFailure() { + String realm = "missing-realm"; + PurgeCommand command = new PurgeCommand(); + command.metaStoreManagerFactory = + new FakeMetaStoreManagerFactory( + Map.of( + realm, + new BaseResult(BaseResult.ReturnStatus.ENTITY_NOT_FOUND, "Not bootstrapped"))); + + CommandLine commandLine = new CommandLine(command); + StringWriter out = new StringWriter(); + StringWriter err = new StringWriter(); + commandLine.setOut(new PrintWriter(out)); + commandLine.setErr(new PrintWriter(err)); + + int exitCode = commandLine.execute("-r", realm); + + assertThat(exitCode).isEqualTo(BaseCommand.EXIT_CODE_PURGE_ERROR); + assertThat(out.toString()) + .contains( + "Realm " + + realm + + " is not bootstrapped, could not load root principal. Please run Bootstrap command."); + assertThat(err.toString()).contains("Purge encountered errors during operation."); + } + /** * When {@code purgeRealms} fails (e.g. database connection failure, permission denied), the * command must surface the underlying exception to stderr so operators can diagnose it, in @@ -48,7 +75,7 @@ void testPurgeFailurePrintsStackTrace() { String failureMessage = "simulated database connection failure"; PurgeCommand command = new PurgeCommand(); command.metaStoreManagerFactory = - new ThrowingMetaStoreManagerFactory(new RuntimeException(failureMessage)); + new FakeMetaStoreManagerFactory(new RuntimeException(failureMessage)); CommandLine commandLine = new CommandLine(command); StringWriter err = new StringWriter(); @@ -63,17 +90,27 @@ void testPurgeFailurePrintsStackTrace() { .contains("Purge encountered errors during operation."); } - /** Minimal {@link MetaStoreManagerFactory} whose {@code purgeRealms} always throws. */ - private static final class ThrowingMetaStoreManagerFactory implements MetaStoreManagerFactory { + /** Minimal {@link MetaStoreManagerFactory} for testing purge command results and failures. */ + private static final class FakeMetaStoreManagerFactory implements MetaStoreManagerFactory { + private final Map results; private final RuntimeException failure; - private ThrowingMetaStoreManagerFactory(RuntimeException failure) { + private FakeMetaStoreManagerFactory(Map results) { + this.results = results; + this.failure = null; + } + + private FakeMetaStoreManagerFactory(RuntimeException failure) { + this.results = null; this.failure = failure; } @Override public Map purgeRealms(Iterable realms) { - throw failure; + if (failure != null) { + throw failure; + } + return results; } @Override diff --git a/runtime/admin/src/test/java/org/apache/polaris/admintool/PurgeCommandTestBase.java b/runtime/admin/src/test/java/org/apache/polaris/admintool/PurgeCommandTestBase.java index 22cf62ab240..2d84f5b02b2 100644 --- a/runtime/admin/src/test/java/org/apache/polaris/admintool/PurgeCommandTestBase.java +++ b/runtime/admin/src/test/java/org/apache/polaris/admintool/PurgeCommandTestBase.java @@ -28,28 +28,12 @@ import java.util.List; import org.apache.polaris.core.persistence.MetaStoreManagerFactory; import org.apache.polaris.core.persistence.bootstrap.RootCredentialsSet; -import org.assertj.core.api.SoftAssertions; -import org.junit.jupiter.api.AfterEach; -import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; @QuarkusMainTest public abstract class PurgeCommandTestBase { private static final String REALM1 = "purge-test-realm1"; private static final String REALM2 = "purge-test-realm2"; - private static final String MISSING_REALM = "purge-test-missing-realm"; - - protected SoftAssertions soft; - - @BeforeEach - void setup() { - soft = new SoftAssertions(); - } - - @AfterEach - void after() { - soft.assertAll(); - } void preBootstrap(@Observes StartupEvent event, MetaStoreManagerFactory metaStoreManagerFactory) { metaStoreManagerFactory.bootstrapRealms(List.of(REALM1, REALM2), RootCredentialsSet.EMPTY); @@ -60,17 +44,4 @@ void preBootstrap(@Observes StartupEvent event, MetaStoreManagerFactory metaStor public void testPurge(LaunchResult result) { assertThat(result.getOutput()).contains("Purge completed successfully."); } - - @Test - @Launch( - value = {"purge", "-r", MISSING_REALM}, - exitCode = BaseCommand.EXIT_CODE_PURGE_ERROR) - public void testPurgeFailure(LaunchResult result) { - soft.assertThat(result.getOutput()) - .contains( - "Realm " - + MISSING_REALM - + " is not bootstrapped, could not load root principal. Please run Bootstrap command."); - soft.assertThat(result.getErrorOutput()).contains("Purge encountered errors during operation."); - } } From a32b51154353184140db66f26468650ac2ec3161 Mon Sep 17 00:00:00 2001 From: Robert Stupp Date: Sat, 13 Jun 2026 15:09:33 +0200 Subject: [PATCH 16/22] polaris-runtime-service/test: slight test-profile improvement Save one profile (and its Quarkus augmentation), move more test profiles to `Profiles`. --- .../org/apache/polaris/service/Profiles.java | 52 +++++++++++++++++++ .../it/ApplicationIntegrationTest.java | 28 +--------- .../it/ManagementServiceIntegrationTest.java | 24 ++------- .../it/PolicyServiceIntegrationTest.java | 23 ++------ .../it/RestCatalogFileIntegrationTest.java | 26 +--------- .../RestCatalogViewFileIntegrationTest.java | 22 ++------ 6 files changed, 65 insertions(+), 110 deletions(-) diff --git a/runtime/service/src/test/java/org/apache/polaris/service/Profiles.java b/runtime/service/src/test/java/org/apache/polaris/service/Profiles.java index ef1d6cab2f2..db3476e9fa3 100644 --- a/runtime/service/src/test/java/org/apache/polaris/service/Profiles.java +++ b/runtime/service/src/test/java/org/apache/polaris/service/Profiles.java @@ -50,6 +50,58 @@ public Map getConfigOverrides() { } } + public static class ApplicationIntegrationProfile extends DefaultProfile { + @Override + public Map getConfigOverrides() { + return ImmutableMap.builder() + .putAll(super.getConfigOverrides()) + .put("quarkus.http.limits.max-body-size", "1000000") + .put("polaris.realm-context.realms", "POLARIS,OTHER") + .put("polaris.features.\"ALLOW_OVERLAPPING_CATALOG_URLS\"", "true") + .put("polaris.features.\"SKIP_CREDENTIAL_SUBSCOPING_INDIRECTION\"", "true") + .build(); + } + } + + public static class ManagementIntegrationProfile implements QuarkusTestProfile { + @Override + public Map getConfigOverrides() { + return Map.of( + "polaris.features.\"ALLOW_OVERLAPPING_CATALOG_URLS\"", + "true", + "polaris.features.\"ENFORCE_PRINCIPAL_CREDENTIAL_ROTATION_REQUIRED_CHECKING\"", + "true", + "polaris.storage.gcp.token", + "token", + "polaris.storage.gcp.lifespan", + "PT1H"); + } + } + + public static class RestCatalogFileIntegrationProfile extends DefaultProfile { + @Override + public Map getConfigOverrides() { + return ImmutableMap.builder() + .putAll(super.getConfigOverrides()) + .put("polaris.features.\"ALLOW_EXTERNAL_CATALOG_CREDENTIAL_VENDING\"", "false") + .put("polaris.features.\"ALLOW_NAMESPACE_CUSTOM_LOCATION\"", "true") + .build(); + } + } + + public static class RestCatalogViewFileIntegrationProfile implements QuarkusTestProfile { + @Override + public Map getConfigOverrides() { + return Map.of( + "polaris.features.\"ALLOW_INSECURE_STORAGE_TYPES\"", + "true", + "polaris.features.\"SUPPORTED_CATALOG_STORAGE_TYPES\"", + "[\"FILE\"]", + "polaris.readiness.ignore-severe-issues", + "true"); + } + } + public static class DefaultNoSqlProfile extends DefaultProfile { @Override public Map getConfigOverrides() { diff --git a/runtime/service/src/test/java/org/apache/polaris/service/it/ApplicationIntegrationTest.java b/runtime/service/src/test/java/org/apache/polaris/service/it/ApplicationIntegrationTest.java index b0f27fba592..214276723bc 100644 --- a/runtime/service/src/test/java/org/apache/polaris/service/it/ApplicationIntegrationTest.java +++ b/runtime/service/src/test/java/org/apache/polaris/service/it/ApplicationIntegrationTest.java @@ -24,7 +24,6 @@ import com.auth0.jwt.JWT; import com.auth0.jwt.algorithms.Algorithm; import io.quarkus.test.junit.QuarkusTest; -import io.quarkus.test.junit.QuarkusTestProfile; import io.quarkus.test.junit.TestProfile; import java.io.IOException; import java.time.Instant; @@ -37,38 +36,15 @@ import org.apache.iceberg.rest.auth.AuthSession; import org.apache.iceberg.rest.auth.OAuth2Util; import org.apache.iceberg.rest.responses.OAuthTokenResponse; +import org.apache.polaris.service.Profiles; import org.apache.polaris.service.it.env.ClientCredentials; import org.apache.polaris.service.it.env.PolarisApiEndpoints; import org.apache.polaris.service.it.test.PolarisApplicationIntegrationTest; import org.junit.jupiter.api.Test; @QuarkusTest -@TestProfile(ApplicationIntegrationTest.Profile.class) +@TestProfile(Profiles.ApplicationIntegrationProfile.class) public class ApplicationIntegrationTest extends PolarisApplicationIntegrationTest { - - public static class Profile implements QuarkusTestProfile { - @Override - public Map getConfigOverrides() { - return Map.of( - "quarkus.http.limits.max-body-size", - "1000000", - "polaris.realm-context.realms", - "POLARIS,OTHER", - "polaris.features.\"ALLOW_OVERLAPPING_CATALOG_URLS\"", - "true", - "polaris.features.\"SKIP_CREDENTIAL_SUBSCOPING_INDIRECTION\"", - "true", - "polaris.features.\"ALLOW_SPECIFYING_FILE_IO_IMPL\"", - "true", - "polaris.features.\"ALLOW_INSECURE_STORAGE_TYPES\"", - "true", - "polaris.features.\"SUPPORTED_CATALOG_STORAGE_TYPES\"", - "[\"FILE\",\"S3\"]", - "polaris.readiness.ignore-severe-issues", - "true"); - } - } - @Test public void testIcebergRestApiRefreshExpiredToken( PolarisApiEndpoints endpoints, ClientCredentials clientCredentials) throws IOException { diff --git a/runtime/service/src/test/java/org/apache/polaris/service/it/ManagementServiceIntegrationTest.java b/runtime/service/src/test/java/org/apache/polaris/service/it/ManagementServiceIntegrationTest.java index 057e251d95b..8a66b5945e4 100644 --- a/runtime/service/src/test/java/org/apache/polaris/service/it/ManagementServiceIntegrationTest.java +++ b/runtime/service/src/test/java/org/apache/polaris/service/it/ManagementServiceIntegrationTest.java @@ -19,28 +19,10 @@ package org.apache.polaris.service.it; import io.quarkus.test.junit.QuarkusTest; -import io.quarkus.test.junit.QuarkusTestProfile; import io.quarkus.test.junit.TestProfile; -import java.util.Map; +import org.apache.polaris.service.Profiles; import org.apache.polaris.service.it.test.PolarisManagementServiceIntegrationTest; @QuarkusTest -@TestProfile(ManagementServiceIntegrationTest.Profile.class) -public class ManagementServiceIntegrationTest extends PolarisManagementServiceIntegrationTest { - - public static class Profile implements QuarkusTestProfile { - - @Override - public Map getConfigOverrides() { - return Map.of( - "polaris.features.\"ALLOW_OVERLAPPING_CATALOG_URLS\"", - "true", - "polaris.features.\"ENFORCE_PRINCIPAL_CREDENTIAL_ROTATION_REQUIRED_CHECKING\"", - "true", - "polaris.storage.gcp.token", - "token", - "polaris.storage.gcp.lifespan", - "PT1H"); - } - } -} +@TestProfile(Profiles.ManagementIntegrationProfile.class) +public class ManagementServiceIntegrationTest extends PolarisManagementServiceIntegrationTest {} diff --git a/runtime/service/src/test/java/org/apache/polaris/service/it/PolicyServiceIntegrationTest.java b/runtime/service/src/test/java/org/apache/polaris/service/it/PolicyServiceIntegrationTest.java index 280f381bbf1..46914160197 100644 --- a/runtime/service/src/test/java/org/apache/polaris/service/it/PolicyServiceIntegrationTest.java +++ b/runtime/service/src/test/java/org/apache/polaris/service/it/PolicyServiceIntegrationTest.java @@ -19,27 +19,10 @@ package org.apache.polaris.service.it; import io.quarkus.test.junit.QuarkusTest; -import io.quarkus.test.junit.QuarkusTestProfile; import io.quarkus.test.junit.TestProfile; -import java.util.Map; +import org.apache.polaris.service.Profiles; import org.apache.polaris.service.it.test.PolarisPolicyServiceIntegrationTest; @QuarkusTest -@TestProfile(PolicyServiceIntegrationTest.Profile.class) -public class PolicyServiceIntegrationTest extends PolarisPolicyServiceIntegrationTest { - - public static class Profile implements QuarkusTestProfile { - @Override - public Map getConfigOverrides() { - return Map.of( - "polaris.features.\"ALLOW_SPECIFYING_FILE_IO_IMPL\"", - "true", - "polaris.features.\"ALLOW_INSECURE_STORAGE_TYPES\"", - "true", - "polaris.features.\"SUPPORTED_CATALOG_STORAGE_TYPES\"", - "[\"FILE\",\"S3\"]", - "polaris.readiness.ignore-severe-issues", - "true"); - } - } -} +@TestProfile(Profiles.DefaultProfile.class) +public class PolicyServiceIntegrationTest extends PolarisPolicyServiceIntegrationTest {} diff --git a/runtime/service/src/test/java/org/apache/polaris/service/it/RestCatalogFileIntegrationTest.java b/runtime/service/src/test/java/org/apache/polaris/service/it/RestCatalogFileIntegrationTest.java index c08d81e2f66..f30740bf96b 100644 --- a/runtime/service/src/test/java/org/apache/polaris/service/it/RestCatalogFileIntegrationTest.java +++ b/runtime/service/src/test/java/org/apache/polaris/service/it/RestCatalogFileIntegrationTest.java @@ -19,39 +19,17 @@ package org.apache.polaris.service.it; import io.quarkus.test.junit.QuarkusTest; -import io.quarkus.test.junit.QuarkusTestProfile; import io.quarkus.test.junit.TestProfile; import io.smallrye.common.annotation.Identifier; import jakarta.inject.Inject; -import java.util.Map; +import org.apache.polaris.service.Profiles; import org.apache.polaris.service.it.test.PolarisRestCatalogFileIntegrationTest; import org.apache.polaris.service.task.TaskErrorHandler; import org.junit.jupiter.api.AfterEach; @QuarkusTest -@TestProfile(RestCatalogFileIntegrationTest.Profile.class) +@TestProfile(Profiles.RestCatalogFileIntegrationProfile.class) public class RestCatalogFileIntegrationTest extends PolarisRestCatalogFileIntegrationTest { - - public static class Profile implements QuarkusTestProfile { - - @Override - public Map getConfigOverrides() { - return Map.of( - "polaris.features.\"ALLOW_SPECIFYING_FILE_IO_IMPL\"", - "true", - "polaris.features.\"ALLOW_EXTERNAL_CATALOG_CREDENTIAL_VENDING\"", - "false", - "polaris.features.\"ALLOW_INSECURE_STORAGE_TYPES\"", - "true", - "polaris.features.\"SUPPORTED_CATALOG_STORAGE_TYPES\"", - "[\"FILE\",\"S3\"]", - "polaris.readiness.ignore-severe-issues", - "true", - "polaris.features.\"ALLOW_NAMESPACE_CUSTOM_LOCATION\"", - "true"); - } - } - @Inject @Identifier("task-error-handler") TaskErrorHandler taskErrorHandler; diff --git a/runtime/service/src/test/java/org/apache/polaris/service/it/RestCatalogViewFileIntegrationTest.java b/runtime/service/src/test/java/org/apache/polaris/service/it/RestCatalogViewFileIntegrationTest.java index 79114767a3f..204921a1beb 100644 --- a/runtime/service/src/test/java/org/apache/polaris/service/it/RestCatalogViewFileIntegrationTest.java +++ b/runtime/service/src/test/java/org/apache/polaris/service/it/RestCatalogViewFileIntegrationTest.java @@ -19,27 +19,11 @@ package org.apache.polaris.service.it; import io.quarkus.test.junit.QuarkusTest; -import io.quarkus.test.junit.QuarkusTestProfile; import io.quarkus.test.junit.TestProfile; -import java.util.Map; +import org.apache.polaris.service.Profiles; import org.apache.polaris.service.it.test.PolarisRestCatalogViewFileIntegrationTestBase; @QuarkusTest -@TestProfile(RestCatalogViewFileIntegrationTest.Profile.class) +@TestProfile(Profiles.RestCatalogViewFileIntegrationProfile.class) public class RestCatalogViewFileIntegrationTest - extends PolarisRestCatalogViewFileIntegrationTestBase { - - public static class Profile implements QuarkusTestProfile { - - @Override - public Map getConfigOverrides() { - return Map.of( - "polaris.features.\"ALLOW_INSECURE_STORAGE_TYPES\"", - "true", - "polaris.features.\"SUPPORTED_CATALOG_STORAGE_TYPES\"", - "[\"FILE\"]", - "polaris.readiness.ignore-severe-issues", - "true"); - } - } -} + extends PolarisRestCatalogViewFileIntegrationTestBase {} From 8d415736fc8d15bbe9417b786930f86994fcca2c Mon Sep 17 00:00:00 2001 From: Robert Stupp Date: Fri, 12 Jun 2026 19:18:36 +0200 Subject: [PATCH 17/22] polaris-runtime,admin/testing: Update fork/jvm settings Use `forkEvery = 1` only for `:polaris-runtime-service:test`. Use enlarged heap only for `:polaris-runtime-service:test` and `:polaris-admin:test`. `polaris-runtime-service`'s `intTest` and `cloudTest` and `:polaris-admin:test` will use `forkEvery=0` --- .../main/kotlin/polaris-runtime.gradle.kts | 12 ----- runtime/admin/build.gradle.kts | 7 +++ runtime/service/build.gradle.kts | 52 ++++++++++--------- 3 files changed, 34 insertions(+), 37 deletions(-) diff --git a/build-logic/src/main/kotlin/polaris-runtime.gradle.kts b/build-logic/src/main/kotlin/polaris-runtime.gradle.kts index 0d320926ead..b7a72081b42 100644 --- a/build-logic/src/main/kotlin/polaris-runtime.gradle.kts +++ b/build-logic/src/main/kotlin/polaris-runtime.gradle.kts @@ -112,20 +112,8 @@ tasks.withType(Test::class.java).configureEach { // Gradle's Jacoco plugin doesn't work well with Quarkus's test coverage extensions.configure(JacocoTaskExtension::class) { isEnabled = false } - // Quarkus tests run "in isolated class loaders", which means that class-statically active - // resources pile up used JVM, as those classes cannot be GC'd. - // Examples of those statically held active resources are: - // - Iceberg's worker pools (thread pools, executors, etc.) - // - Hadoop's stats-cleaner (org.apache.hadoop.fs.FileSystem.Statistics.STATS_DATA_CLEANER) - // - Guava's 'MoreExecutors' (via Iceberg `ThreadPools`)` - // Forcing a new JVM after each test class works around this issue. - forkEvery = 1 - maxParallelForks = 1 - // enlarge the max heap size to avoid out of memory error - maxHeapSize = "4g" - // Silence the 'OpenJDK 64-Bit Server VM warning: Sharing is only supported for boot loader // classes because bootstrap classpath has been appended' warning from OpenJDK. jvmArgs("-Xshare:off") diff --git a/runtime/admin/build.gradle.kts b/runtime/admin/build.gradle.kts index f751e8c7000..5dae356cd81 100644 --- a/runtime/admin/build.gradle.kts +++ b/runtime/admin/build.gradle.kts @@ -107,3 +107,10 @@ artifacts { add("distributionElements", layout.buildDirectory.dir("quarkus-app")) { builtBy("quarkusBuild") } add("licenseNoticeElements", layout.projectDirectory.dir("distribution")) } + +tasks.withType().configureEach { + forkEvery = 0 + + // enlarge the max heap size to avoid out of memory error + maxHeapSize = "4g" +} diff --git a/runtime/service/build.gradle.kts b/runtime/service/build.gradle.kts index 27d13eccec7..d3608738c7a 100644 --- a/runtime/service/build.gradle.kts +++ b/runtime/service/build.gradle.kts @@ -211,19 +211,6 @@ dependencies { tasks.named("javadoc") { dependsOn("jandex") } -tasks.withType(Test::class.java).configureEach { - if (System.getenv("AWS_REGION") == null) { - environment("AWS_REGION", "us-west-2") - } - // Note: the test secrets are referenced in - // org.apache.polaris.service.it.ServerManager - environment("POLARIS_BOOTSTRAP_CREDENTIALS", "POLARIS,test-admin,test-secret") - jvmArgs("--add-exports", "java.base/sun.nio.ch=ALL-UNNAMED") - // Need to allow a java security manager after Java 21, for Subject.getSubject to work - // "getSubject is supported only if a security manager is allowed". - systemProperty("java.security.manager", "allow") -} - val buildDir = project.layout.buildDirectory // Same issue as above: allow a java security manager after Java 21 @@ -243,27 +230,42 @@ val quarkusTestArgLine = val quarkusProperties = providers.systemPropertiesPrefixedBy("quarkus.") -listOf("intTest", "cloudTest").forEach { - tasks.named(it).configure(IntTestConfig(buildDir, quarkusProperties, quarkusTestArgLine)) -} +tasks.withType(Test::class.java).configureEach { + if (System.getenv("AWS_REGION") == null) { + environment("AWS_REGION", "us-west-2") + } + // Note: the test secrets are referenced in + // org.apache.polaris.service.it.ServerManager + environment("POLARIS_BOOTSTRAP_CREDENTIALS", "POLARIS,test-admin,test-secret") + jvmArgs("--add-exports", "java.base/sun.nio.ch=ALL-UNNAMED") + // Need to allow a java security manager after Java 21, for Subject.getSubject to work + // "getSubject is supported only if a security manager is allowed". + systemProperty("java.security.manager", "allow") -class IntTestConfig( - @get:Internal val buildDir: DirectoryProperty, - @get:Input val quarkusProperties: Provider>, - @get:Input val quarkusTestArgLine: String, -) : Action { - override fun execute(t: Test) { - t.maxParallelForks = 1 + // Quarkus tests run "in isolated class loaders", which means that class-statically active + // resources pile up used JVM, as those classes cannot be GC'd. + // Examples of those statically held active resources are: + // - Iceberg's worker pools (thread pools, executors, etc.) + // - Hadoop's stats-cleaner (org.apache.hadoop.fs.FileSystem.Statistics.STATS_DATA_CLEANER) + // - Guava's 'MoreExecutors' (via Iceberg `ThreadPools`)` + // Forcing a new JVM after each test class works around this issue. + if ("test" == name) { + forkEvery = 1 + + // enlarge the max heap size to avoid out of memory error + maxHeapSize = "4g" + } + if ("intTest" == name || "cloutTest" == name) { val logsDir = buildDir.map { b -> b.asFile.resolve("logs") } // JVM arguments provider does not interfere with Gradle's cache keys - t.jvmArgumentProviders.add( + jvmArgumentProviders.add( IntTestArgumentProvider(logsDir, quarkusProperties, quarkusTestArgLine) ) // delete files from previous runs - t.doFirst(IntTestCleanup(logsDir, buildDir)) + doFirst(IntTestCleanup(logsDir, buildDir)) } } From 2b1f36a9f8fca3d64fa27897711b4444e9b11161 Mon Sep 17 00:00:00 2001 From: Robert Stupp Date: Sat, 13 Jun 2026 10:50:12 +0200 Subject: [PATCH 18/22] Build: remove another usage of `Project.extra` * Replaced with simple `noSourceCheckProjects` in `Utilities.kt` * Evaluate in polaris-spotless and polaris-java convention plugins --- build-logic/src/main/kotlin/Utilities.kt | 2 ++ .../src/main/kotlin/polaris-java.gradle.kts | 10 ++++++++-- .../src/main/kotlin/polaris-root.gradle.kts | 18 +----------------- .../main/kotlin/polaris-spotless.gradle.kts | 16 ++++++++++++++-- settings.gradle.kts | 9 --------- 5 files changed, 25 insertions(+), 30 deletions(-) diff --git a/build-logic/src/main/kotlin/Utilities.kt b/build-logic/src/main/kotlin/Utilities.kt index f75ff5b3f8c..638ad515f74 100644 --- a/build-logic/src/main/kotlin/Utilities.kt +++ b/build-logic/src/main/kotlin/Utilities.kt @@ -17,6 +17,8 @@ import org.gradle.api.Project import org.gradle.process.JavaForkOptions +val noSourceCheckProjects = listOf(":polaris-spark-3.5_2.13") + /** * Extract the scala version from polaris spark project, and points the build directory to a sub-dir * that uses scala version as name. The polaris spark project name is in format of diff --git a/build-logic/src/main/kotlin/polaris-java.gradle.kts b/build-logic/src/main/kotlin/polaris-java.gradle.kts index e2976b8cdac..ddf86326f25 100644 --- a/build-logic/src/main/kotlin/polaris-java.gradle.kts +++ b/build-logic/src/main/kotlin/polaris-java.gradle.kts @@ -78,8 +78,14 @@ checkstyle { maxWarnings = 0 } -// Ensure Checkstyle runs after jandex to avoid task dependency issues -tasks.withType().configureEach { tasks.findByName("jandex")?.let { mustRunAfter(it) } } +tasks.withType().configureEach { + if (noSourceCheckProjects.contains(project.path)) { + enabled = false + } else { + // Ensure Checkstyle runs after jandex to avoid task dependency issues + tasks.findByName("jandex")?.let { mustRunAfter(it) } + } +} tasks.withType(JavaCompile::class.java).configureEach { options.compilerArgs.addAll( diff --git a/build-logic/src/main/kotlin/polaris-root.gradle.kts b/build-logic/src/main/kotlin/polaris-root.gradle.kts index 778cfd449c0..34fac94679b 100644 --- a/build-logic/src/main/kotlin/polaris-root.gradle.kts +++ b/build-logic/src/main/kotlin/polaris-root.gradle.kts @@ -25,28 +25,12 @@ import publishing.PublishingHelperPlugin plugins { id("polaris-base") - id("com.diffplug.spotless") + id("polaris-spotless") id("org.jetbrains.gradle.plugin.idea-ext") } apply() -if (!project.extra.has("duplicated-project-sources")) { - spotless { - kotlinGradle { - ktfmt().googleStyle() - // licenseHeaderFile(rootProject.file("codestyle/copyright-header-java.txt"), "$") - target( - "*.gradle.kts", - "build-logic/*.gradle.kts", - "build-logic/src/**/*.kt*", - "gradle/server-test-runner/**/*.gradle.kts", - "gradle/server-test-runner/src/**/*.kt*", - ) - } - } -} - if (providers.systemProperty("idea.sync.active").getOrElse("false").toBoolean()) { idea { module { diff --git a/build-logic/src/main/kotlin/polaris-spotless.gradle.kts b/build-logic/src/main/kotlin/polaris-spotless.gradle.kts index b6ae54efa6b..a6ccae867bd 100644 --- a/build-logic/src/main/kotlin/polaris-spotless.gradle.kts +++ b/build-logic/src/main/kotlin/polaris-spotless.gradle.kts @@ -20,8 +20,20 @@ plugins { id("com.diffplug.spotless") } // skip spotless check for duplicated projects -if (!project.extra.has("duplicated-project-sources")) { - spotless { +spotless { + if (project.path == ":") { + // Root project + kotlinGradle { + ktfmt().googleStyle() + target( + "*.gradle.kts", + "build-logic/*.gradle.kts", + "build-logic/src/**/*.kt*", + "gradle/server-test-runner/**/*.gradle.kts", + "gradle/server-test-runner/src/**/*.kt*", + ) + } + } else if (!noSourceCheckProjects.contains(project.path)) { java { target("src/*/java/**/*.java") googleJavaFormat() diff --git a/settings.gradle.kts b/settings.gradle.kts index 5eea168385a..7a665ded3d5 100644 --- a/settings.gradle.kts +++ b/settings.gradle.kts @@ -57,10 +57,6 @@ val baseVersion = layout.rootDirectory.file("version.txt").asFile.readText().tri gradle.beforeProject { version = baseVersion group = "org.apache.polaris" - - if (noSourceChecksProjects.contains(this.path)) { - project.extra["duplicated-project-sources"] = true - } } fun loadProperties(file: File): Properties { @@ -89,9 +85,6 @@ val polarisSparkDir = "plugins/spark" val sparkScalaVersions = loadProperties(file("${polarisSparkDir}/spark-scala.properties")) val sparkVersions = sparkScalaVersions["sparkVersions"].toString().split(",").map { it.trim() } -// records the spark projects that maps to the same project dir -val noSourceChecksProjects = mutableSetOf() - for (sparkVersion in sparkVersions) { val scalaVersionsKey = "scalaVersions.${sparkVersion}" val scalaVersionsStr = sparkScalaVersions[scalaVersionsKey].toString() @@ -105,8 +98,6 @@ for (sparkVersion in sparkVersions) { ) if (first) { first = false - } else { - noSourceChecksProjects.add(":$sparkArtifactId") } // Skip all duplicated spark client projects while using Intelij IDE. // This is to avoid problems during dependency analysis and sync when From 5e563ef2306152bd2d48554154880a71dcf238de Mon Sep 17 00:00:00 2001 From: Robert Stupp Date: Sat, 13 Jun 2026 11:21:23 +0200 Subject: [PATCH 19/22] Build: Truely memoize errorprone config --- .../src/main/kotlin/polaris-java.gradle.kts | 43 +++++++++++++------ 1 file changed, 31 insertions(+), 12 deletions(-) diff --git a/build-logic/src/main/kotlin/polaris-java.gradle.kts b/build-logic/src/main/kotlin/polaris-java.gradle.kts index ddf86326f25..b498824f7e0 100644 --- a/build-logic/src/main/kotlin/polaris-java.gradle.kts +++ b/build-logic/src/main/kotlin/polaris-java.gradle.kts @@ -95,22 +95,41 @@ tasks.withType(JavaCompile::class.java).configureEach { options.errorprone.disableWarningsInGeneratedCode = true options.errorprone.excludedPaths = ".*/${project.layout.buildDirectory.get().asFile.relativeTo(projectDir)}/generated(-openapi)?/.*" - val errorproneRules = rootProject.projectDir.resolve("codestyle/errorprone-rules.properties") + val errorproneRules = + rootProject.layout.projectDirectory.file("codestyle/errorprone-rules.properties") inputs.file(errorproneRules).withPathSensitivity(PathSensitivity.RELATIVE) - options.errorprone.checks.putAll(provider { memoizedErrorproneRules(errorproneRules) }) + options.errorprone.checks.putAll( + provider { + val service = + project.gradle.sharedServices.registerIfAbsent( + "errorProneConfig", + ErrorProneConfigService::class.java, + ) { + parameters.configFile = errorproneRules + } + service.get().errorproneConfig + } + ) } -private fun memoizedErrorproneRules(rulesFile: File): Map = - rulesFile.reader().use { - val rules = Properties() - rules.load(it) - rules - .mapKeys { e -> (e.key as String).trim() } - .mapValues { e -> (e.value as String).trim() } - .filter { e -> e.key.isNotEmpty() && e.value.isNotEmpty() } - .mapValues { e -> CheckSeverity.valueOf(e.value) } - .toMap() +abstract class ErrorProneConfigService : BuildService { + interface Parameters : BuildServiceParameters { + val configFile: RegularFileProperty + } + + val errorproneConfig: Map by lazy { + parameters.configFile.get().asFile.reader().use { + val rules = Properties() + rules.load(it) + rules + .mapKeys { e -> (e.key as String).trim() } + .mapValues { e -> (e.value as String).trim() } + .filter { e -> e.key.isNotEmpty() && e.value.isNotEmpty() } + .mapValues { e -> CheckSeverity.valueOf(e.value) } + .toMap() + } } +} tasks.register("compileAll") { group = "build" From 3c856b4a9ed3491da9a773c6520cb7bb529da1eb Mon Sep 17 00:00:00 2001 From: Robert Stupp Date: Sat, 13 Jun 2026 11:37:24 +0200 Subject: [PATCH 20/22] Build: refactor some more eager resolutions Makes a couple of eager resolutions across the build scripts lazily evaluated. This should save a bunch of potentially unnecessary resolutions, depending on the Gradle invocation. --- bom/build.gradle.kts | 2 +- build-logic/src/main/kotlin/polaris-java.gradle.kts | 4 ++-- .../src/main/kotlin/polaris-runtime.gradle.kts | 12 ++++++------ .../main/kotlin/publishing/PublishingHelperPlugin.kt | 2 +- build-logic/src/main/kotlin/publishing/shadowPub.kt | 1 - .../nosql/persistence/correctness/build.gradle.kts | 6 ++++-- plugins/spark/v3.5/spark/build.gradle.kts | 2 +- plugins/spark/v4.0/spark/build.gradle.kts | 2 +- runtime/spark-tests/build.gradle.kts | 8 ++++---- runtime/test-common/build.gradle.kts | 2 +- tools/version/build.gradle.kts | 2 +- 11 files changed, 22 insertions(+), 21 deletions(-) diff --git a/bom/build.gradle.kts b/bom/build.gradle.kts index 0a9530f60cc..bae2bb969c8 100644 --- a/bom/build.gradle.kts +++ b/bom/build.gradle.kts @@ -128,7 +128,7 @@ tasks.register("verifyBomDependencies") { configurations.api.map { it.dependencyConstraints.map { "${it.group}:${it.name}" } } ) projectCoordinatesByPath.set( - rootProject.allprojects.associate { it.path to "${it.group}:${it.name}" } + provider { rootProject.allprojects.associate { it.path to "${it.group}:${it.name}" } } ) excludedProjectPaths.set( setOf( diff --git a/build-logic/src/main/kotlin/polaris-java.gradle.kts b/build-logic/src/main/kotlin/polaris-java.gradle.kts index b498824f7e0..d9df2b3602b 100644 --- a/build-logic/src/main/kotlin/polaris-java.gradle.kts +++ b/build-logic/src/main/kotlin/polaris-java.gradle.kts @@ -164,7 +164,7 @@ testing { // Special handling for test-suites with names containing `manualtest`, which are intended to // be run on demand rather than implicitly via `check`. if (!name.lowercase().contains("manualtest")) { - targets.all { + targets.configureEach { if (testTask.name != "test") { testTask.configure { shouldRunAfter("test") } tasks.named("check").configure { dependsOn(testTask) } @@ -298,7 +298,7 @@ class BannedDependencies( } fun applyTo(configurations: ConfigurationContainer) { - configurations.all { applyTo(this) } + configurations.configureEach { applyTo(this) } } } diff --git a/build-logic/src/main/kotlin/polaris-runtime.gradle.kts b/build-logic/src/main/kotlin/polaris-runtime.gradle.kts index b7a72081b42..036a86c90a1 100644 --- a/build-logic/src/main/kotlin/polaris-runtime.gradle.kts +++ b/build-logic/src/main/kotlin/polaris-runtime.gradle.kts @@ -33,7 +33,7 @@ plugins { id("polaris-server") } testing { suites { withType { - targets.all { + targets.configureEach { testTask.configure { systemProperty("java.util.logging.manager", "org.jboss.logmanager.LogManager") // Enable automatic extension detection to execute GradleDuplicateLoggingWorkaround @@ -45,7 +45,7 @@ testing { } fun intTestSuiteConfigure(testSuite: JvmTestSuite) = testSuite.run { - targets.all { + targets.configureEach { testTask.configure { // For Quarkus... // @@ -61,12 +61,12 @@ testing { dependsOn("compileQuarkusTestGeneratedSourcesJava") } configurations.named(sources.runtimeOnlyConfigurationName).configure { - extendsFrom(configurations.getByName("testRuntimeOnly")) + extendsFrom(configurations.named("testRuntimeOnly")) } configurations.named(sources.implementationConfigurationName).configure { // Let the test's implementation config extend testImplementation, so it also inherits the // project's "main" implementation dependencies (not just the "api" configuration) - extendsFrom(configurations.getByName("testImplementation")) + extendsFrom(configurations.named("testImplementation")) } sources { java.srcDirs(tasks.named("quarkusGenerateCodeTests")) } } @@ -82,7 +82,7 @@ dependencies { implementation("org.jboss.slf4j:slf4j-jboss-logmanager") } -configurations.all { +configurations.configureEach { // Validate that Logback dependencies are not used in Quarkus modules. dependencies.configureEach { if (group == "ch.qos.logback") { @@ -95,7 +95,7 @@ configurations.all { } configurations.named("intTestRuntimeOnly").configure { - extendsFrom(configurations.getByName("testRuntimeOnly")) + extendsFrom(configurations.named("testRuntimeOnly")) } tasks.named("compileJava") { dependsOn("compileQuarkusGeneratedSourcesJava") } diff --git a/build-logic/src/main/kotlin/publishing/PublishingHelperPlugin.kt b/build-logic/src/main/kotlin/publishing/PublishingHelperPlugin.kt index 708ca122c90..cf0b548cff0 100644 --- a/build-logic/src/main/kotlin/publishing/PublishingHelperPlugin.kt +++ b/build-logic/src/main/kotlin/publishing/PublishingHelperPlugin.kt @@ -156,7 +156,7 @@ constructor(private val softwareComponentFactory: SoftwareComponentFactory) : Pl val testFixturesSourcesJar by tasks.registering(org.gradle.api.tasks.bundling.Jar::class) { val sourceSets: SourceSetContainer by project - from(sourceSets.named("testFixtures").get().allSource) + from(sourceSets.named("testFixtures").map { it.allSource }) archiveClassifier.set("test-fixtures-sources") } tasks.named("testFixturesJavadoc") { isFailOnError = false } diff --git a/build-logic/src/main/kotlin/publishing/shadowPub.kt b/build-logic/src/main/kotlin/publishing/shadowPub.kt index dd4f4730d54..89ed236f850 100644 --- a/build-logic/src/main/kotlin/publishing/shadowPub.kt +++ b/build-logic/src/main/kotlin/publishing/shadowPub.kt @@ -86,7 +86,6 @@ internal fun configureShadowPublishing( skip() } } - // component.addVariantsFromConfiguration(configurations.getByName("runtimeElements")) { component.addVariantsFromConfiguration( project.configurations.getByName("shadowRuntimeElements") ) { diff --git a/persistence/nosql/persistence/correctness/build.gradle.kts b/persistence/nosql/persistence/correctness/build.gradle.kts index e938e791ad0..d27bdf72493 100644 --- a/persistence/nosql/persistence/correctness/build.gradle.kts +++ b/persistence/nosql/persistence/correctness/build.gradle.kts @@ -66,7 +66,9 @@ testing { implementation(testFixtures(project(":polaris-persistence-nosql-$prj"))) } - targets.all { testTask.configure { systemProperty("polaris.testBackend.name", db) } } + targets.configureEach { + testTask.configure { systemProperty("polaris.testBackend.name", db) } + } } tasks.named("intTest") { dependsOn(dbTaskName) } @@ -81,7 +83,7 @@ testing { // Pass system properties starting with `polaris.` down to the manually executed test(s) so // they can setup the backend via // `o.a.p.persistence.api.BackendConfigurer.defaultBackendConfigurer` using smallrye-config. - targets.all { + targets.configureEach { testTask.configure { providers.systemPropertiesPrefixedBy("polaris").get().forEach { (k, v) -> systemProperty(k, v) diff --git a/plugins/spark/v3.5/spark/build.gradle.kts b/plugins/spark/v3.5/spark/build.gradle.kts index 30ec54ded42..4c8db6aa09b 100644 --- a/plugins/spark/v3.5/spark/build.gradle.kts +++ b/plugins/spark/v3.5/spark/build.gradle.kts @@ -182,7 +182,7 @@ testing { implementation(libs.antlr4.runtime.spark35) } - targets.all { + targets.configureEach { testTask.configure { environment( "AWS_REGION", diff --git a/plugins/spark/v4.0/spark/build.gradle.kts b/plugins/spark/v4.0/spark/build.gradle.kts index 7fa04c77a4b..6e998e88bc4 100644 --- a/plugins/spark/v4.0/spark/build.gradle.kts +++ b/plugins/spark/v4.0/spark/build.gradle.kts @@ -178,7 +178,7 @@ testing { implementation(libs.antlr4.runtime.spark40) } - targets.all { + targets.configureEach { testTask.configure { environment( "AWS_REGION", diff --git a/runtime/spark-tests/build.gradle.kts b/runtime/spark-tests/build.gradle.kts index 011590f397a..cefd1ade52d 100644 --- a/runtime/spark-tests/build.gradle.kts +++ b/runtime/spark-tests/build.gradle.kts @@ -39,7 +39,7 @@ val localPolarisServer = files(provider { quarkusBuild.get().fastJar.resolve("qu localPolarisServer.builtBy(quarkusBuild) configurations.named(sparkStartupAction.runtimeOnlyConfigurationName) { - extendsFrom(configurations.getByName(sparkStartupAction.implementationConfigurationName)) + extendsFrom(configurations.named(sparkStartupAction.implementationConfigurationName)) } dependencies { @@ -188,7 +188,7 @@ testing { useJUnitJupiter() configureSparkIntegrationDependencies() - targets.all { + targets.configureEach { testTask.configure { configureSparkIntegrationTestTask( suiteName = "sparkIntTest", @@ -202,7 +202,7 @@ testing { useJUnitJupiter() configureSparkIntegrationDependencies() - targets.all { + targets.configureEach { testTask.configure { configureSparkIntegrationTestTask( suiteName = "catalogFederationIntTest", @@ -219,7 +219,7 @@ testing { configureSparkIntegrationDependencies() dependencies { implementation(project(":polaris-rustfs-testcontainer")) } - targets.all { + targets.configureEach { testTask.configure { configureSparkIntegrationTestTask( suiteName = "hiveFederationIntTest", diff --git a/runtime/test-common/build.gradle.kts b/runtime/test-common/build.gradle.kts index ff22995ecaf..73725c032cc 100644 --- a/runtime/test-common/build.gradle.kts +++ b/runtime/test-common/build.gradle.kts @@ -23,7 +23,7 @@ plugins { id("polaris-runtime") } -configurations.all { +configurations.configureEach { if (name != "checkstyle") { exclude(group = "org.antlr", module = "antlr4-runtime") exclude(group = "org.scala-lang", module = "scala-library") diff --git a/tools/version/build.gradle.kts b/tools/version/build.gradle.kts index dd443e74f8b..e2ddfa824c8 100644 --- a/tools/version/build.gradle.kts +++ b/tools/version/build.gradle.kts @@ -86,7 +86,7 @@ testing { register("jarTest") { dependencies { runtimeOnly(files(jarTestJar.map { it.archiveFile })) } - targets.all { + targets.configureEach { testTask.configure { dependsOn("jar", jarTestJar) systemProperty("rootProjectDir", rootProject.rootDir.relativeTo(project.projectDir)) From c039d7fcbf278fa469230b09b860a96aebb2fa5f Mon Sep 17 00:00:00 2001 From: Robert Stupp Date: Sat, 13 Jun 2026 10:08:14 +0200 Subject: [PATCH 21/22] Build: better project isolation This change mostly focuses on isolating projects even further. Also a few more direct usages of `System.getenv` are moved to the Gradle provider. --- api/iceberg-service/build.gradle.kts | 2 +- api/management-model/build.gradle.kts | 2 +- api/management-service/build.gradle.kts | 2 +- api/polaris-catalog-service/build.gradle.kts | 2 +- build-logic/src/main/kotlin/GitInfo.kt | 4 +-- .../src/main/kotlin/polaris-java.gradle.kts | 11 +++--- .../main/kotlin/polaris-spotless.gradle.kts | 2 +- .../main/kotlin/publishing/MemoizedJarInfo.kt | 16 +++++---- .../publishing/PublishingHelperPlugin.kt | 4 +-- .../src/main/kotlin/publishing/maven-utils.kt | 5 ++- .../src/main/kotlin/publishing/rootProject.kt | 2 +- .../src/main/kotlin/publishing/signing.kt | 4 ++- build.gradle.kts | 2 +- runtime/server/build.gradle.kts | 6 ++-- runtime/service/build.gradle.kts | 4 +-- settings.gradle.kts | 36 +++++++++++-------- tools/version/build.gradle.kts | 9 +++-- 17 files changed, 61 insertions(+), 52 deletions(-) diff --git a/api/iceberg-service/build.gradle.kts b/api/iceberg-service/build.gradle.kts index 6217f0bd03c..247efad2a7b 100644 --- a/api/iceberg-service/build.gradle.kts +++ b/api/iceberg-service/build.gradle.kts @@ -51,7 +51,7 @@ dependencies { compileOnly(libs.microprofile.fault.tolerance.api) } -val rootDir = rootProject.layout.projectDirectory +val rootDir = layout.settingsDirectory val specsDir = rootDir.dir("spec") val templatesDir = rootDir.dir("server-templates") // Use a different directory than 'generated/', because OpenAPI generator's `GenerateTask` adds the diff --git a/api/management-model/build.gradle.kts b/api/management-model/build.gradle.kts index 6858e00e4d5..17501a2041c 100644 --- a/api/management-model/build.gradle.kts +++ b/api/management-model/build.gradle.kts @@ -38,7 +38,7 @@ dependencies { testImplementation("com.fasterxml.jackson.core:jackson-databind") } -val rootDir = rootProject.layout.projectDirectory +val rootDir = layout.settingsDirectory val specsDir = rootDir.dir("spec") val templatesDir = rootDir.dir("server-templates") // Use a different directory than 'generated/', because OpenAPI generator's `GenerateTask` adds the diff --git a/api/management-service/build.gradle.kts b/api/management-service/build.gradle.kts index ef5199c728d..eb8a06019ea 100644 --- a/api/management-service/build.gradle.kts +++ b/api/management-service/build.gradle.kts @@ -47,7 +47,7 @@ dependencies { implementation(libs.slf4j.api) } -val rootDir = rootProject.layout.projectDirectory +val rootDir = layout.settingsDirectory val specsDir = rootDir.dir("spec") val templatesDir = rootDir.dir("server-templates") // Use a different directory than 'generated/', because OpenAPI generator's `GenerateTask` adds the diff --git a/api/polaris-catalog-service/build.gradle.kts b/api/polaris-catalog-service/build.gradle.kts index d61b6f646da..5475b718760 100644 --- a/api/polaris-catalog-service/build.gradle.kts +++ b/api/polaris-catalog-service/build.gradle.kts @@ -81,7 +81,7 @@ dependencies { annotationProcessor(project(":polaris-immutables", configuration = "processor")) } -val rootDir = rootProject.layout.projectDirectory +val rootDir = layout.settingsDirectory val specsDir = rootDir.dir("spec") val templatesDir = rootDir.dir("server-templates") // Use a different directory than 'generated/', because OpenAPI generator's `GenerateTask` adds the diff --git a/build-logic/src/main/kotlin/GitInfo.kt b/build-logic/src/main/kotlin/GitInfo.kt index 044fc816555..f3e5053edbf 100644 --- a/build-logic/src/main/kotlin/GitInfo.kt +++ b/build-logic/src/main/kotlin/GitInfo.kt @@ -36,9 +36,9 @@ class GitInfo(val gitHead: String, val gitDescribe: String, private val rawLinkR companion object { fun memoized(project: Project): GitInfo { - val rootProject = project.rootProject val isRelease = - rootProject.hasProperty("release") || rootProject.hasProperty("jarWithGitInfo") + project.providers.gradleProperty("release").isPresent || + project.providers.gradleProperty("jarWithGitInfo").isPresent val service = project.gradle.sharedServices.registerIfAbsent( "gitInfo-$isRelease", diff --git a/build-logic/src/main/kotlin/polaris-java.gradle.kts b/build-logic/src/main/kotlin/polaris-java.gradle.kts index d9df2b3602b..b4e886fbba8 100644 --- a/build-logic/src/main/kotlin/polaris-java.gradle.kts +++ b/build-logic/src/main/kotlin/polaris-java.gradle.kts @@ -72,7 +72,7 @@ checkstyle { .orElseThrow { GradleException("checkstyle version not found in libs.versions.toml") } .requiredVersion toolVersion = checkstyleVersion - configFile = rootProject.file("codestyle/checkstyle.xml") + configFile = layout.settingsDirectory.file("codestyle/checkstyle.xml").asFile isIgnoreFailures = false maxErrors = 0 maxWarnings = 0 @@ -95,8 +95,7 @@ tasks.withType(JavaCompile::class.java).configureEach { options.errorprone.disableWarningsInGeneratedCode = true options.errorprone.excludedPaths = ".*/${project.layout.buildDirectory.get().asFile.relativeTo(projectDir)}/generated(-openapi)?/.*" - val errorproneRules = - rootProject.layout.projectDirectory.file("codestyle/errorprone-rules.properties") + val errorproneRules = layout.settingsDirectory.file("codestyle/errorprone-rules.properties") inputs.file(errorproneRules).withPathSensitivity(PathSensitivity.RELATIVE) options.errorprone.checks.putAll( provider { @@ -309,10 +308,10 @@ fun bannedDependencies(): BannedDependencies { BannedDependenciesService::class.java, ) { parameters.globallyBannedFile.set( - rootProject.layout.projectDirectory.file("gradle/banned-dependencies.txt") + layout.settingsDirectory.file("gradle/banned-dependencies.txt") ) parameters.quarkusProdBannedFile.set( - rootProject.layout.projectDirectory.file("gradle/banned-quarkus-prod-dependencies.txt") + layout.settingsDirectory.file("gradle/banned-quarkus-prod-dependencies.txt") ) } return service.get().bannedDependencies @@ -365,7 +364,7 @@ tasks.withType().configureEach { val constraintName = if (isTestTask) "testParallelismConstraint" else "intTestParallelismConstraint" usesService(gradle.sharedServices.registrations.named(constraintName).get().service) - if (project.hasProperty("noIntegrationTests") && !isTestTask) { + if (providers.gradleProperty("noIntegrationTests").isPresent && !isTestTask) { enabled = false } } diff --git a/build-logic/src/main/kotlin/polaris-spotless.gradle.kts b/build-logic/src/main/kotlin/polaris-spotless.gradle.kts index a6ccae867bd..8d995f71122 100644 --- a/build-logic/src/main/kotlin/polaris-spotless.gradle.kts +++ b/build-logic/src/main/kotlin/polaris-spotless.gradle.kts @@ -50,7 +50,7 @@ spotless { target("src/**/*.xml", "src/**/*.xsd") targetExclude("codestyle/copyright-header.xml") eclipseWtp(com.diffplug.spotless.extra.wtp.EclipseWtpFormatterStep.XML) - .configFile(rootProject.file("codestyle/org.eclipse.wst.xml.core.prefs")) + .configFile(layout.settingsDirectory.file("codestyle/org.eclipse.wst.xml.core.prefs")) // getting the license-header delimiter right is a bit tricky. // licenseHeaderFile(rootProject.file("codestyle/copyright-header.xml"), '<^[!?].*$') } diff --git a/build-logic/src/main/kotlin/publishing/MemoizedJarInfo.kt b/build-logic/src/main/kotlin/publishing/MemoizedJarInfo.kt index 9b3a12d889b..a1655f4086b 100644 --- a/build-logic/src/main/kotlin/publishing/MemoizedJarInfo.kt +++ b/build-logic/src/main/kotlin/publishing/MemoizedJarInfo.kt @@ -29,19 +29,21 @@ import org.gradle.api.java.archives.Attributes */ internal class MemoizedJarInfo { companion object { - fun applyJarManifestAttributes(rootProject: Project, attribs: Attributes) { - val props = jarManifestAttributes(rootProject) + fun applyJarManifestAttributes(project: Project, attribs: Attributes) { + val props = jarManifestAttributes(project) attribs.putAll(props) } - private fun jarManifestAttributes(rootProject: Project): Map { - val version = rootProject.version.toString() - val javaSpecificationVersion = System.getProperty("java.specification.version") + private fun jarManifestAttributes(project: Project): Map { + val version = project.version.toString() + val javaSpecificationVersion = + project.providers.systemProperty("java.specification.version").get() val includeGitInformation = - rootProject.hasProperty("release") || rootProject.hasProperty("jarWithGitInfo") + project.providers.gradleProperty("release").isPresent || + project.providers.gradleProperty("jarWithGitInfo").isPresent return if (includeGitInformation) { - val gi = GitInfo.memoized(rootProject) + val gi = GitInfo.memoized(project) mapOf( "Implementation-Version" to version, "Apache-Polaris-Version" to version, diff --git a/build-logic/src/main/kotlin/publishing/PublishingHelperPlugin.kt b/build-logic/src/main/kotlin/publishing/PublishingHelperPlugin.kt index cf0b548cff0..89131718bc4 100644 --- a/build-logic/src/main/kotlin/publishing/PublishingHelperPlugin.kt +++ b/build-logic/src/main/kotlin/publishing/PublishingHelperPlugin.kt @@ -80,7 +80,7 @@ constructor(private val softwareComponentFactory: SoftwareComponentFactory) : Pl extensions.create("publishingHelper", PublishingHelperExtension::class.java) tasks.withType().configureEach { - manifest { MemoizedJarInfo.applyJarManifestAttributes(rootProject, attributes) } + manifest { MemoizedJarInfo.applyJarManifestAttributes(project, attributes) } } apply(plugin = "maven-publish") @@ -99,7 +99,7 @@ constructor(private val softwareComponentFactory: SoftwareComponentFactory) : Pl val signingPassword: String? by project useInMemoryPgpKeys(signingKey, signingPassword) - if (project.hasProperty("useGpgAgent")) { + if (project.providers.gradleProperty("useGpgAgent").isPresent) { useGpgCmd() } } diff --git a/build-logic/src/main/kotlin/publishing/maven-utils.kt b/build-logic/src/main/kotlin/publishing/maven-utils.kt index 2301d374bea..498bea753ec 100644 --- a/build-logic/src/main/kotlin/publishing/maven-utils.kt +++ b/build-logic/src/main/kotlin/publishing/maven-utils.kt @@ -87,7 +87,10 @@ fun addAdditionalJarContent(project: Project): Unit = // Letting jars depend on the pom.xml (Gradle's `GenerateMavenPom` task, annotated with // `@UntrackedTask`) breaks up-to-date checks for all jars and dependent project // dependencies, leading to unnecessarily long build times. - if (rootProject.hasProperty("release") || rootProject.hasProperty("jarWithGitInfo")) { + if ( + providers.gradleProperty("release").isPresent || + providers.gradleProperty("jarWithGitInfo").isPresent + ) { val pomPath = "META-INF/maven/${project.group}/${project.name}/pom.xml" val generatePomFileTask = tasks.named("generatePomFileForMavenPublication") diff --git a/build-logic/src/main/kotlin/publishing/rootProject.kt b/build-logic/src/main/kotlin/publishing/rootProject.kt index bfe3889ae4b..25e0627d904 100644 --- a/build-logic/src/main/kotlin/publishing/rootProject.kt +++ b/build-logic/src/main/kotlin/publishing/rootProject.kt @@ -38,7 +38,7 @@ internal fun configureOnRootProject(project: Project) = project.run { apply() - val isRelease = project.hasProperty("release") + val isRelease = project.providers.gradleProperty("release").isPresent val sourceTarball = tasks.register("sourceTarball") sourceTarball.configure { diff --git a/build-logic/src/main/kotlin/publishing/signing.kt b/build-logic/src/main/kotlin/publishing/signing.kt index 246ffa9a76a..37557a90e43 100644 --- a/build-logic/src/main/kotlin/publishing/signing.kt +++ b/build-logic/src/main/kotlin/publishing/signing.kt @@ -24,7 +24,9 @@ import org.gradle.api.tasks.TaskProvider import org.gradle.internal.extensions.stdlib.capitalized import org.gradle.plugins.signing.SigningExtension -fun Project.isSigningEnabled(): Boolean = hasProperty("release") || hasProperty("signArtifacts") +fun Project.isSigningEnabled(): Boolean = + providers.gradleProperty("release").isPresent || + providers.gradleProperty("signArtifacts").isPresent /** * Convenience function to sign all output files of the given task. diff --git a/build.gradle.kts b/build.gradle.kts index 3e91577bb25..66031e0ef7f 100644 --- a/build.gradle.kts +++ b/build.gradle.kts @@ -162,7 +162,7 @@ tasks.register("buildPythonClient") { description = "Build the python client" workingDir = project.projectDir - if (project.hasProperty("python.format")) { + if (providers.gradleProperty("python.format").isPresent) { environment("FORMAT", project.property("python.format") as String) } commandLine("make", "client-build") diff --git a/runtime/server/build.gradle.kts b/runtime/server/build.gradle.kts index 87989965a0a..461b55b795d 100644 --- a/runtime/server/build.gradle.kts +++ b/runtime/server/build.gradle.kts @@ -40,11 +40,11 @@ dependencies { runtimeOnly(project(":polaris-extensions-auth-opa")) runtimeOnly(project(":polaris-extensions-auth-ranger")) - if ((project.findProperty("NonRESTCatalogs") as String?)?.contains("HIVE") == true) { + val nonRestCatalogs = providers.gradleProperty("NonRESTCatalogs").orNull + if (nonRestCatalogs?.contains("HIVE") == true) { runtimeOnly(project(":polaris-extensions-federation-hive")) } - - if ((project.findProperty("NonRESTCatalogs") as String?)?.contains("BIGQUERY") == true) { + if (nonRestCatalogs?.contains("BIGQUERY") == true) { runtimeOnly(project(":polaris-extensions-federation-bigquery")) } diff --git a/runtime/service/build.gradle.kts b/runtime/service/build.gradle.kts index d3608738c7a..60ae5380df0 100644 --- a/runtime/service/build.gradle.kts +++ b/runtime/service/build.gradle.kts @@ -231,9 +231,7 @@ val quarkusTestArgLine = val quarkusProperties = providers.systemPropertiesPrefixedBy("quarkus.") tasks.withType(Test::class.java).configureEach { - if (System.getenv("AWS_REGION") == null) { - environment("AWS_REGION", "us-west-2") - } + environment("AWS_REGION", providers.environmentVariable("AWS_REGION").getOrElse("us-west-2")) // Note: the test secrets are referenced in // org.apache.polaris.service.it.ServerManager environment("POLARIS_BOOTSTRAP_CREDENTIALS", "POLARIS,test-admin,test-secret") diff --git a/settings.gradle.kts b/settings.gradle.kts index 7a665ded3d5..44070ca6aaf 100644 --- a/settings.gradle.kts +++ b/settings.gradle.kts @@ -52,10 +52,11 @@ if (!JavaVersion.current().isCompatibleWith(JavaVersion.VERSION_21)) { rootProject.name = "polaris" -val baseVersion = layout.rootDirectory.file("version.txt").asFile.readText().trim() +val baseVersion = + providers.fileContents(layout.rootDirectory.file("version.txt")).asText.map { it.trim() } gradle.beforeProject { - version = baseVersion + version = baseVersion.get() group = "org.apache.polaris" } @@ -146,12 +147,13 @@ dependencyResolutionManagement { } } -val isCI = System.getenv("CI") != null +val isCI = providers.environmentVariable("CI").isPresent val isBuildScanRequested = gradle.startParameter.isBuildScan develocity { - val isApachePolarisGitHub = "apache/polaris" == System.getenv("GITHUB_REPOSITORY") - val gitHubRef: String? = System.getenv("GITHUB_REF") + val isApachePolarisGitHub = + "apache/polaris" == providers.environmentVariable("GITHUB_REPOSITORY").orNull + val gitHubRef: String? = providers.environmentVariable("GITHUB_REF").orNull val isGitHubBranchOrTag = gitHubRef != null && (gitHubRef.startsWith("refs/heads/") || gitHubRef.startsWith("refs/tags/")) if (isApachePolarisGitHub && isGitHubBranchOrTag) { @@ -167,25 +169,29 @@ develocity { } } else { // In all other cases, especially PR CI runs, use Gradle's public Develocity instance. - var cfgPrjId: String? = System.getenv("DEVELOCITY_PROJECT_ID") - projectId = if (cfgPrjId.isNullOrEmpty()) "polaris" else cfgPrjId + projectId = + providers + .environmentVariable("DEVELOCITY_PROJECT_ID") + .filter { it.isNotBlank() } + .getOrElse("polaris") buildScan { - val isGradleTosAccepted = "true" == System.getenv("GRADLE_TOS_ACCEPTED") + val isGradleTosAccepted = + "true" == providers.environmentVariable("GRADLE_TOS_ACCEPTED").orNull val isGitHubPullRequest = gitHubRef?.startsWith("refs/pull/") ?: false if (isGradleTosAccepted || (isCI && isGitHubPullRequest && isApachePolarisGitHub)) { // Leave TOS agreement to the user, if not running in CI. termsOfUseUrl = "https://gradle.com/terms-of-service" termsOfUseAgree = "yes" } - System.getenv("DEVELOCITY_SERVER")?.run { - if (isNotEmpty()) { - server = this - } - } + providers + .environmentVariable("DEVELOCITY_SERVER") + .filter { it.isNotBlank() } + .orNull + ?.run { server = this } if (isGitHubPullRequest) { - System.getenv("GITHUB_SERVER_URL")?.run { + providers.environmentVariable("GITHUB_SERVER_URL").orNull?.run { val ghUrl = this - val ghRepo = System.getenv("GITHUB_REPOSITORY") + val ghRepo = providers.environmentVariable("GITHUB_REPOSITORY").get() val prNumber = gitHubRef.substringAfter("refs/pull/").substringBefore("/merge") link("GitHub pull request", "$ghUrl/$ghRepo/pull/$prNumber") } diff --git a/tools/version/build.gradle.kts b/tools/version/build.gradle.kts index e2ddfa824c8..2b23916d6b7 100644 --- a/tools/version/build.gradle.kts +++ b/tools/version/build.gradle.kts @@ -32,20 +32,19 @@ description = val syncNoticeAndLicense by tasks.registering(Sync::class) { // Have to manually declare the inputs of this task here on top of the from/include below + val rootDir = layout.settingsDirectory inputs - .files( - rootProject.fileTree(rootProject.rootDir) { include("NOTICE*", "LICENSE*", "version.txt") } - ) + .files(fileTree(rootDir) { include("NOTICE*", "LICENSE*", "version.txt") }) .withPathSensitivity(PathSensitivity.RELATIVE) inputs.property("version", project.version) destinationDir = project.layout.buildDirectory.dir("notice-licenses").get().asFile - from(rootProject.rootDir) { + from(rootDir) { include("NOTICE*", "LICENSE*") // Put NOTICE/LICENSE* files under META-INF/resources/ so those files can be directly // accessed as static web resources in Quarkus. eachFile { path = "META-INF/resources/apache-polaris/${file.name}.txt" } } - from(rootProject.rootDir) { + from(rootDir) { include("version.txt") // Put NOTICE/LICENSE* files under META-INF/resources/ so those files can be directly // accessed as static web resources in Quarkus. From 67de53e450a4659fc8b75a8805d2efc0da327c93 Mon Sep 17 00:00:00 2001 From: Robert Stupp Date: Sat, 13 Jun 2026 18:58:16 +0200 Subject: [PATCH 22/22] runtime-service/int-test: reuse Minio + RustFS containers across tests This saves the creation of new containers for each integration-test. --- .../service/it/PolarisRestCatalogMinIOIT.java | 42 +++++++----- .../it/PolarisRestCatalogRustFSIT.java | 33 +++++----- .../service/it/RestCatalogMinIOSpecialIT.java | 45 ++++++++----- .../it/RestCatalogRustFSSpecialIT.java | 45 ++++++++----- tools/minio-testcontainer/build.gradle.kts | 3 + .../test/minio/MinioConditionExtension.java | 46 +++++++++++++ .../polaris/test/minio/MinioExtension.java | 24 +------ .../polaris/test/minio/MinioTestResource.java | 65 +++++++++++++++++++ tools/rustfs-testcontainer/build.gradle.kts | 3 + .../test/rustfs/RustfsConditionExtension.java | 46 +++++++++++++ .../polaris/test/rustfs/RustfsExtension.java | 24 +------ .../test/rustfs/RustfsTestResource.java | 65 +++++++++++++++++++ 12 files changed, 327 insertions(+), 114 deletions(-) create mode 100644 tools/minio-testcontainer/src/main/java/org/apache/polaris/test/minio/MinioConditionExtension.java create mode 100644 tools/minio-testcontainer/src/main/java/org/apache/polaris/test/minio/MinioTestResource.java create mode 100644 tools/rustfs-testcontainer/src/main/java/org/apache/polaris/test/rustfs/RustfsConditionExtension.java create mode 100644 tools/rustfs-testcontainer/src/main/java/org/apache/polaris/test/rustfs/RustfsTestResource.java diff --git a/runtime/service/src/intTest/java/org/apache/polaris/service/it/PolarisRestCatalogMinIOIT.java b/runtime/service/src/intTest/java/org/apache/polaris/service/it/PolarisRestCatalogMinIOIT.java index ffccea4c778..50d7b8e8115 100644 --- a/runtime/service/src/intTest/java/org/apache/polaris/service/it/PolarisRestCatalogMinIOIT.java +++ b/runtime/service/src/intTest/java/org/apache/polaris/service/it/PolarisRestCatalogMinIOIT.java @@ -22,9 +22,10 @@ import static org.apache.polaris.test.commons.MinioRustProfile.SECRET_KEY; import com.google.common.collect.ImmutableMap; +import io.quarkus.test.common.QuarkusTestResource; +import io.quarkus.test.common.ResourceArg; import io.quarkus.test.junit.QuarkusIntegrationTest; import io.quarkus.test.junit.TestProfile; -import java.net.URI; import java.util.List; import java.util.Map; import org.apache.iceberg.aws.s3.S3FileIOProperties; @@ -36,35 +37,42 @@ import org.apache.polaris.test.commons.MinioRustProfile; import org.apache.polaris.test.minio.Minio; import org.apache.polaris.test.minio.MinioAccess; -import org.apache.polaris.test.minio.MinioExtension; -import org.junit.jupiter.api.BeforeAll; +import org.apache.polaris.test.minio.MinioConditionExtension; +import org.apache.polaris.test.minio.MinioTestResource; +import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.extension.ExtendWith; @QuarkusIntegrationTest @TestProfile(MinioRustProfile.class) -@ExtendWith(MinioExtension.class) +@QuarkusTestResource( + value = MinioTestResource.class, + initArgs = { + @ResourceArg(name = "accessKey", value = ACCESS_KEY), + @ResourceArg(name = "secretKey", value = SECRET_KEY) + }) +@ExtendWith(MinioConditionExtension.class) @ExtendWith(PolarisIntegrationTestExtension.class) @RestCatalogConfig({"header.X-Iceberg-Access-Delegation", "vended-credentials"}) public class PolarisRestCatalogMinIOIT extends PolarisRestCatalogIntegrationBase { protected static final String BUCKET_URI_PREFIX = "/minio-test-polaris"; - private static URI storageBase; - private static String endpoint; + @Minio static MinioAccess minioAccess; private static Map s3Properties; - @BeforeAll - static void setup( - @Minio(accessKey = ACCESS_KEY, secretKey = SECRET_KEY) MinioAccess minioAccess) { - storageBase = minioAccess.s3BucketUri(BUCKET_URI_PREFIX); - endpoint = minioAccess.s3endpoint(); + @BeforeEach + void setup() { s3Properties = Map.of( - S3FileIOProperties.ENDPOINT, endpoint, - S3FileIOProperties.PATH_STYLE_ACCESS, "true", - S3FileIOProperties.ACCESS_KEY_ID, ACCESS_KEY, - S3FileIOProperties.SECRET_ACCESS_KEY, SECRET_KEY); + S3FileIOProperties.ENDPOINT, + minioAccess.s3endpoint(), + S3FileIOProperties.PATH_STYLE_ACCESS, + "true", + S3FileIOProperties.ACCESS_KEY_ID, + ACCESS_KEY, + S3FileIOProperties.SECRET_ACCESS_KEY, + SECRET_KEY); } @Override @@ -78,8 +86,8 @@ protected StorageConfigInfo getStorageConfigInfo() { AwsStorageConfigInfo.builder() .setStorageType(StorageConfigInfo.StorageTypeEnum.S3) .setPathStyleAccess(true) - .setEndpoint(endpoint) - .setAllowedLocations(List.of(storageBase.toString())); + .setEndpoint(minioAccess.s3endpoint()) + .setAllowedLocations(List.of(minioAccess.s3BucketUri(BUCKET_URI_PREFIX).toString())); return storageConfig.build(); } diff --git a/runtime/service/src/intTest/java/org/apache/polaris/service/it/PolarisRestCatalogRustFSIT.java b/runtime/service/src/intTest/java/org/apache/polaris/service/it/PolarisRestCatalogRustFSIT.java index 5c3b833806a..f8c2f3626bc 100644 --- a/runtime/service/src/intTest/java/org/apache/polaris/service/it/PolarisRestCatalogRustFSIT.java +++ b/runtime/service/src/intTest/java/org/apache/polaris/service/it/PolarisRestCatalogRustFSIT.java @@ -22,9 +22,10 @@ import static org.apache.polaris.test.commons.MinioRustProfile.SECRET_KEY; import com.google.common.collect.ImmutableMap; +import io.quarkus.test.common.QuarkusTestResource; +import io.quarkus.test.common.ResourceArg; import io.quarkus.test.junit.QuarkusIntegrationTest; import io.quarkus.test.junit.TestProfile; -import java.net.URI; import java.util.List; import java.util.Map; import org.apache.polaris.core.admin.model.AwsStorageConfigInfo; @@ -35,33 +36,31 @@ import org.apache.polaris.test.commons.MinioRustProfile; import org.apache.polaris.test.rustfs.Rustfs; import org.apache.polaris.test.rustfs.RustfsAccess; -import org.apache.polaris.test.rustfs.RustfsExtension; -import org.junit.jupiter.api.BeforeAll; +import org.apache.polaris.test.rustfs.RustfsConditionExtension; +import org.apache.polaris.test.rustfs.RustfsTestResource; import org.junit.jupiter.api.TestInfo; import org.junit.jupiter.api.extension.ExtendWith; @QuarkusIntegrationTest @TestProfile(MinioRustProfile.class) -@ExtendWith(RustfsExtension.class) +@QuarkusTestResource( + value = RustfsTestResource.class, + initArgs = { + @ResourceArg(name = "accessKey", value = ACCESS_KEY), + @ResourceArg(name = "secretKey", value = SECRET_KEY) + }) +@ExtendWith(RustfsConditionExtension.class) @ExtendWith(PolarisIntegrationTestExtension.class) public class PolarisRestCatalogRustFSIT extends PolarisRestCatalogIntegrationBase { protected static final String BUCKET_URI_PREFIX = "/rustfs-test-polaris"; - private static URI storageBase; - private static String endpoint; - - @BeforeAll - static void setup( - @Rustfs(accessKey = ACCESS_KEY, secretKey = SECRET_KEY) RustfsAccess rustfsAccess) { - storageBase = rustfsAccess.s3BucketUri(BUCKET_URI_PREFIX); - endpoint = rustfsAccess.s3endpoint(); - } + @Rustfs static RustfsAccess rustfsAccess; @Override protected ImmutableMap.Builder clientFileIOProperties() { return super.clientFileIOProperties() - .put(StorageAccessProperty.AWS_ENDPOINT.getPropertyName(), endpoint) + .put(StorageAccessProperty.AWS_ENDPOINT.getPropertyName(), rustfsAccess.s3endpoint()) .put(StorageAccessProperty.AWS_PATH_STYLE_ACCESS.getPropertyName(), "true") .put(StorageAccessProperty.AWS_KEY_ID.getPropertyName(), ACCESS_KEY) .put(StorageAccessProperty.AWS_SECRET_KEY.getPropertyName(), SECRET_KEY); @@ -73,8 +72,8 @@ protected StorageConfigInfo getStorageConfigInfo() { AwsStorageConfigInfo.builder() .setStorageType(StorageConfigInfo.StorageTypeEnum.S3) .setPathStyleAccess(true) - .setEndpoint(endpoint) - .setAllowedLocations(List.of(storageBase.toString())); + .setEndpoint(rustfsAccess.s3endpoint()) + .setAllowedLocations(List.of(rustfsAccess.s3BucketUri(BUCKET_URI_PREFIX).toString())); return storageConfig.build(); } @@ -82,7 +81,7 @@ protected StorageConfigInfo getStorageConfigInfo() { @Override protected Map extraCatalogProperties(TestInfo testInfo) { return ImmutableMap.builder() - .put(StorageAccessProperty.AWS_ENDPOINT.getPropertyName(), endpoint) + .put(StorageAccessProperty.AWS_ENDPOINT.getPropertyName(), rustfsAccess.s3endpoint()) .put(StorageAccessProperty.AWS_PATH_STYLE_ACCESS.getPropertyName(), "true") .put(StorageAccessProperty.AWS_KEY_ID.getPropertyName(), ACCESS_KEY) .put(StorageAccessProperty.AWS_SECRET_KEY.getPropertyName(), SECRET_KEY) diff --git a/runtime/service/src/intTest/java/org/apache/polaris/service/it/RestCatalogMinIOSpecialIT.java b/runtime/service/src/intTest/java/org/apache/polaris/service/it/RestCatalogMinIOSpecialIT.java index 86e8b34e75d..7153150e6e7 100644 --- a/runtime/service/src/intTest/java/org/apache/polaris/service/it/RestCatalogMinIOSpecialIT.java +++ b/runtime/service/src/intTest/java/org/apache/polaris/service/it/RestCatalogMinIOSpecialIT.java @@ -36,6 +36,8 @@ import static org.assertj.core.api.Assertions.assertThatThrownBy; import com.google.common.collect.ImmutableMap; +import io.quarkus.test.common.QuarkusTestResource; +import io.quarkus.test.common.ResourceArg; import io.quarkus.test.junit.QuarkusIntegrationTest; import io.quarkus.test.junit.TestProfile; import java.io.IOException; @@ -44,6 +46,7 @@ import java.util.List; import java.util.Map; import java.util.Optional; +import java.util.concurrent.atomic.AtomicBoolean; import org.apache.iceberg.DataFile; import org.apache.iceberg.DataFiles; import org.apache.iceberg.FileFormat; @@ -76,10 +79,10 @@ import org.apache.polaris.test.commons.MinioRustProfile; import org.apache.polaris.test.minio.Minio; import org.apache.polaris.test.minio.MinioAccess; -import org.apache.polaris.test.minio.MinioExtension; +import org.apache.polaris.test.minio.MinioConditionExtension; +import org.apache.polaris.test.minio.MinioTestResource; import org.junit.jupiter.api.AfterAll; import org.junit.jupiter.api.AfterEach; -import org.junit.jupiter.api.BeforeAll; import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; import org.junit.jupiter.api.TestInfo; @@ -98,7 +101,13 @@ */ @QuarkusIntegrationTest @TestProfile(MinioRustProfile.class) -@ExtendWith(MinioExtension.class) +@QuarkusTestResource( + value = MinioTestResource.class, + initArgs = { + @ResourceArg(name = "accessKey", value = ACCESS_KEY), + @ResourceArg(name = "secretKey", value = SECRET_KEY) + }) +@ExtendWith(MinioConditionExtension.class) @ExtendWith(PolarisIntegrationTestExtension.class) public class RestCatalogMinIOSpecialIT { @@ -112,6 +121,9 @@ public class RestCatalogMinIOSpecialIT { required(1, "id", Types.IntegerType.get(), "doc"), optional(2, "data", Types.StringType.get())); + @Minio static MinioAccess minioAccess; + + private static final AtomicBoolean initialized = new AtomicBoolean(false); private static PolarisApiEndpoints endpoints; private static PolarisClient client; private static ManagementApi managementApi; @@ -124,27 +136,24 @@ public class RestCatalogMinIOSpecialIT { private PrincipalWithCredentials principalCredentials; private String catalogName; - @BeforeAll - static void setup( - PolarisApiEndpoints apiEndpoints, - @Minio(accessKey = ACCESS_KEY, secretKey = SECRET_KEY) MinioAccess minioAccess, - ClientCredentials credentials) { - s3Client = minioAccess.s3Client(); - endpoints = apiEndpoints; - client = polarisClient(endpoints); - adminToken = client.obtainToken(credentials); - managementApi = client.managementApi(adminToken); - storageBase = minioAccess.s3BucketUri(BUCKET_URI_PREFIX); - endpoint = minioAccess.s3endpoint(); - } - @AfterAll static void close() throws Exception { client.close(); } @BeforeEach - public void before(TestInfo testInfo) { + public void before( + TestInfo testInfo, PolarisApiEndpoints apiEndpoints, ClientCredentials credentials) { + if (initialized.compareAndSet(false, true)) { + endpoints = apiEndpoints; + s3Client = minioAccess.s3Client(); + client = polarisClient(endpoints); + adminToken = client.obtainToken(credentials); + managementApi = client.managementApi(adminToken); + storageBase = minioAccess.s3BucketUri(BUCKET_URI_PREFIX); + endpoint = minioAccess.s3endpoint(); + } + String principalName = client.newEntityName("test-user"); principalRoleName = client.newEntityName("test-admin"); principalCredentials = managementApi.createPrincipalWithRole(principalName, principalRoleName); diff --git a/runtime/service/src/intTest/java/org/apache/polaris/service/it/RestCatalogRustFSSpecialIT.java b/runtime/service/src/intTest/java/org/apache/polaris/service/it/RestCatalogRustFSSpecialIT.java index 569c68d6acd..71ca79f080d 100644 --- a/runtime/service/src/intTest/java/org/apache/polaris/service/it/RestCatalogRustFSSpecialIT.java +++ b/runtime/service/src/intTest/java/org/apache/polaris/service/it/RestCatalogRustFSSpecialIT.java @@ -36,6 +36,8 @@ import static org.assertj.core.api.Assertions.assertThatThrownBy; import com.google.common.collect.ImmutableMap; +import io.quarkus.test.common.QuarkusTestResource; +import io.quarkus.test.common.ResourceArg; import io.quarkus.test.junit.QuarkusIntegrationTest; import io.quarkus.test.junit.TestProfile; import java.io.IOException; @@ -44,6 +46,7 @@ import java.util.List; import java.util.Map; import java.util.Optional; +import java.util.concurrent.atomic.AtomicBoolean; import org.apache.iceberg.DataFile; import org.apache.iceberg.DataFiles; import org.apache.iceberg.FileFormat; @@ -76,10 +79,10 @@ import org.apache.polaris.test.commons.MinioRustProfile; import org.apache.polaris.test.rustfs.Rustfs; import org.apache.polaris.test.rustfs.RustfsAccess; -import org.apache.polaris.test.rustfs.RustfsExtension; +import org.apache.polaris.test.rustfs.RustfsConditionExtension; +import org.apache.polaris.test.rustfs.RustfsTestResource; import org.junit.jupiter.api.AfterAll; import org.junit.jupiter.api.AfterEach; -import org.junit.jupiter.api.BeforeAll; import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; import org.junit.jupiter.api.TestInfo; @@ -98,7 +101,13 @@ */ @QuarkusIntegrationTest @TestProfile(MinioRustProfile.class) -@ExtendWith(RustfsExtension.class) +@QuarkusTestResource( + value = RustfsTestResource.class, + initArgs = { + @ResourceArg(name = "accessKey", value = ACCESS_KEY), + @ResourceArg(name = "secretKey", value = SECRET_KEY) + }) +@ExtendWith(RustfsConditionExtension.class) @ExtendWith(PolarisIntegrationTestExtension.class) public class RestCatalogRustFSSpecialIT { @@ -112,6 +121,9 @@ public class RestCatalogRustFSSpecialIT { required(1, "id", Types.IntegerType.get(), "doc"), optional(2, "data", Types.StringType.get())); + @Rustfs static RustfsAccess rustfsAccess; + + private static final AtomicBoolean initialized = new AtomicBoolean(false); private static PolarisApiEndpoints endpoints; private static PolarisClient client; private static ManagementApi managementApi; @@ -124,27 +136,24 @@ public class RestCatalogRustFSSpecialIT { private PrincipalWithCredentials principalCredentials; private String catalogName; - @BeforeAll - static void setup( - PolarisApiEndpoints apiEndpoints, - @Rustfs(accessKey = ACCESS_KEY, secretKey = SECRET_KEY) RustfsAccess rustfsAccess, - ClientCredentials credentials) { - s3Client = rustfsAccess.s3Client(); - endpoints = apiEndpoints; - client = polarisClient(endpoints); - adminToken = client.obtainToken(credentials); - managementApi = client.managementApi(adminToken); - storageBase = rustfsAccess.s3BucketUri(BUCKET_URI_PREFIX); - endpoint = rustfsAccess.s3endpoint(); - } - @AfterAll static void close() throws Exception { client.close(); } @BeforeEach - public void before(TestInfo testInfo) { + public void before( + TestInfo testInfo, PolarisApiEndpoints apiEndpoints, ClientCredentials credentials) { + if (initialized.compareAndSet(false, true)) { + endpoints = apiEndpoints; + s3Client = rustfsAccess.s3Client(); + client = polarisClient(endpoints); + adminToken = client.obtainToken(credentials); + managementApi = client.managementApi(adminToken); + storageBase = rustfsAccess.s3BucketUri(BUCKET_URI_PREFIX); + endpoint = rustfsAccess.s3endpoint(); + } + String principalName = client.newEntityName("test-user"); principalRoleName = client.newEntityName("test-admin"); principalCredentials = managementApi.createPrincipalWithRole(principalName, principalRoleName); diff --git a/tools/minio-testcontainer/build.gradle.kts b/tools/minio-testcontainer/build.gradle.kts index 5aa1eb9cd00..8a558aa891e 100644 --- a/tools/minio-testcontainer/build.gradle.kts +++ b/tools/minio-testcontainer/build.gradle.kts @@ -36,4 +36,7 @@ dependencies { compileOnly(platform(libs.junit.bom)) compileOnly("org.junit.jupiter:junit-jupiter-api") + + compileOnly(platform(libs.quarkus.bom)) + compileOnly("io.quarkus:quarkus-test-common") } diff --git a/tools/minio-testcontainer/src/main/java/org/apache/polaris/test/minio/MinioConditionExtension.java b/tools/minio-testcontainer/src/main/java/org/apache/polaris/test/minio/MinioConditionExtension.java new file mode 100644 index 00000000000..fbb96539dd3 --- /dev/null +++ b/tools/minio-testcontainer/src/main/java/org/apache/polaris/test/minio/MinioConditionExtension.java @@ -0,0 +1,46 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.polaris.test.minio; + +import static java.lang.String.format; +import static org.junit.jupiter.api.extension.ConditionEvaluationResult.disabled; +import static org.junit.jupiter.api.extension.ConditionEvaluationResult.enabled; + +import org.junit.jupiter.api.condition.OS; +import org.junit.jupiter.api.extension.ConditionEvaluationResult; +import org.junit.jupiter.api.extension.ExecutionCondition; +import org.junit.jupiter.api.extension.ExtensionContext; + +/** JUnit extension that gates whether the Minio container can be run on the current platform. */ +public class MinioConditionExtension implements ExecutionCondition { + @Override + public ConditionEvaluationResult evaluateExecutionCondition(ExtensionContext context) { + if (OS.current() == OS.LINUX) { + return enabled("Running on Linux"); + } + if (OS.current() == OS.MAC + && System.getenv("CI_MAC") == null + && MinioContainer.canRunOnMacOs()) { + // Disable tests on GitHub Actions + return enabled("Running on macOS locally"); + } + return disabled(format("Disabled on %s", OS.current().name())); + } +} diff --git a/tools/minio-testcontainer/src/main/java/org/apache/polaris/test/minio/MinioExtension.java b/tools/minio-testcontainer/src/main/java/org/apache/polaris/test/minio/MinioExtension.java index 4063e13a6fd..e30e60685f5 100644 --- a/tools/minio-testcontainer/src/main/java/org/apache/polaris/test/minio/MinioExtension.java +++ b/tools/minio-testcontainer/src/main/java/org/apache/polaris/test/minio/MinioExtension.java @@ -19,18 +19,12 @@ package org.apache.polaris.test.minio; -import static java.lang.String.format; -import static org.junit.jupiter.api.extension.ConditionEvaluationResult.disabled; -import static org.junit.jupiter.api.extension.ConditionEvaluationResult.enabled; import static org.junit.platform.commons.util.AnnotationUtils.findAnnotatedFields; import static org.junit.platform.commons.util.ReflectionUtils.makeAccessible; import java.lang.reflect.Field; -import org.junit.jupiter.api.condition.OS; import org.junit.jupiter.api.extension.BeforeAllCallback; import org.junit.jupiter.api.extension.BeforeEachCallback; -import org.junit.jupiter.api.extension.ConditionEvaluationResult; -import org.junit.jupiter.api.extension.ExecutionCondition; import org.junit.jupiter.api.extension.ExtensionContext; import org.junit.jupiter.api.extension.ParameterContext; import org.junit.jupiter.api.extension.ParameterResolutionException; @@ -46,25 +40,11 @@ * annotated with {@link Minio}. */ // CODE_COPIED_TO_POLARIS from Project Nessie 0.104.2 -public class MinioExtension - implements BeforeAllCallback, BeforeEachCallback, ParameterResolver, ExecutionCondition { +public class MinioExtension extends MinioConditionExtension + implements BeforeAllCallback, BeforeEachCallback, ParameterResolver { private static final ExtensionContext.Namespace NAMESPACE = ExtensionContext.Namespace.create(MinioExtension.class); - @Override - public ConditionEvaluationResult evaluateExecutionCondition(ExtensionContext context) { - if (OS.current() == OS.LINUX) { - return enabled("Running on Linux"); - } - if (OS.current() == OS.MAC - && System.getenv("CI_MAC") == null - && MinioContainer.canRunOnMacOs()) { - // Disable tests on GitHub Actions - return enabled("Running on macOS locally"); - } - return disabled(format("Disabled on %s", OS.current().name())); - } - @Override public void beforeAll(ExtensionContext context) { Class testClass = context.getRequiredTestClass(); diff --git a/tools/minio-testcontainer/src/main/java/org/apache/polaris/test/minio/MinioTestResource.java b/tools/minio-testcontainer/src/main/java/org/apache/polaris/test/minio/MinioTestResource.java new file mode 100644 index 00000000000..df2d26d6830 --- /dev/null +++ b/tools/minio-testcontainer/src/main/java/org/apache/polaris/test/minio/MinioTestResource.java @@ -0,0 +1,65 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.polaris.test.minio; + +import static java.util.Objects.requireNonNull; + +import io.quarkus.test.common.QuarkusTestResourceLifecycleManager; +import java.util.Map; + +public class MinioTestResource implements QuarkusTestResourceLifecycleManager { + + private Map initArgs = Map.of(); + private MinioContainer container; + + @Override + public void init(Map initArgs) { + this.initArgs = Map.copyOf(initArgs); + } + + @Override + public Map start() { + var accessKey = initArgs.get("accessKey"); + var secretKey = initArgs.get("secretKey"); + var bucket = initArgs.get("bucket"); + var region = initArgs.get("region"); + this.container = + new MinioContainer(null, accessKey, secretKey, bucket, region).withStartupAttempts(5); + this.container.start(); + return Map.of(); + } + + @Override + public void stop() { + if (container != null) { + try { + container.close(); + } catch (Exception e) { + throw new RuntimeException(e); + } + } + } + + @Override + public void inject(TestInjector testInjector) { + MinioAccess minioAccess = requireNonNull(container, "Minio not started"); + testInjector.injectIntoFields(minioAccess, field -> field.isAnnotationPresent(Minio.class)); + } +} diff --git a/tools/rustfs-testcontainer/build.gradle.kts b/tools/rustfs-testcontainer/build.gradle.kts index 5aa1eb9cd00..8a558aa891e 100644 --- a/tools/rustfs-testcontainer/build.gradle.kts +++ b/tools/rustfs-testcontainer/build.gradle.kts @@ -36,4 +36,7 @@ dependencies { compileOnly(platform(libs.junit.bom)) compileOnly("org.junit.jupiter:junit-jupiter-api") + + compileOnly(platform(libs.quarkus.bom)) + compileOnly("io.quarkus:quarkus-test-common") } diff --git a/tools/rustfs-testcontainer/src/main/java/org/apache/polaris/test/rustfs/RustfsConditionExtension.java b/tools/rustfs-testcontainer/src/main/java/org/apache/polaris/test/rustfs/RustfsConditionExtension.java new file mode 100644 index 00000000000..9dcaa6e693b --- /dev/null +++ b/tools/rustfs-testcontainer/src/main/java/org/apache/polaris/test/rustfs/RustfsConditionExtension.java @@ -0,0 +1,46 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.polaris.test.rustfs; + +import static java.lang.String.format; +import static org.junit.jupiter.api.extension.ConditionEvaluationResult.disabled; +import static org.junit.jupiter.api.extension.ConditionEvaluationResult.enabled; + +import org.junit.jupiter.api.condition.OS; +import org.junit.jupiter.api.extension.ConditionEvaluationResult; +import org.junit.jupiter.api.extension.ExecutionCondition; +import org.junit.jupiter.api.extension.ExtensionContext; + +/** JUnit extension that gates whether the Rustfs container can be run on the current platform. */ +public class RustfsConditionExtension implements ExecutionCondition { + @Override + public ConditionEvaluationResult evaluateExecutionCondition(ExtensionContext context) { + if (OS.current() == OS.LINUX) { + return enabled("Running on Linux"); + } + if (OS.current() == OS.MAC + && System.getenv("CI_MAC") == null + && RustfsContainer.canRunOnMacOs()) { + // Disable tests on GitHub Actions + return enabled("Running on macOS locally"); + } + return disabled(format("Disabled on %s", OS.current().name())); + } +} diff --git a/tools/rustfs-testcontainer/src/main/java/org/apache/polaris/test/rustfs/RustfsExtension.java b/tools/rustfs-testcontainer/src/main/java/org/apache/polaris/test/rustfs/RustfsExtension.java index 3205cd9e616..deeae4cd9b2 100644 --- a/tools/rustfs-testcontainer/src/main/java/org/apache/polaris/test/rustfs/RustfsExtension.java +++ b/tools/rustfs-testcontainer/src/main/java/org/apache/polaris/test/rustfs/RustfsExtension.java @@ -19,18 +19,12 @@ package org.apache.polaris.test.rustfs; -import static java.lang.String.format; -import static org.junit.jupiter.api.extension.ConditionEvaluationResult.disabled; -import static org.junit.jupiter.api.extension.ConditionEvaluationResult.enabled; import static org.junit.platform.commons.util.AnnotationUtils.findAnnotatedFields; import static org.junit.platform.commons.util.ReflectionUtils.makeAccessible; import java.lang.reflect.Field; -import org.junit.jupiter.api.condition.OS; import org.junit.jupiter.api.extension.BeforeAllCallback; import org.junit.jupiter.api.extension.BeforeEachCallback; -import org.junit.jupiter.api.extension.ConditionEvaluationResult; -import org.junit.jupiter.api.extension.ExecutionCondition; import org.junit.jupiter.api.extension.ExtensionContext; import org.junit.jupiter.api.extension.ParameterContext; import org.junit.jupiter.api.extension.ParameterResolutionException; @@ -46,25 +40,11 @@ * annotated with {@link Rustfs}. */ // CODE_COPIED_TO_POLARIS from Project Nessie 0.104.2 -public class RustfsExtension - implements BeforeAllCallback, BeforeEachCallback, ParameterResolver, ExecutionCondition { +public class RustfsExtension extends RustfsConditionExtension + implements BeforeAllCallback, BeforeEachCallback, ParameterResolver { private static final ExtensionContext.Namespace NAMESPACE = ExtensionContext.Namespace.create(RustfsExtension.class); - @Override - public ConditionEvaluationResult evaluateExecutionCondition(ExtensionContext context) { - if (OS.current() == OS.LINUX) { - return enabled("Running on Linux"); - } - if (OS.current() == OS.MAC - && System.getenv("CI_MAC") == null - && RustfsContainer.canRunOnMacOs()) { - // Disable tests on GitHub Actions - return enabled("Running on macOS locally"); - } - return disabled(format("Disabled on %s", OS.current().name())); - } - @Override public void beforeAll(ExtensionContext context) { Class testClass = context.getRequiredTestClass(); diff --git a/tools/rustfs-testcontainer/src/main/java/org/apache/polaris/test/rustfs/RustfsTestResource.java b/tools/rustfs-testcontainer/src/main/java/org/apache/polaris/test/rustfs/RustfsTestResource.java new file mode 100644 index 00000000000..7063084f611 --- /dev/null +++ b/tools/rustfs-testcontainer/src/main/java/org/apache/polaris/test/rustfs/RustfsTestResource.java @@ -0,0 +1,65 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.polaris.test.rustfs; + +import static java.util.Objects.requireNonNull; + +import io.quarkus.test.common.QuarkusTestResourceLifecycleManager; +import java.util.Map; + +public class RustfsTestResource implements QuarkusTestResourceLifecycleManager { + + private Map initArgs = Map.of(); + private RustfsContainer container; + + @Override + public void init(Map initArgs) { + this.initArgs = Map.copyOf(initArgs); + } + + @Override + public Map start() { + var accessKey = initArgs.get("accessKey"); + var secretKey = initArgs.get("secretKey"); + var bucket = initArgs.get("bucket"); + var region = initArgs.get("region"); + this.container = + new RustfsContainer(null, accessKey, secretKey, bucket, region).withStartupAttempts(5); + this.container.start(); + return Map.of(); + } + + @Override + public void stop() { + if (container != null) { + try { + container.close(); + } catch (Exception e) { + throw new RuntimeException(e); + } + } + } + + @Override + public void inject(TestInjector testInjector) { + RustfsAccess rustfsAccess = requireNonNull(container, "RustFS not started"); + testInjector.injectIntoFields(rustfsAccess, field -> field.isAnnotationPresent(Rustfs.class)); + } +}