diff --git a/.gitignore b/.gitignore index bfaa0356c11..477e2406a78 100644 --- a/.gitignore +++ b/.gitignore @@ -69,6 +69,8 @@ gradle/wrapper/gradle-wrapper-*.sha256 /.gradle /build-logic/.gradle /build-logic/.kotlin +/gradle/server-test-runner/.gradle +/gradle/server-test-runner/.kotlin **/build/ !src/**/build/ @@ -111,6 +113,8 @@ venv # And then use `./gradlew run -Dquarkus.profile=local` to run Polaris with dev profile. application-local.properties +/.env + # AI / Agentic Development Tools .agents/ .aider/ diff --git a/api/iceberg-service/build.gradle.kts b/api/iceberg-service/build.gradle.kts index 6217f0bd03c..247efad2a7b 100644 --- a/api/iceberg-service/build.gradle.kts +++ b/api/iceberg-service/build.gradle.kts @@ -51,7 +51,7 @@ dependencies { compileOnly(libs.microprofile.fault.tolerance.api) } -val rootDir = rootProject.layout.projectDirectory +val rootDir = layout.settingsDirectory val specsDir = rootDir.dir("spec") val templatesDir = rootDir.dir("server-templates") // Use a different directory than 'generated/', because OpenAPI generator's `GenerateTask` adds the diff --git a/api/management-model/build.gradle.kts b/api/management-model/build.gradle.kts index 6858e00e4d5..17501a2041c 100644 --- a/api/management-model/build.gradle.kts +++ b/api/management-model/build.gradle.kts @@ -38,7 +38,7 @@ dependencies { testImplementation("com.fasterxml.jackson.core:jackson-databind") } -val rootDir = rootProject.layout.projectDirectory +val rootDir = layout.settingsDirectory val specsDir = rootDir.dir("spec") val templatesDir = rootDir.dir("server-templates") // Use a different directory than 'generated/', because OpenAPI generator's `GenerateTask` adds the diff --git a/api/management-service/build.gradle.kts b/api/management-service/build.gradle.kts index ef5199c728d..eb8a06019ea 100644 --- a/api/management-service/build.gradle.kts +++ b/api/management-service/build.gradle.kts @@ -47,7 +47,7 @@ dependencies { implementation(libs.slf4j.api) } -val rootDir = rootProject.layout.projectDirectory +val rootDir = layout.settingsDirectory val specsDir = rootDir.dir("spec") val templatesDir = rootDir.dir("server-templates") // Use a different directory than 'generated/', because OpenAPI generator's `GenerateTask` adds the diff --git a/api/polaris-catalog-service/build.gradle.kts b/api/polaris-catalog-service/build.gradle.kts index d61b6f646da..5475b718760 100644 --- a/api/polaris-catalog-service/build.gradle.kts +++ b/api/polaris-catalog-service/build.gradle.kts @@ -81,7 +81,7 @@ dependencies { annotationProcessor(project(":polaris-immutables", configuration = "processor")) } -val rootDir = rootProject.layout.projectDirectory +val rootDir = layout.settingsDirectory val specsDir = rootDir.dir("spec") val templatesDir = rootDir.dir("server-templates") // Use a different directory than 'generated/', because OpenAPI generator's `GenerateTask` adds the diff --git a/bom/build.gradle.kts b/bom/build.gradle.kts index 6a67f1e9488..bae2bb969c8 100644 --- a/bom/build.gradle.kts +++ b/bom/build.gradle.kts @@ -95,22 +95,17 @@ dependencies { api(project(":polaris-relational-jdbc")) api(project(":polaris-extensions-auth-opa")) - api(project(":polaris-extensions-auth-opa-tests")) api(project(":polaris-extensions-auth-ranger")) - api(project(":polaris-extensions-auth-ranger-tests")) api(project(":polaris-extensions-federation-bigquery")) api(project(":polaris-extensions-federation-hadoop")) api(project(":polaris-extensions-federation-hive")) api(project(":polaris-hms-testcontainer")) api(project(":polaris-spark-3.5_2.12")) - api(project(":polaris-spark-integration-3.5_2.12")) val ideaActive = providers.systemProperty("idea.active").getOrElse("false").toBoolean() if (!ideaActive) { api(project(":polaris-spark-3.5_2.13")) - api(project(":polaris-spark-integration-3.5_2.13")) api(project(":polaris-spark-4.0_2.13")) - api(project(":polaris-spark-integration-4.0_2.13")) } api(project(":polaris-admin")) @@ -133,7 +128,7 @@ tasks.register("verifyBomDependencies") { configurations.api.map { it.dependencyConstraints.map { "${it.group}:${it.name}" } } ) projectCoordinatesByPath.set( - rootProject.allprojects.associate { it.path to "${it.group}:${it.name}" } + provider { rootProject.allprojects.associate { it.path to "${it.group}:${it.name}" } } ) excludedProjectPaths.set( setOf( diff --git a/build-logic/src/main/kotlin/GitInfo.kt b/build-logic/src/main/kotlin/GitInfo.kt index 044fc816555..f3e5053edbf 100644 --- a/build-logic/src/main/kotlin/GitInfo.kt +++ b/build-logic/src/main/kotlin/GitInfo.kt @@ -36,9 +36,9 @@ class GitInfo(val gitHead: String, val gitDescribe: String, private val rawLinkR companion object { fun memoized(project: Project): GitInfo { - val rootProject = project.rootProject val isRelease = - rootProject.hasProperty("release") || rootProject.hasProperty("jarWithGitInfo") + project.providers.gradleProperty("release").isPresent || + project.providers.gradleProperty("jarWithGitInfo").isPresent val service = project.gradle.sharedServices.registerIfAbsent( "gitInfo-$isRelease", diff --git a/build-logic/src/main/kotlin/Utilities.kt b/build-logic/src/main/kotlin/Utilities.kt index f75ff5b3f8c..638ad515f74 100644 --- a/build-logic/src/main/kotlin/Utilities.kt +++ b/build-logic/src/main/kotlin/Utilities.kt @@ -17,6 +17,8 @@ import org.gradle.api.Project import org.gradle.process.JavaForkOptions +val noSourceCheckProjects = listOf(":polaris-spark-3.5_2.13") + /** * Extract the scala version from polaris spark project, and points the build directory to a sub-dir * that uses scala version as name. The polaris spark project name is in format of diff --git a/build-logic/src/main/kotlin/polaris-java.gradle.kts b/build-logic/src/main/kotlin/polaris-java.gradle.kts index e2976b8cdac..b4e886fbba8 100644 --- a/build-logic/src/main/kotlin/polaris-java.gradle.kts +++ b/build-logic/src/main/kotlin/polaris-java.gradle.kts @@ -72,14 +72,20 @@ checkstyle { .orElseThrow { GradleException("checkstyle version not found in libs.versions.toml") } .requiredVersion toolVersion = checkstyleVersion - configFile = rootProject.file("codestyle/checkstyle.xml") + configFile = layout.settingsDirectory.file("codestyle/checkstyle.xml").asFile isIgnoreFailures = false maxErrors = 0 maxWarnings = 0 } -// Ensure Checkstyle runs after jandex to avoid task dependency issues -tasks.withType().configureEach { tasks.findByName("jandex")?.let { mustRunAfter(it) } } +tasks.withType().configureEach { + if (noSourceCheckProjects.contains(project.path)) { + enabled = false + } else { + // Ensure Checkstyle runs after jandex to avoid task dependency issues + tasks.findByName("jandex")?.let { mustRunAfter(it) } + } +} tasks.withType(JavaCompile::class.java).configureEach { options.compilerArgs.addAll( @@ -89,23 +95,41 @@ tasks.withType(JavaCompile::class.java).configureEach { options.errorprone.disableWarningsInGeneratedCode = true options.errorprone.excludedPaths = ".*/${project.layout.buildDirectory.get().asFile.relativeTo(projectDir)}/generated(-openapi)?/.*" - val errorproneRules = rootProject.projectDir.resolve("codestyle/errorprone-rules.properties") + val errorproneRules = layout.settingsDirectory.file("codestyle/errorprone-rules.properties") inputs.file(errorproneRules).withPathSensitivity(PathSensitivity.RELATIVE) - options.errorprone.checks.putAll(provider { memoizedErrorproneRules(errorproneRules) }) + options.errorprone.checks.putAll( + provider { + val service = + project.gradle.sharedServices.registerIfAbsent( + "errorProneConfig", + ErrorProneConfigService::class.java, + ) { + parameters.configFile = errorproneRules + } + service.get().errorproneConfig + } + ) } -private fun memoizedErrorproneRules(rulesFile: File): Map = - rulesFile.reader().use { - val rules = Properties() - rules.load(it) - rules - .mapKeys { e -> (e.key as String).trim() } - .mapValues { e -> (e.value as String).trim() } - .filter { e -> e.key.isNotEmpty() && e.value.isNotEmpty() } - .mapValues { e -> CheckSeverity.valueOf(e.value) } - .toMap() +abstract class ErrorProneConfigService : BuildService { + interface Parameters : BuildServiceParameters { + val configFile: RegularFileProperty } + val errorproneConfig: Map by lazy { + parameters.configFile.get().asFile.reader().use { + val rules = Properties() + rules.load(it) + rules + .mapKeys { e -> (e.key as String).trim() } + .mapValues { e -> (e.value as String).trim() } + .filter { e -> e.key.isNotEmpty() && e.value.isNotEmpty() } + .mapValues { e -> CheckSeverity.valueOf(e.value) } + .toMap() + } + } +} + tasks.register("compileAll") { group = "build" description = "Runs all compilation and jar tasks" @@ -139,7 +163,7 @@ testing { // Special handling for test-suites with names containing `manualtest`, which are intended to // be run on demand rather than implicitly via `check`. if (!name.lowercase().contains("manualtest")) { - targets.all { + targets.configureEach { if (testTask.name != "test") { testTask.configure { shouldRunAfter("test") } tasks.named("check").configure { dependsOn(testTask) } @@ -273,7 +297,7 @@ class BannedDependencies( } fun applyTo(configurations: ConfigurationContainer) { - configurations.all { applyTo(this) } + configurations.configureEach { applyTo(this) } } } @@ -284,10 +308,10 @@ fun bannedDependencies(): BannedDependencies { BannedDependenciesService::class.java, ) { parameters.globallyBannedFile.set( - rootProject.layout.projectDirectory.file("gradle/banned-dependencies.txt") + layout.settingsDirectory.file("gradle/banned-dependencies.txt") ) parameters.quarkusProdBannedFile.set( - rootProject.layout.projectDirectory.file("gradle/banned-quarkus-prod-dependencies.txt") + layout.settingsDirectory.file("gradle/banned-quarkus-prod-dependencies.txt") ) } return service.get().bannedDependencies @@ -340,7 +364,7 @@ tasks.withType().configureEach { val constraintName = if (isTestTask) "testParallelismConstraint" else "intTestParallelismConstraint" usesService(gradle.sharedServices.registrations.named(constraintName).get().service) - if (project.hasProperty("noIntegrationTests") && !isTestTask) { + if (providers.gradleProperty("noIntegrationTests").isPresent && !isTestTask) { enabled = false } } diff --git a/build-logic/src/main/kotlin/polaris-root.gradle.kts b/build-logic/src/main/kotlin/polaris-root.gradle.kts index 4e9b4584c03..34fac94679b 100644 --- a/build-logic/src/main/kotlin/polaris-root.gradle.kts +++ b/build-logic/src/main/kotlin/polaris-root.gradle.kts @@ -25,22 +25,12 @@ import publishing.PublishingHelperPlugin plugins { id("polaris-base") - id("com.diffplug.spotless") + id("polaris-spotless") id("org.jetbrains.gradle.plugin.idea-ext") } apply() -if (!project.extra.has("duplicated-project-sources")) { - spotless { - kotlinGradle { - ktfmt().googleStyle() - // licenseHeaderFile(rootProject.file("codestyle/copyright-header-java.txt"), "$") - target("*.gradle.kts", "build-logic/*.gradle.kts", "build-logic/src/**/*.kt*") - } - } -} - if (providers.systemProperty("idea.sync.active").getOrElse("false").toBoolean()) { idea { module { diff --git a/build-logic/src/main/kotlin/polaris-runtime.gradle.kts b/build-logic/src/main/kotlin/polaris-runtime.gradle.kts index 0d320926ead..036a86c90a1 100644 --- a/build-logic/src/main/kotlin/polaris-runtime.gradle.kts +++ b/build-logic/src/main/kotlin/polaris-runtime.gradle.kts @@ -33,7 +33,7 @@ plugins { id("polaris-server") } testing { suites { withType { - targets.all { + targets.configureEach { testTask.configure { systemProperty("java.util.logging.manager", "org.jboss.logmanager.LogManager") // Enable automatic extension detection to execute GradleDuplicateLoggingWorkaround @@ -45,7 +45,7 @@ testing { } fun intTestSuiteConfigure(testSuite: JvmTestSuite) = testSuite.run { - targets.all { + targets.configureEach { testTask.configure { // For Quarkus... // @@ -61,12 +61,12 @@ testing { dependsOn("compileQuarkusTestGeneratedSourcesJava") } configurations.named(sources.runtimeOnlyConfigurationName).configure { - extendsFrom(configurations.getByName("testRuntimeOnly")) + extendsFrom(configurations.named("testRuntimeOnly")) } configurations.named(sources.implementationConfigurationName).configure { // Let the test's implementation config extend testImplementation, so it also inherits the // project's "main" implementation dependencies (not just the "api" configuration) - extendsFrom(configurations.getByName("testImplementation")) + extendsFrom(configurations.named("testImplementation")) } sources { java.srcDirs(tasks.named("quarkusGenerateCodeTests")) } } @@ -82,7 +82,7 @@ dependencies { implementation("org.jboss.slf4j:slf4j-jboss-logmanager") } -configurations.all { +configurations.configureEach { // Validate that Logback dependencies are not used in Quarkus modules. dependencies.configureEach { if (group == "ch.qos.logback") { @@ -95,7 +95,7 @@ configurations.all { } configurations.named("intTestRuntimeOnly").configure { - extendsFrom(configurations.getByName("testRuntimeOnly")) + extendsFrom(configurations.named("testRuntimeOnly")) } tasks.named("compileJava") { dependsOn("compileQuarkusGeneratedSourcesJava") } @@ -112,20 +112,8 @@ tasks.withType(Test::class.java).configureEach { // Gradle's Jacoco plugin doesn't work well with Quarkus's test coverage extensions.configure(JacocoTaskExtension::class) { isEnabled = false } - // Quarkus tests run "in isolated class loaders", which means that class-statically active - // resources pile up used JVM, as those classes cannot be GC'd. - // Examples of those statically held active resources are: - // - Iceberg's worker pools (thread pools, executors, etc.) - // - Hadoop's stats-cleaner (org.apache.hadoop.fs.FileSystem.Statistics.STATS_DATA_CLEANER) - // - Guava's 'MoreExecutors' (via Iceberg `ThreadPools`)` - // Forcing a new JVM after each test class works around this issue. - forkEvery = 1 - maxParallelForks = 1 - // enlarge the max heap size to avoid out of memory error - maxHeapSize = "4g" - // Silence the 'OpenJDK 64-Bit Server VM warning: Sharing is only supported for boot loader // classes because bootstrap classpath has been appended' warning from OpenJDK. jvmArgs("-Xshare:off") diff --git a/build-logic/src/main/kotlin/polaris-spotless.gradle.kts b/build-logic/src/main/kotlin/polaris-spotless.gradle.kts index b6ae54efa6b..8d995f71122 100644 --- a/build-logic/src/main/kotlin/polaris-spotless.gradle.kts +++ b/build-logic/src/main/kotlin/polaris-spotless.gradle.kts @@ -20,8 +20,20 @@ plugins { id("com.diffplug.spotless") } // skip spotless check for duplicated projects -if (!project.extra.has("duplicated-project-sources")) { - spotless { +spotless { + if (project.path == ":") { + // Root project + kotlinGradle { + ktfmt().googleStyle() + target( + "*.gradle.kts", + "build-logic/*.gradle.kts", + "build-logic/src/**/*.kt*", + "gradle/server-test-runner/**/*.gradle.kts", + "gradle/server-test-runner/src/**/*.kt*", + ) + } + } else if (!noSourceCheckProjects.contains(project.path)) { java { target("src/*/java/**/*.java") googleJavaFormat() @@ -38,7 +50,7 @@ if (!project.extra.has("duplicated-project-sources")) { target("src/**/*.xml", "src/**/*.xsd") targetExclude("codestyle/copyright-header.xml") eclipseWtp(com.diffplug.spotless.extra.wtp.EclipseWtpFormatterStep.XML) - .configFile(rootProject.file("codestyle/org.eclipse.wst.xml.core.prefs")) + .configFile(layout.settingsDirectory.file("codestyle/org.eclipse.wst.xml.core.prefs")) // getting the license-header delimiter right is a bit tricky. // licenseHeaderFile(rootProject.file("codestyle/copyright-header.xml"), '<^[!?].*$') } diff --git a/build-logic/src/main/kotlin/publishing/MemoizedJarInfo.kt b/build-logic/src/main/kotlin/publishing/MemoizedJarInfo.kt index 9b3a12d889b..a1655f4086b 100644 --- a/build-logic/src/main/kotlin/publishing/MemoizedJarInfo.kt +++ b/build-logic/src/main/kotlin/publishing/MemoizedJarInfo.kt @@ -29,19 +29,21 @@ import org.gradle.api.java.archives.Attributes */ internal class MemoizedJarInfo { companion object { - fun applyJarManifestAttributes(rootProject: Project, attribs: Attributes) { - val props = jarManifestAttributes(rootProject) + fun applyJarManifestAttributes(project: Project, attribs: Attributes) { + val props = jarManifestAttributes(project) attribs.putAll(props) } - private fun jarManifestAttributes(rootProject: Project): Map { - val version = rootProject.version.toString() - val javaSpecificationVersion = System.getProperty("java.specification.version") + private fun jarManifestAttributes(project: Project): Map { + val version = project.version.toString() + val javaSpecificationVersion = + project.providers.systemProperty("java.specification.version").get() val includeGitInformation = - rootProject.hasProperty("release") || rootProject.hasProperty("jarWithGitInfo") + project.providers.gradleProperty("release").isPresent || + project.providers.gradleProperty("jarWithGitInfo").isPresent return if (includeGitInformation) { - val gi = GitInfo.memoized(rootProject) + val gi = GitInfo.memoized(project) mapOf( "Implementation-Version" to version, "Apache-Polaris-Version" to version, diff --git a/build-logic/src/main/kotlin/publishing/PublishingHelperPlugin.kt b/build-logic/src/main/kotlin/publishing/PublishingHelperPlugin.kt index 708ca122c90..89131718bc4 100644 --- a/build-logic/src/main/kotlin/publishing/PublishingHelperPlugin.kt +++ b/build-logic/src/main/kotlin/publishing/PublishingHelperPlugin.kt @@ -80,7 +80,7 @@ constructor(private val softwareComponentFactory: SoftwareComponentFactory) : Pl extensions.create("publishingHelper", PublishingHelperExtension::class.java) tasks.withType().configureEach { - manifest { MemoizedJarInfo.applyJarManifestAttributes(rootProject, attributes) } + manifest { MemoizedJarInfo.applyJarManifestAttributes(project, attributes) } } apply(plugin = "maven-publish") @@ -99,7 +99,7 @@ constructor(private val softwareComponentFactory: SoftwareComponentFactory) : Pl val signingPassword: String? by project useInMemoryPgpKeys(signingKey, signingPassword) - if (project.hasProperty("useGpgAgent")) { + if (project.providers.gradleProperty("useGpgAgent").isPresent) { useGpgCmd() } } @@ -156,7 +156,7 @@ constructor(private val softwareComponentFactory: SoftwareComponentFactory) : Pl val testFixturesSourcesJar by tasks.registering(org.gradle.api.tasks.bundling.Jar::class) { val sourceSets: SourceSetContainer by project - from(sourceSets.named("testFixtures").get().allSource) + from(sourceSets.named("testFixtures").map { it.allSource }) archiveClassifier.set("test-fixtures-sources") } tasks.named("testFixturesJavadoc") { isFailOnError = false } diff --git a/build-logic/src/main/kotlin/publishing/maven-utils.kt b/build-logic/src/main/kotlin/publishing/maven-utils.kt index 2301d374bea..498bea753ec 100644 --- a/build-logic/src/main/kotlin/publishing/maven-utils.kt +++ b/build-logic/src/main/kotlin/publishing/maven-utils.kt @@ -87,7 +87,10 @@ fun addAdditionalJarContent(project: Project): Unit = // Letting jars depend on the pom.xml (Gradle's `GenerateMavenPom` task, annotated with // `@UntrackedTask`) breaks up-to-date checks for all jars and dependent project // dependencies, leading to unnecessarily long build times. - if (rootProject.hasProperty("release") || rootProject.hasProperty("jarWithGitInfo")) { + if ( + providers.gradleProperty("release").isPresent || + providers.gradleProperty("jarWithGitInfo").isPresent + ) { val pomPath = "META-INF/maven/${project.group}/${project.name}/pom.xml" val generatePomFileTask = tasks.named("generatePomFileForMavenPublication") diff --git a/build-logic/src/main/kotlin/publishing/rootProject.kt b/build-logic/src/main/kotlin/publishing/rootProject.kt index bfe3889ae4b..25e0627d904 100644 --- a/build-logic/src/main/kotlin/publishing/rootProject.kt +++ b/build-logic/src/main/kotlin/publishing/rootProject.kt @@ -38,7 +38,7 @@ internal fun configureOnRootProject(project: Project) = project.run { apply() - val isRelease = project.hasProperty("release") + val isRelease = project.providers.gradleProperty("release").isPresent val sourceTarball = tasks.register("sourceTarball") sourceTarball.configure { diff --git a/build-logic/src/main/kotlin/publishing/shadowPub.kt b/build-logic/src/main/kotlin/publishing/shadowPub.kt index dd4f4730d54..89ed236f850 100644 --- a/build-logic/src/main/kotlin/publishing/shadowPub.kt +++ b/build-logic/src/main/kotlin/publishing/shadowPub.kt @@ -86,7 +86,6 @@ internal fun configureShadowPublishing( skip() } } - // component.addVariantsFromConfiguration(configurations.getByName("runtimeElements")) { component.addVariantsFromConfiguration( project.configurations.getByName("shadowRuntimeElements") ) { diff --git a/build-logic/src/main/kotlin/publishing/signing.kt b/build-logic/src/main/kotlin/publishing/signing.kt index 246ffa9a76a..37557a90e43 100644 --- a/build-logic/src/main/kotlin/publishing/signing.kt +++ b/build-logic/src/main/kotlin/publishing/signing.kt @@ -24,7 +24,9 @@ import org.gradle.api.tasks.TaskProvider import org.gradle.internal.extensions.stdlib.capitalized import org.gradle.plugins.signing.SigningExtension -fun Project.isSigningEnabled(): Boolean = hasProperty("release") || hasProperty("signArtifacts") +fun Project.isSigningEnabled(): Boolean = + providers.gradleProperty("release").isPresent || + providers.gradleProperty("signArtifacts").isPresent /** * Convenience function to sign all output files of the given task. diff --git a/build.gradle.kts b/build.gradle.kts index 9c8ff902d08..66031e0ef7f 100644 --- a/build.gradle.kts +++ b/build.gradle.kts @@ -63,6 +63,8 @@ tasks.named("rat").configure { excludes.add("ide-name.txt") excludes.add("version.txt") + excludes.add(".env") + excludes.add("**/LICENSE*") excludes.add("**/NOTICE*") @@ -160,7 +162,7 @@ tasks.register("buildPythonClient") { description = "Build the python client" workingDir = project.projectDir - if (project.hasProperty("python.format")) { + if (providers.gradleProperty("python.format").isPresent) { environment("FORMAT", project.property("python.format") as String) } commandLine("make", "client-build") @@ -269,6 +271,7 @@ tasks.named("wrapper") { val insertAtLine = scriptLines.indexOf("# Use the maximum available, or set MAX_FD != -1 to use that value.") scriptLines.add(insertAtLine, "") + scriptLines.add(insertAtLine, $$"[ -f \"${APP_HOME}/.env\" ] && . \"${APP_HOME}/.env\"") scriptLines.add(insertAtLine, $$". \"${APP_HOME}/gradle/gradlew-include.sh\"") scriptFile.writeText(scriptLines.joinToString("\n")) diff --git a/extensions/auth/opa/impl/SCHEMA.md b/extensions/auth/opa/SCHEMA.md similarity index 98% rename from extensions/auth/opa/impl/SCHEMA.md rename to extensions/auth/opa/SCHEMA.md index ffebd5002bd..7905d6ab981 100644 --- a/extensions/auth/opa/impl/SCHEMA.md +++ b/extensions/auth/opa/SCHEMA.md @@ -96,7 +96,7 @@ Generates the JSON Schema from model classes. ./gradlew :polaris-extensions-auth-opa:generateOpaSchema ``` -**Output**: `extensions/auth/opa/impl/opa-input-schema.json` +**Output**: `extensions/auth/opa/opa-input-schema.json` ### `validateOpaSchema` Validates that committed schema matches the code. diff --git a/extensions/auth/opa/build.gradle.kts b/extensions/auth/opa/build.gradle.kts new file mode 100644 index 00000000000..5298d287294 --- /dev/null +++ b/extensions/auth/opa/build.gradle.kts @@ -0,0 +1,331 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +import java.io.OutputStream +import org.gradle.api.attributes.java.TargetJvmVersion +import org.gradle.api.plugins.jvm.JvmTestSuite +import org.gradle.language.base.plugins.LifecycleBasePlugin + +plugins { + id("polaris-server") + id("polaris-server-test-runner") + id("org.kordamp.gradle.jandex") +} + +val intTestJvmVersion = 21 +val intTestBase by + configurations.creating { + isCanBeConsumed = false + isCanBeResolved = false + } +val intTestBaseSources = sourceSets.create("intTestBase") +val jsonSchemaGenerator = sourceSets.create("jsonSchemaGenerator") +val jsonSchemaGeneratorImplementation by configurations.getting +val opaStartupAction = sourceSets.create("opaStartupAction") +val opaStartupActionCompileOnly by configurations.getting +val opaStartupActionImplementation by configurations.getting +val opaBearerTokenRefreshIntervalProperty = + "polaris.authorization.opa.auth.bearer.file-based.refresh-interval" + +listOf( + intTestBaseSources.implementationConfigurationName, + opaStartupAction.runtimeOnlyConfigurationName, + ) + .forEach { configurations.named(it) { extendsFrom(intTestBase) } } + +dependencies { + polarisServer(project(path = ":polaris-server", configuration = "quarkusRunner")) + + implementation(project(":polaris-core")) + implementation(libs.apache.httpclient5) + implementation(platform(libs.jackson.bom)) + implementation("com.fasterxml.jackson.core:jackson-core") + implementation("com.fasterxml.jackson.core:jackson-databind") + implementation(libs.guava) + implementation(libs.slf4j.api) + implementation(libs.auth0.jwt) + implementation(project(":polaris-async-api")) + + jsonSchemaGeneratorImplementation(project(":polaris-extensions-auth-opa")) + jsonSchemaGeneratorImplementation(platform(libs.jackson.bom)) + jsonSchemaGeneratorImplementation("com.fasterxml.jackson.module:jackson-module-jsonSchema") + + // Iceberg dependency for ForbiddenException + implementation(platform(libs.iceberg.bom)) + implementation("org.apache.iceberg:iceberg-api") + + compileOnly(project(":polaris-immutables")) + annotationProcessor(project(":polaris-immutables", configuration = "processor")) + + compileOnly(libs.jspecify) + compileOnly(libs.jakarta.annotation.api) + compileOnly(libs.jakarta.enterprise.cdi.api) + compileOnly(libs.jakarta.inject.api) + compileOnly(libs.smallrye.config.core) + + testCompileOnly(project(":polaris-immutables")) + testAnnotationProcessor(project(":polaris-immutables", configuration = "processor")) + + testImplementation(testFixtures(project(":polaris-core"))) + testImplementation(platform(libs.junit.bom)) + testImplementation("org.junit.jupiter:junit-jupiter") + testImplementation(libs.assertj.core) + testImplementation(libs.mockito.core) + testImplementation(libs.threeten.extra) + testImplementation(testFixtures(project(":polaris-async-api"))) + testImplementation(project(":polaris-async-java")) + testImplementation(project(":polaris-idgen-mocks")) + + opaStartupActionCompileOnly("org.apache.polaris.server-test-runner:polaris-server-test-runner") + opaStartupActionImplementation(platform(libs.testcontainers.bom)) + opaStartupActionImplementation("org.testcontainers:testcontainers") + opaStartupActionImplementation(project(":polaris-container-spec-helper")) + + intTestBase(platform(libs.junit.bom)) + intTestBase("org.junit.jupiter:junit-jupiter") + intTestBase("org.junit.platform:junit-platform-launcher") + intTestBase(enforcedPlatform(libs.quarkus.bom)) + intTestBase("io.rest-assured:rest-assured") + intTestBase(project(":polaris-tests")) + intTestBase(project(":polaris-runtime-test-common")) + intTestBase(project(":polaris-api-management-model")) + intTestBase(platform(libs.iceberg.bom)) + intTestBase("org.apache.iceberg:iceberg-api") + intTestBase("org.apache.iceberg:iceberg-core") +} + +// Task to generate JSON Schema from model classes +tasks.register("generateOpaSchema") { + group = "documentation" + description = "Generates JSON Schema for OPA authorization input" + + dependsOn(tasks.compileJava, tasks.named("jandex")) + + // Only execute generation if anything changed + outputs.cacheIf { true } + outputs.file("./opa-input-schema.json") + inputs.files(jsonSchemaGenerator.runtimeClasspath).withNormalizer(ClasspathNormalizer::class.java) + + classpath = jsonSchemaGenerator.runtimeClasspath + mainClass.set("org.apache.polaris.extension.auth.opa.model.OpaSchemaGenerator") + args("./opa-input-schema.json") +} + +// Task to validate that the committed schema matches the generated schema +tasks.register("validateOpaSchema") { + group = "verification" + description = "Validates that the committed OPA schema matches the generated schema" + + dependsOn(tasks.compileJava, tasks.named("jandex")) + + val tempSchemaFile = + layout.buildDirectory + .file("opa-schema/opa-input-schema-generated.json") + .get() + .asFile + .relativeTo(projectDir) + val committedSchemaFile = file("${projectDir}/opa-input-schema.json") + val logFile = layout.buildDirectory.file("opa-schema/generator.log") + + // Only execute validation if anything changed + outputs.cacheIf { true } + outputs.file(tempSchemaFile) + inputs.file(committedSchemaFile).withPathSensitivity(PathSensitivity.RELATIVE) + inputs.files(jsonSchemaGenerator.runtimeClasspath).withNormalizer(ClasspathNormalizer::class.java) + + classpath = jsonSchemaGenerator.runtimeClasspath + mainClass.set("org.apache.polaris.extension.auth.opa.model.OpaSchemaGenerator") + args(tempSchemaFile) + isIgnoreExitValue = true + + var outStream: OutputStream? = null + doFirst { + // Ensure temp directory exists + tempSchemaFile.parentFile.mkdirs() + outStream = logFile.get().asFile.outputStream() + standardOutput = outStream + errorOutput = outStream + } + + val prjDir = projectDir + + doLast { + outStream?.close() + + if (executionResult.get().exitValue != 0) { + throw GradleException( + """ + |OPA Schema validation failed! + | + |${logFile.get().asFile.readText()} + """ + .trimMargin() + ) + } + + val generatedContent = prjDir.resolve(tempSchemaFile).readText().trim() + val committedContent = committedSchemaFile.readText().trim() + + if (generatedContent != committedContent) { + throw GradleException( + """ + |OPA Schema validation failed! + | + |The committed opa-input-schema.json does not match the generated schema. + |This means the schema is out of sync with the model classes. + | + |To fix this, run: + | ./gradlew :polaris-extensions-auth-opa:generateOpaSchema + | + |Then commit the updated opa-input-schema.json file. + | + |Committed file: ${committedSchemaFile.absolutePath} + |Generated file: ${tempSchemaFile.absolutePath} + """ + .trimMargin() + ) + } + + logger.info("OPA schema validation passed - schema is up to date") + } +} + +fun Test.configureOpaTestTask( + buildDir: DirectoryProperty, + staticToken: String? = null, + fileTokenPath: Provider? = null, +) { + description = "Runs OPA integration tests against an external Polaris server." + group = LifecycleBasePlugin.VERIFICATION_GROUP + + environment("AWS_REGION", providers.environmentVariable("AWS_REGION").getOrElse("us-west-2")) + environment(mapOf("POLARIS_BOOTSTRAP_CREDENTIALS" to "POLARIS,test-admin,test-secret")) + val apiVersion = providers.environmentVariable("DOCKER_API_VERSION").getOrElse("1.44") + systemProperty("api.version", apiVersion) + jvmArgs("--add-exports", "java.base/sun.nio.ch=ALL-UNNAMED") + systemProperty("java.security.manager", "allow") + + val logsDir = buildDir.get().asFile.resolve("logs/$name") + + doFirst { + logsDir.deleteRecursively() + buildDir.get().asFile.resolve("quarkus.log").delete() + } + + withPolarisServer(configurations.polarisServer) { + startupActionClasspath.from(opaStartupAction.runtimeClasspath) + startupActionClass.set("org.apache.polaris.extension.auth.opa.test.OpaStartupAction") + + environment.put("AWS_REGION", providers.environmentVariable("AWS_REGION").orElse("us-west-2")) + environment.putAll(mapOf("POLARIS_BOOTSTRAP_CREDENTIALS" to "POLARIS,test-admin,test-secret")) + systemProperties.putAll( + mapOf( + "quarkus.log.file.path" to logsDir.resolve("polaris.log").absolutePath, + "polaris.authorization.type" to "opa", + "polaris.authorization.opa.auth.type" to "bearer", + "polaris.features.\"SUPPORTED_CATALOG_STORAGE_TYPES\"" to "[\"FILE\"]", + "polaris.features.\"ALLOW_INSECURE_STORAGE_TYPES\"" to "true", + "polaris.readiness.ignore-severe-issues" to "true", + ) + ) + staticToken?.let { + systemProperties.putAll( + mapOf("polaris.authorization.opa.auth.bearer.static-token.value" to it) + ) + } + fileTokenPath?.orNull?.let { + systemProperties.putAll( + mapOf( + "polaris.authorization.opa.auth.bearer.file-based.path" to it, + opaBearerTokenRefreshIntervalProperty to "PT1S", + ) + ) + } + } +} + +@Suppress("UnstableApiUsage") +fun JvmTestSuite.configureIntTest() { + useJUnitJupiter() + + configurations.named(sources.implementationConfigurationName) { extendsFrom(intTestBase) } + sources.compileClasspath += intTestBaseSources.output + sources.runtimeClasspath += intTestBaseSources.output +} + +testing { + suites { + val buildDir = project.layout.buildDirectory + + @Suppress("UnstableApiUsage") + register("bearerTokenIntTest") { + configureIntTest() + targets { + all { + testTask.configure { + configureOpaTestTask(buildDir, staticToken = "test-opa-bearer-token-12345") + } + } + } + } + + @Suppress("UnstableApiUsage") + register("opaFileTokenIntTest") { + configureIntTest() + val tokenFile = buildDir.file("opa-file-token/token.txt") + targets { + all { + testTask.configure { + configureOpaTestTask(buildDir, fileTokenPath = tokenFile.map { it.asFile.absolutePath }) + + doFirst { + val file = tokenFile.get().asFile + file.parentFile.mkdirs() + file.writeText("test-opa-bearer-token-from-file") + } + } + } + } + } + } +} + +listOf( + intTestBaseSources.compileClasspathConfigurationName, + intTestBaseSources.runtimeClasspathConfigurationName, + "bearerTokenIntTestCompileClasspath", + "bearerTokenIntTestRuntimeClasspath", + "opaFileTokenIntTestCompileClasspath", + "opaFileTokenIntTestRuntimeClasspath", + "opaStartupActionCompileClasspath", + "opaStartupActionRuntimeClasspath", + ) + .forEach { + configurations.named(it).configure { + attributes.attribute(TargetJvmVersion.TARGET_JVM_VERSION_ATTRIBUTE, intTestJvmVersion) + } + } + +tasks.register("intTest") { + description = "Runs all OPA integration tests." + group = LifecycleBasePlugin.VERIFICATION_GROUP + dependsOn("bearerTokenIntTest", "opaFileTokenIntTest") +} + +tasks.named("check") { dependsOn("validateOpaSchema", "intTest") } diff --git a/extensions/auth/opa/impl/build.gradle.kts b/extensions/auth/opa/impl/build.gradle.kts deleted file mode 100644 index 165c19d9b79..00000000000 --- a/extensions/auth/opa/impl/build.gradle.kts +++ /dev/null @@ -1,169 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -import java.io.OutputStream - -plugins { - id("polaris-server") - id("org.kordamp.gradle.jandex") -} - -val jsonSchemaGenerator = sourceSets.create("jsonSchemaGenerator") - -dependencies { - implementation(project(":polaris-core")) - implementation(libs.apache.httpclient5) - implementation(platform(libs.jackson.bom)) - implementation("com.fasterxml.jackson.core:jackson-core") - implementation("com.fasterxml.jackson.core:jackson-databind") - implementation(libs.guava) - implementation(libs.slf4j.api) - implementation(libs.auth0.jwt) - implementation(project(":polaris-async-api")) - - add(jsonSchemaGenerator.implementationConfigurationName, project(":polaris-extensions-auth-opa")) - add(jsonSchemaGenerator.implementationConfigurationName, platform(libs.jackson.bom)) - add( - jsonSchemaGenerator.implementationConfigurationName, - "com.fasterxml.jackson.module:jackson-module-jsonSchema", - ) - - // Iceberg dependency for ForbiddenException - implementation(platform(libs.iceberg.bom)) - implementation("org.apache.iceberg:iceberg-api") - - compileOnly(project(":polaris-immutables")) - annotationProcessor(project(":polaris-immutables", configuration = "processor")) - - compileOnly(libs.jspecify) - compileOnly(libs.jakarta.annotation.api) - compileOnly(libs.jakarta.enterprise.cdi.api) - compileOnly(libs.jakarta.inject.api) - compileOnly(libs.smallrye.config.core) - - testCompileOnly(project(":polaris-immutables")) - testAnnotationProcessor(project(":polaris-immutables", configuration = "processor")) - - testImplementation(testFixtures(project(":polaris-core"))) - testImplementation(platform(libs.junit.bom)) - testImplementation("org.junit.jupiter:junit-jupiter") - testImplementation(libs.assertj.core) - testImplementation(libs.mockito.core) - testImplementation(libs.threeten.extra) - testImplementation(testFixtures(project(":polaris-async-api"))) - testImplementation(project(":polaris-async-java")) - testImplementation(project(":polaris-idgen-mocks")) -} - -// Task to generate JSON Schema from model classes -tasks.register("generateOpaSchema") { - group = "documentation" - description = "Generates JSON Schema for OPA authorization input" - - dependsOn(tasks.compileJava, tasks.named("jandex")) - - // Only execute generation if anything changed - outputs.cacheIf { true } - outputs.file("./opa-input-schema.json") - inputs.files(jsonSchemaGenerator.runtimeClasspath).withNormalizer(ClasspathNormalizer::class.java) - - classpath = jsonSchemaGenerator.runtimeClasspath - mainClass.set("org.apache.polaris.extension.auth.opa.model.OpaSchemaGenerator") - args("./opa-input-schema.json") -} - -// Task to validate that the committed schema matches the generated schema -tasks.register("validateOpaSchema") { - group = "verification" - description = "Validates that the committed OPA schema matches the generated schema" - - dependsOn(tasks.compileJava, tasks.named("jandex")) - - val tempSchemaFile = - layout.buildDirectory - .file("opa-schema/opa-input-schema-generated.json") - .get() - .asFile - .relativeTo(projectDir) - val committedSchemaFile = file("${projectDir}/opa-input-schema.json") - val logFile = layout.buildDirectory.file("opa-schema/generator.log") - - // Only execute validation if anything changed - outputs.cacheIf { true } - outputs.file(tempSchemaFile) - inputs.file(committedSchemaFile).withPathSensitivity(PathSensitivity.RELATIVE) - inputs.files(jsonSchemaGenerator.runtimeClasspath).withNormalizer(ClasspathNormalizer::class.java) - - classpath = jsonSchemaGenerator.runtimeClasspath - mainClass.set("org.apache.polaris.extension.auth.opa.model.OpaSchemaGenerator") - args(tempSchemaFile) - isIgnoreExitValue = true - - var outStream: OutputStream? = null - doFirst { - // Ensure temp directory exists - tempSchemaFile.parentFile.mkdirs() - outStream = logFile.get().asFile.outputStream() - standardOutput = outStream - errorOutput = outStream - } - - doLast { - outStream?.close() - - if (executionResult.get().exitValue != 0) { - throw GradleException( - """ - |OPA Schema validation failed! - | - |${logFile.get().asFile.readText()} - """ - .trimMargin() - ) - } - - val generatedContent = projectDir.resolve(tempSchemaFile).readText().trim() - val committedContent = committedSchemaFile.readText().trim() - - if (generatedContent != committedContent) { - throw GradleException( - """ - |OPA Schema validation failed! - | - |The committed opa-input-schema.json does not match the generated schema. - |This means the schema is out of sync with the model classes. - | - |To fix this, run: - | ./gradlew :polaris-extensions-auth-opa:generateOpaSchema - | - |Then commit the updated opa-input-schema.json file. - | - |Committed file: ${committedSchemaFile.absolutePath} - |Generated file: ${tempSchemaFile.absolutePath} - """ - .trimMargin() - ) - } - - logger.info("OPA schema validation passed - schema is up to date") - } -} - -// Add schema validation to the check task -tasks.named("check") { dependsOn("validateOpaSchema") } diff --git a/extensions/auth/opa/impl/opa-input-schema.json b/extensions/auth/opa/opa-input-schema.json similarity index 100% rename from extensions/auth/opa/impl/opa-input-schema.json rename to extensions/auth/opa/opa-input-schema.json diff --git a/extensions/auth/opa/tests/src/intTest/java/org/apache/polaris/extension/auth/opa/test/OpaAdminServiceIT.java b/extensions/auth/opa/src/bearerTokenIntTest/java/org/apache/polaris/extension/auth/opa/test/OpaAdminServiceIT.java similarity index 99% rename from extensions/auth/opa/tests/src/intTest/java/org/apache/polaris/extension/auth/opa/test/OpaAdminServiceIT.java rename to extensions/auth/opa/src/bearerTokenIntTest/java/org/apache/polaris/extension/auth/opa/test/OpaAdminServiceIT.java index 86be8d058cb..e2f259e8ae8 100644 --- a/extensions/auth/opa/tests/src/intTest/java/org/apache/polaris/extension/auth/opa/test/OpaAdminServiceIT.java +++ b/extensions/auth/opa/src/bearerTokenIntTest/java/org/apache/polaris/extension/auth/opa/test/OpaAdminServiceIT.java @@ -20,8 +20,6 @@ import static io.restassured.RestAssured.given; -import io.quarkus.test.junit.QuarkusTest; -import io.quarkus.test.junit.TestProfile; import io.restassured.http.ContentType; import java.net.URI; import java.nio.file.Files; @@ -49,8 +47,6 @@ *
  • Catalog-role grant listings * */ -@QuarkusTest -@TestProfile(OpaTestProfiles.StaticToken.class) public class OpaAdminServiceIT extends OpaIntegrationTestBase { private @TempDir Path baseCatalogLocationPath; diff --git a/extensions/auth/opa/tests/src/intTest/java/org/apache/polaris/extension/auth/opa/test/OpaGenericTableHandlerIT.java b/extensions/auth/opa/src/bearerTokenIntTest/java/org/apache/polaris/extension/auth/opa/test/OpaGenericTableHandlerIT.java similarity index 96% rename from extensions/auth/opa/tests/src/intTest/java/org/apache/polaris/extension/auth/opa/test/OpaGenericTableHandlerIT.java rename to extensions/auth/opa/src/bearerTokenIntTest/java/org/apache/polaris/extension/auth/opa/test/OpaGenericTableHandlerIT.java index 54a8d41a45d..ef221979a95 100644 --- a/extensions/auth/opa/tests/src/intTest/java/org/apache/polaris/extension/auth/opa/test/OpaGenericTableHandlerIT.java +++ b/extensions/auth/opa/src/bearerTokenIntTest/java/org/apache/polaris/extension/auth/opa/test/OpaGenericTableHandlerIT.java @@ -20,8 +20,6 @@ import static io.restassured.RestAssured.given; -import io.quarkus.test.junit.QuarkusTest; -import io.quarkus.test.junit.TestProfile; import io.restassured.http.ContentType; import java.nio.file.Files; import java.nio.file.Path; @@ -41,8 +39,6 @@ *
  • Drop generic table * */ -@QuarkusTest -@TestProfile(OpaTestProfiles.StaticToken.class) public class OpaGenericTableHandlerIT extends OpaIntegrationTestBase { private String catalogName; diff --git a/extensions/auth/opa/tests/src/intTest/java/org/apache/polaris/extension/auth/opa/test/OpaIcebergCatalogHandlerIT.java b/extensions/auth/opa/src/bearerTokenIntTest/java/org/apache/polaris/extension/auth/opa/test/OpaIcebergCatalogHandlerIT.java similarity index 97% rename from extensions/auth/opa/tests/src/intTest/java/org/apache/polaris/extension/auth/opa/test/OpaIcebergCatalogHandlerIT.java rename to extensions/auth/opa/src/bearerTokenIntTest/java/org/apache/polaris/extension/auth/opa/test/OpaIcebergCatalogHandlerIT.java index 882de5adfc3..6ddfca80bcb 100644 --- a/extensions/auth/opa/tests/src/intTest/java/org/apache/polaris/extension/auth/opa/test/OpaIcebergCatalogHandlerIT.java +++ b/extensions/auth/opa/src/bearerTokenIntTest/java/org/apache/polaris/extension/auth/opa/test/OpaIcebergCatalogHandlerIT.java @@ -20,8 +20,6 @@ import static io.restassured.RestAssured.given; -import io.quarkus.test.junit.QuarkusTest; -import io.quarkus.test.junit.TestProfile; import io.restassured.http.ContentType; import java.net.URI; import java.nio.file.Files; @@ -47,8 +45,6 @@ *
  • Drop table * */ -@QuarkusTest -@TestProfile(OpaTestProfiles.StaticToken.class) public class OpaIcebergCatalogHandlerIT extends OpaIntegrationTestBase { private String catalogName; diff --git a/extensions/auth/opa/tests/src/intTest/java/org/apache/polaris/extension/auth/opa/test/OpaIntegrationTest.java b/extensions/auth/opa/src/bearerTokenIntTest/java/org/apache/polaris/extension/auth/opa/test/OpaIntegrationTest.java similarity index 94% rename from extensions/auth/opa/tests/src/intTest/java/org/apache/polaris/extension/auth/opa/test/OpaIntegrationTest.java rename to extensions/auth/opa/src/bearerTokenIntTest/java/org/apache/polaris/extension/auth/opa/test/OpaIntegrationTest.java index 1cf4cb6deb2..8891801dcbf 100644 --- a/extensions/auth/opa/tests/src/intTest/java/org/apache/polaris/extension/auth/opa/test/OpaIntegrationTest.java +++ b/extensions/auth/opa/src/bearerTokenIntTest/java/org/apache/polaris/extension/auth/opa/test/OpaIntegrationTest.java @@ -21,12 +21,8 @@ import static io.restassured.RestAssured.given; import static org.assertj.core.api.Assertions.assertThatNoException; -import io.quarkus.test.junit.QuarkusTest; -import io.quarkus.test.junit.TestProfile; import org.junit.jupiter.api.Test; -@QuarkusTest -@TestProfile(OpaTestProfiles.StaticToken.class) public class OpaIntegrationTest extends OpaIntegrationTestBase { @Test diff --git a/extensions/auth/opa/tests/src/intTest/java/org/apache/polaris/extension/auth/opa/test/OpaPolicyCatalogHandlerIT.java b/extensions/auth/opa/src/bearerTokenIntTest/java/org/apache/polaris/extension/auth/opa/test/OpaPolicyCatalogHandlerIT.java similarity index 97% rename from extensions/auth/opa/tests/src/intTest/java/org/apache/polaris/extension/auth/opa/test/OpaPolicyCatalogHandlerIT.java rename to extensions/auth/opa/src/bearerTokenIntTest/java/org/apache/polaris/extension/auth/opa/test/OpaPolicyCatalogHandlerIT.java index 6400a47c042..44bb5cd750f 100644 --- a/extensions/auth/opa/tests/src/intTest/java/org/apache/polaris/extension/auth/opa/test/OpaPolicyCatalogHandlerIT.java +++ b/extensions/auth/opa/src/bearerTokenIntTest/java/org/apache/polaris/extension/auth/opa/test/OpaPolicyCatalogHandlerIT.java @@ -20,8 +20,6 @@ import static io.restassured.RestAssured.given; -import io.quarkus.test.junit.QuarkusTest; -import io.quarkus.test.junit.TestProfile; import io.restassured.http.ContentType; import java.nio.file.Files; import java.nio.file.Path; @@ -41,8 +39,6 @@ *
  • Delete policies * */ -@QuarkusTest -@TestProfile(OpaTestProfiles.StaticToken.class) public class OpaPolicyCatalogHandlerIT extends OpaIntegrationTestBase { private String catalogName; diff --git a/extensions/auth/opa/tests/src/intTest/java/org/apache/polaris/extension/auth/opa/test/OpaIntegrationTestBase.java b/extensions/auth/opa/src/intTestBase/java/org/apache/polaris/extension/auth/opa/test/OpaIntegrationTestBase.java similarity index 95% rename from extensions/auth/opa/tests/src/intTest/java/org/apache/polaris/extension/auth/opa/test/OpaIntegrationTestBase.java rename to extensions/auth/opa/src/intTestBase/java/org/apache/polaris/extension/auth/opa/test/OpaIntegrationTestBase.java index aa121c521f1..269be94e8be 100644 --- a/extensions/auth/opa/tests/src/intTest/java/org/apache/polaris/extension/auth/opa/test/OpaIntegrationTestBase.java +++ b/extensions/auth/opa/src/intTestBase/java/org/apache/polaris/extension/auth/opa/test/OpaIntegrationTestBase.java @@ -23,13 +23,16 @@ import com.fasterxml.jackson.databind.JsonNode; import com.fasterxml.jackson.databind.json.JsonMapper; +import io.restassured.RestAssured; import io.restassured.http.ContentType; import java.io.UncheckedIOException; import java.util.ArrayList; import java.util.Collections; import java.util.List; import java.util.Map; +import org.junit.jupiter.api.AfterAll; import org.junit.jupiter.api.AfterEach; +import org.junit.jupiter.api.BeforeAll; /** * Base class for OPA integration tests providing common helper methods for authentication and @@ -40,6 +43,17 @@ public abstract class OpaIntegrationTestBase { private static final JsonMapper mapper = JsonMapper.builder().build(); private final List catalogsToCleanup = new ArrayList<>(); + @BeforeAll + static void configureRestAssured() { + RestAssured.baseURI = "http://localhost"; + RestAssured.port = Integer.getInteger("quarkus.http.test-port"); + } + + @AfterAll + static void resetRestAssured() { + RestAssured.reset(); + } + protected String toJson(Object value) { try { return mapper.writeValueAsString(value); diff --git a/runtime/spark-tests/src/intTest/resources/META-INF/services/org.apache.polaris.service.it.ext.PolarisServerManager b/extensions/auth/opa/src/intTestBase/resources/META-INF/services/org.apache.polaris.service.it.ext.PolarisServerManager similarity index 92% rename from runtime/spark-tests/src/intTest/resources/META-INF/services/org.apache.polaris.service.it.ext.PolarisServerManager rename to extensions/auth/opa/src/intTestBase/resources/META-INF/services/org.apache.polaris.service.it.ext.PolarisServerManager index 07ecf4b1fd0..6aa2dac9e0c 100644 --- a/runtime/spark-tests/src/intTest/resources/META-INF/services/org.apache.polaris.service.it.ext.PolarisServerManager +++ b/extensions/auth/opa/src/intTestBase/resources/META-INF/services/org.apache.polaris.service.it.ext.PolarisServerManager @@ -17,4 +17,4 @@ # under the License. # -org.apache.polaris.service.it.ServerManager \ No newline at end of file +org.apache.polaris.service.it.ext.ExternalPolarisServerManager diff --git a/extensions/auth/opa/impl/src/jsonSchemaGenerator/java/org/apache/polaris/extension/auth/opa/model/OpaSchemaGenerator.java b/extensions/auth/opa/src/jsonSchemaGenerator/java/org/apache/polaris/extension/auth/opa/model/OpaSchemaGenerator.java similarity index 100% rename from extensions/auth/opa/impl/src/jsonSchemaGenerator/java/org/apache/polaris/extension/auth/opa/model/OpaSchemaGenerator.java rename to extensions/auth/opa/src/jsonSchemaGenerator/java/org/apache/polaris/extension/auth/opa/model/OpaSchemaGenerator.java diff --git a/extensions/auth/opa/impl/src/main/java/org/apache/polaris/extension/auth/opa/OpaAuthorizationConfig.java b/extensions/auth/opa/src/main/java/org/apache/polaris/extension/auth/opa/OpaAuthorizationConfig.java similarity index 100% rename from extensions/auth/opa/impl/src/main/java/org/apache/polaris/extension/auth/opa/OpaAuthorizationConfig.java rename to extensions/auth/opa/src/main/java/org/apache/polaris/extension/auth/opa/OpaAuthorizationConfig.java diff --git a/extensions/auth/opa/impl/src/main/java/org/apache/polaris/extension/auth/opa/OpaHttpClientFactory.java b/extensions/auth/opa/src/main/java/org/apache/polaris/extension/auth/opa/OpaHttpClientFactory.java similarity index 100% rename from extensions/auth/opa/impl/src/main/java/org/apache/polaris/extension/auth/opa/OpaHttpClientFactory.java rename to extensions/auth/opa/src/main/java/org/apache/polaris/extension/auth/opa/OpaHttpClientFactory.java diff --git a/extensions/auth/opa/impl/src/main/java/org/apache/polaris/extension/auth/opa/OpaPolarisAuthorizer.java b/extensions/auth/opa/src/main/java/org/apache/polaris/extension/auth/opa/OpaPolarisAuthorizer.java similarity index 100% rename from extensions/auth/opa/impl/src/main/java/org/apache/polaris/extension/auth/opa/OpaPolarisAuthorizer.java rename to extensions/auth/opa/src/main/java/org/apache/polaris/extension/auth/opa/OpaPolarisAuthorizer.java diff --git a/extensions/auth/opa/impl/src/main/java/org/apache/polaris/extension/auth/opa/OpaPolarisAuthorizerFactory.java b/extensions/auth/opa/src/main/java/org/apache/polaris/extension/auth/opa/OpaPolarisAuthorizerFactory.java similarity index 100% rename from extensions/auth/opa/impl/src/main/java/org/apache/polaris/extension/auth/opa/OpaPolarisAuthorizerFactory.java rename to extensions/auth/opa/src/main/java/org/apache/polaris/extension/auth/opa/OpaPolarisAuthorizerFactory.java diff --git a/extensions/auth/opa/impl/src/main/java/org/apache/polaris/extension/auth/opa/OpaProductionReadinessChecks.java b/extensions/auth/opa/src/main/java/org/apache/polaris/extension/auth/opa/OpaProductionReadinessChecks.java similarity index 100% rename from extensions/auth/opa/impl/src/main/java/org/apache/polaris/extension/auth/opa/OpaProductionReadinessChecks.java rename to extensions/auth/opa/src/main/java/org/apache/polaris/extension/auth/opa/OpaProductionReadinessChecks.java diff --git a/extensions/auth/opa/impl/src/main/java/org/apache/polaris/extension/auth/opa/model/Actor.java b/extensions/auth/opa/src/main/java/org/apache/polaris/extension/auth/opa/model/Actor.java similarity index 100% rename from extensions/auth/opa/impl/src/main/java/org/apache/polaris/extension/auth/opa/model/Actor.java rename to extensions/auth/opa/src/main/java/org/apache/polaris/extension/auth/opa/model/Actor.java diff --git a/extensions/auth/opa/impl/src/main/java/org/apache/polaris/extension/auth/opa/model/Context.java b/extensions/auth/opa/src/main/java/org/apache/polaris/extension/auth/opa/model/Context.java similarity index 100% rename from extensions/auth/opa/impl/src/main/java/org/apache/polaris/extension/auth/opa/model/Context.java rename to extensions/auth/opa/src/main/java/org/apache/polaris/extension/auth/opa/model/Context.java diff --git a/extensions/auth/opa/impl/src/main/java/org/apache/polaris/extension/auth/opa/model/OpaAuthorizationInput.java b/extensions/auth/opa/src/main/java/org/apache/polaris/extension/auth/opa/model/OpaAuthorizationInput.java similarity index 100% rename from extensions/auth/opa/impl/src/main/java/org/apache/polaris/extension/auth/opa/model/OpaAuthorizationInput.java rename to extensions/auth/opa/src/main/java/org/apache/polaris/extension/auth/opa/model/OpaAuthorizationInput.java diff --git a/extensions/auth/opa/impl/src/main/java/org/apache/polaris/extension/auth/opa/model/OpaRequest.java b/extensions/auth/opa/src/main/java/org/apache/polaris/extension/auth/opa/model/OpaRequest.java similarity index 100% rename from extensions/auth/opa/impl/src/main/java/org/apache/polaris/extension/auth/opa/model/OpaRequest.java rename to extensions/auth/opa/src/main/java/org/apache/polaris/extension/auth/opa/model/OpaRequest.java diff --git a/extensions/auth/opa/impl/src/main/java/org/apache/polaris/extension/auth/opa/model/README.md b/extensions/auth/opa/src/main/java/org/apache/polaris/extension/auth/opa/model/README.md similarity index 96% rename from extensions/auth/opa/impl/src/main/java/org/apache/polaris/extension/auth/opa/model/README.md rename to extensions/auth/opa/src/main/java/org/apache/polaris/extension/auth/opa/model/README.md index d478e60f8b1..8c3bd5ee9c6 100644 --- a/extensions/auth/opa/impl/src/main/java/org/apache/polaris/extension/auth/opa/model/README.md +++ b/extensions/auth/opa/src/main/java/org/apache/polaris/extension/auth/opa/model/README.md @@ -33,7 +33,7 @@ Run the Gradle task to regenerate the schema: ./gradlew :polaris-extensions-auth-opa:generateOpaSchema ``` -The schema will be generated at: `extensions/auth/opa/impl/opa-input-schema.json` +The schema will be generated at: `extensions/auth/opa/opa-input-schema.json` ## Model Classes diff --git a/extensions/auth/opa/impl/src/main/java/org/apache/polaris/extension/auth/opa/model/Resource.java b/extensions/auth/opa/src/main/java/org/apache/polaris/extension/auth/opa/model/Resource.java similarity index 100% rename from extensions/auth/opa/impl/src/main/java/org/apache/polaris/extension/auth/opa/model/Resource.java rename to extensions/auth/opa/src/main/java/org/apache/polaris/extension/auth/opa/model/Resource.java diff --git a/extensions/auth/opa/impl/src/main/java/org/apache/polaris/extension/auth/opa/model/ResourceEntity.java b/extensions/auth/opa/src/main/java/org/apache/polaris/extension/auth/opa/model/ResourceEntity.java similarity index 100% rename from extensions/auth/opa/impl/src/main/java/org/apache/polaris/extension/auth/opa/model/ResourceEntity.java rename to extensions/auth/opa/src/main/java/org/apache/polaris/extension/auth/opa/model/ResourceEntity.java diff --git a/extensions/auth/opa/impl/src/main/java/org/apache/polaris/extension/auth/opa/model/package-info.java b/extensions/auth/opa/src/main/java/org/apache/polaris/extension/auth/opa/model/package-info.java similarity index 100% rename from extensions/auth/opa/impl/src/main/java/org/apache/polaris/extension/auth/opa/model/package-info.java rename to extensions/auth/opa/src/main/java/org/apache/polaris/extension/auth/opa/model/package-info.java diff --git a/extensions/auth/opa/impl/src/main/java/org/apache/polaris/extension/auth/opa/token/BearerTokenProvider.java b/extensions/auth/opa/src/main/java/org/apache/polaris/extension/auth/opa/token/BearerTokenProvider.java similarity index 100% rename from extensions/auth/opa/impl/src/main/java/org/apache/polaris/extension/auth/opa/token/BearerTokenProvider.java rename to extensions/auth/opa/src/main/java/org/apache/polaris/extension/auth/opa/token/BearerTokenProvider.java diff --git a/extensions/auth/opa/impl/src/main/java/org/apache/polaris/extension/auth/opa/token/FileBearerTokenProvider.java b/extensions/auth/opa/src/main/java/org/apache/polaris/extension/auth/opa/token/FileBearerTokenProvider.java similarity index 100% rename from extensions/auth/opa/impl/src/main/java/org/apache/polaris/extension/auth/opa/token/FileBearerTokenProvider.java rename to extensions/auth/opa/src/main/java/org/apache/polaris/extension/auth/opa/token/FileBearerTokenProvider.java diff --git a/extensions/auth/opa/impl/src/main/java/org/apache/polaris/extension/auth/opa/token/StaticBearerTokenProvider.java b/extensions/auth/opa/src/main/java/org/apache/polaris/extension/auth/opa/token/StaticBearerTokenProvider.java similarity index 100% rename from extensions/auth/opa/impl/src/main/java/org/apache/polaris/extension/auth/opa/token/StaticBearerTokenProvider.java rename to extensions/auth/opa/src/main/java/org/apache/polaris/extension/auth/opa/token/StaticBearerTokenProvider.java diff --git a/extensions/auth/opa/tests/src/intTest/java/org/apache/polaris/extension/auth/opa/test/OpaFileTokenIntegrationTest.java b/extensions/auth/opa/src/opaFileTokenIntTest/java/org/apache/polaris/extension/auth/opa/test/OpaFileTokenIntegrationTest.java similarity index 95% rename from extensions/auth/opa/tests/src/intTest/java/org/apache/polaris/extension/auth/opa/test/OpaFileTokenIntegrationTest.java rename to extensions/auth/opa/src/opaFileTokenIntTest/java/org/apache/polaris/extension/auth/opa/test/OpaFileTokenIntegrationTest.java index 81c2a04364a..e473b9325c1 100644 --- a/extensions/auth/opa/tests/src/intTest/java/org/apache/polaris/extension/auth/opa/test/OpaFileTokenIntegrationTest.java +++ b/extensions/auth/opa/src/opaFileTokenIntTest/java/org/apache/polaris/extension/auth/opa/test/OpaFileTokenIntegrationTest.java @@ -21,8 +21,7 @@ import static io.restassured.RestAssured.given; import static org.assertj.core.api.Assertions.assertThatNoException; -import io.quarkus.test.junit.QuarkusTest; -import io.quarkus.test.junit.TestProfile; +import org.junit.jupiter.api.Tag; import org.junit.jupiter.api.Test; /** @@ -31,8 +30,7 @@ *

    These tests verify that OpaPolarisAuthorizer correctly reads bearer tokens from a file and * uses them to authenticate with OPA. */ -@QuarkusTest -@TestProfile(OpaTestProfiles.FileToken.class) +@Tag("opa-file-token") public class OpaFileTokenIntegrationTest extends OpaIntegrationTestBase { @Test diff --git a/extensions/auth/opa/tests/src/intTest/java/org/apache/polaris/extension/auth/opa/test/OpaTestResource.java b/extensions/auth/opa/src/opaStartupAction/java/org/apache/polaris/extension/auth/opa/test/OpaStartupAction.java similarity index 53% rename from extensions/auth/opa/tests/src/intTest/java/org/apache/polaris/extension/auth/opa/test/OpaTestResource.java rename to extensions/auth/opa/src/opaStartupAction/java/org/apache/polaris/extension/auth/opa/test/OpaStartupAction.java index 456f380c0fc..b663ebba256 100644 --- a/extensions/auth/opa/tests/src/intTest/java/org/apache/polaris/extension/auth/opa/test/OpaTestResource.java +++ b/extensions/auth/opa/src/opaStartupAction/java/org/apache/polaris/extension/auth/opa/test/OpaStartupAction.java @@ -18,84 +18,60 @@ */ package org.apache.polaris.extension.auth.opa.test; -import io.quarkus.test.common.QuarkusTestResourceLifecycleManager; import java.io.OutputStream; import java.net.HttpURLConnection; import java.net.URI; import java.net.URL; import java.nio.charset.StandardCharsets; import java.time.Duration; -import java.util.HashMap; -import java.util.Map; import org.apache.polaris.containerspec.ContainerSpecHelper; +import org.apache.polaris.server.test.runner.spi.PolarisServerStartupAction; +import org.apache.polaris.server.test.runner.spi.PolarisServerStartupContext; import org.testcontainers.containers.GenericContainer; import org.testcontainers.containers.wait.strategy.Wait; -public class OpaTestResource implements QuarkusTestResourceLifecycleManager { - private static GenericContainer opa; +/** Starts an OPA test server before the external Polaris server process starts. */ +public class OpaStartupAction implements PolarisServerStartupAction { + private static final int OPA_PORT = 8181; + private static final String POLICY_NAME = "polaris-authz"; - @SuppressWarnings({"resource", "HttpUrlsUsage"}) - @Override - public Map start() { - try { - // Reuse container across tests to speed up execution - if (opa == null || !opa.isRunning()) { - opa = - new GenericContainer<>( - ContainerSpecHelper.containerSpecHelper("opa", OpaTestResource.class) - .dockerImageName(null)) - .withExposedPorts(8181) - .withReuse(true) - .withCommand("run", "--server", "--addr=0.0.0.0:8181") - .waitingFor( - Wait.forHttp("/health") - .forPort(8181) - .forStatusCode(200) - .withStartupTimeout(Duration.ofSeconds(120))); - - opa.start(); - } - - int mappedPort = opa.getMappedPort(8181); - String containerHost = opa.getHost(); - String baseUrl = "http://" + containerHost + ":" + mappedPort; - - // Load Opa Polaris Authorizer Rego policy into OPA - String polarisPolicyName = "polaris-authz"; - String polarisRegoPolicy = - """ - package polaris.authz - - default allow := false - - # Allow root user for all operations - allow if { - input.actor.principal == "root" - } + private GenericContainer opa; - # Allow admin user for all operations - allow if { - input.actor.principal == "admin" - } - """; - loadRegoPolicy(baseUrl, polarisPolicyName, polarisRegoPolicy); - - Map config = new HashMap<>(); - config.put("polaris.authorization.opa.policy-uri", baseUrl + "/v1/data/polaris/authz"); - - return config; + @Override + @SuppressWarnings("resource") + public void start(PolarisServerStartupContext context) { + opa = + new GenericContainer<>( + ContainerSpecHelper.containerSpecHelper("opa", OpaStartupAction.class) + .dockerImageName(null)) + .withExposedPorts(OPA_PORT) + .withCommand("run", "--server", "--addr=0.0.0.0:8181") + .waitingFor( + Wait.forHttp("/health") + .forPort(OPA_PORT) + .forStatusCode(200) + .withStartupTimeout(Duration.ofSeconds(120))); + + opa.start(); + + String baseUrl = "http://" + opa.getHost() + ":" + opa.getMappedPort(OPA_PORT); + loadRegoPolicy(baseUrl, POLICY_NAME, polarisRegoPolicy()); + context + .getSystemProperties() + .put("polaris.authorization.opa.policy-uri", baseUrl + "/v1/data/polaris/authz"); + } - } catch (Exception e) { - throw new RuntimeException("Failed to start OPA test resource", e); + @Override + public void close() { + if (opa != null) { + opa.stop(); + opa = null; } } private void loadRegoPolicy(String baseUrl, String policyName, String regoPolicy) { - // Hardcode the policy directly instead of loading through QuarkusTestProfile try { URL url = URI.create(baseUrl + "/v1/policies/" + policyName).toURL(); - System.out.println("Uploading policy to: " + url); - HttpURLConnection conn = (HttpURLConnection) url.openConnection(); conn.setRequestMethod("PUT"); conn.setDoOutput(true); @@ -106,27 +82,35 @@ private void loadRegoPolicy(String baseUrl, String policyName, String regoPolicy } int code = conn.getResponseCode(); - System.out.println("OPA policy upload response code: " + code); - if (code < 200 || code >= 300) { throw new RuntimeException("OPA policy upload failed, HTTP " + code); } - - System.out.println("Successfully uploaded policy to OPA"); } catch (Exception e) { - // Surface container logs to help debug on CI String logs = ""; try { logs = opa.getLogs(); } catch (Throwable ignored) { + // ignore logging failures while reporting the original startup failure } throw new RuntimeException("Failed to load OPA policy. Container logs:\n" + logs, e); } } - @Override - public void stop() { - // Quarkus takes care of reusing the test resource across tests - opa.stop(); + private String polarisRegoPolicy() { + return """ + package polaris.authz + + default allow := false + + # Allow root user for all operations + allow if { + input.actor.principal == "root" + } + + # Allow admin user for all operations + allow if { + input.actor.principal == "admin" + } + """; } } diff --git a/extensions/auth/opa/tests/src/intTest/resources/org/apache/polaris/extension/auth/opa/test/Dockerfile-opa-version b/extensions/auth/opa/src/opaStartupAction/resources/org/apache/polaris/extension/auth/opa/test/Dockerfile-opa-version similarity index 100% rename from extensions/auth/opa/tests/src/intTest/resources/org/apache/polaris/extension/auth/opa/test/Dockerfile-opa-version rename to extensions/auth/opa/src/opaStartupAction/resources/org/apache/polaris/extension/auth/opa/test/Dockerfile-opa-version diff --git a/extensions/auth/opa/impl/src/test/java/org/apache/polaris/extension/auth/opa/OpaHttpClientFactoryTest.java b/extensions/auth/opa/src/test/java/org/apache/polaris/extension/auth/opa/OpaHttpClientFactoryTest.java similarity index 100% rename from extensions/auth/opa/impl/src/test/java/org/apache/polaris/extension/auth/opa/OpaHttpClientFactoryTest.java rename to extensions/auth/opa/src/test/java/org/apache/polaris/extension/auth/opa/OpaHttpClientFactoryTest.java diff --git a/extensions/auth/opa/impl/src/test/java/org/apache/polaris/extension/auth/opa/OpaPolarisAuthorizerFactoryTest.java b/extensions/auth/opa/src/test/java/org/apache/polaris/extension/auth/opa/OpaPolarisAuthorizerFactoryTest.java similarity index 100% rename from extensions/auth/opa/impl/src/test/java/org/apache/polaris/extension/auth/opa/OpaPolarisAuthorizerFactoryTest.java rename to extensions/auth/opa/src/test/java/org/apache/polaris/extension/auth/opa/OpaPolarisAuthorizerFactoryTest.java diff --git a/extensions/auth/opa/impl/src/test/java/org/apache/polaris/extension/auth/opa/OpaPolarisAuthorizerTest.java b/extensions/auth/opa/src/test/java/org/apache/polaris/extension/auth/opa/OpaPolarisAuthorizerTest.java similarity index 100% rename from extensions/auth/opa/impl/src/test/java/org/apache/polaris/extension/auth/opa/OpaPolarisAuthorizerTest.java rename to extensions/auth/opa/src/test/java/org/apache/polaris/extension/auth/opa/OpaPolarisAuthorizerTest.java diff --git a/extensions/auth/opa/impl/src/test/java/org/apache/polaris/extension/auth/opa/token/FileBearerTokenProviderTest.java b/extensions/auth/opa/src/test/java/org/apache/polaris/extension/auth/opa/token/FileBearerTokenProviderTest.java similarity index 100% rename from extensions/auth/opa/impl/src/test/java/org/apache/polaris/extension/auth/opa/token/FileBearerTokenProviderTest.java rename to extensions/auth/opa/src/test/java/org/apache/polaris/extension/auth/opa/token/FileBearerTokenProviderTest.java diff --git a/extensions/auth/opa/impl/src/test/java/org/apache/polaris/extension/auth/opa/token/StaticBearerTokenProviderTest.java b/extensions/auth/opa/src/test/java/org/apache/polaris/extension/auth/opa/token/StaticBearerTokenProviderTest.java similarity index 100% rename from extensions/auth/opa/impl/src/test/java/org/apache/polaris/extension/auth/opa/token/StaticBearerTokenProviderTest.java rename to extensions/auth/opa/src/test/java/org/apache/polaris/extension/auth/opa/token/StaticBearerTokenProviderTest.java diff --git a/extensions/auth/opa/tests/build.gradle.kts b/extensions/auth/opa/tests/build.gradle.kts deleted file mode 100644 index c83bde936fc..00000000000 --- a/extensions/auth/opa/tests/build.gradle.kts +++ /dev/null @@ -1,79 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -plugins { - alias(libs.plugins.quarkus) - id("org.kordamp.gradle.jandex") - id("polaris-runtime") -} - -dependencies { - // Quarkus platform - implementation(enforcedPlatform(libs.quarkus.bom)) - implementation("io.quarkus:quarkus-rest-jackson") - - // Add the OPA implementation as RUNTIME dependency to include in Quarkus app - implementation(project(":polaris-extensions-auth-opa")) - - // Include all runtime-service dependencies - implementation(project(":polaris-runtime-service")) - - // Test common for integration testing - testImplementation(project(":polaris-runtime-test-common")) - - // Test dependencies - intTestImplementation("io.quarkus:quarkus-junit") - intTestImplementation("io.rest-assured:rest-assured") - intTestImplementation(project(":polaris-api-management-model")) - intTestImplementation(platform(libs.iceberg.bom)) - intTestImplementation("org.apache.iceberg:iceberg-api") - intTestImplementation("org.apache.iceberg:iceberg-core") - - // Test container dependencies - intTestImplementation(platform(libs.testcontainers.bom)) - intTestImplementation("org.testcontainers:testcontainers-junit-jupiter") - intTestImplementation(project(":polaris-container-spec-helper")) -} - -tasks.named("javadoc") { dependsOn("jandex") } - -tasks.withType { - if (System.getenv("AWS_REGION") == null) { - environment("AWS_REGION", "us-west-2") - } - environment("POLARIS_BOOTSTRAP_CREDENTIALS", "POLARIS,test-admin,test-secret") - val apiVersion = System.getenv("DOCKER_API_VERSION") ?: "1.44" - systemProperty("api.version", apiVersion) - jvmArgs("--add-exports", "java.base/sun.nio.ch=ALL-UNNAMED") - systemProperty("java.security.manager", "allow") - maxParallelForks = 1 - - val logsDir = project.layout.buildDirectory.get().asFile.resolve("logs") - - jvmArgumentProviders.add( - CommandLineArgumentProvider { - listOf("-Dquarkus.log.file.path=${logsDir.resolve("polaris.log").absolutePath}") - } - ) - - doFirst { - logsDir.deleteRecursively() - project.layout.buildDirectory.get().asFile.resolve("quarkus.log").delete() - } -} diff --git a/extensions/auth/opa/tests/src/intTest/java/org/apache/polaris/extension/auth/opa/test/OpaTestProfiles.java b/extensions/auth/opa/tests/src/intTest/java/org/apache/polaris/extension/auth/opa/test/OpaTestProfiles.java deleted file mode 100644 index a4864946af2..00000000000 --- a/extensions/auth/opa/tests/src/intTest/java/org/apache/polaris/extension/auth/opa/test/OpaTestProfiles.java +++ /dev/null @@ -1,87 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ -package org.apache.polaris.extension.auth.opa.test; - -import io.quarkus.test.junit.QuarkusTestProfile; -import java.io.IOException; -import java.nio.file.Files; -import java.nio.file.Path; -import java.util.HashMap; -import java.util.List; -import java.util.Map; - -/** Shared Quarkus test profiles for OPA integration tests. */ -public final class OpaTestProfiles { - - private OpaTestProfiles() {} - - /** OPA profile using a static bearer token. */ - public static class StaticToken implements QuarkusTestProfile { - @Override - public Map getConfigOverrides() { - Map config = new HashMap<>(); - config.put("polaris.authorization.type", "opa"); - config.put("polaris.authorization.opa.auth.type", "bearer"); - config.put( - "polaris.authorization.opa.auth.bearer.static-token.value", - "test-opa-bearer-token-12345"); - config.put("polaris.features.\"SUPPORTED_CATALOG_STORAGE_TYPES\"", "[\"FILE\"]"); - config.put("polaris.features.\"ALLOW_INSECURE_STORAGE_TYPES\"", "true"); - config.put("polaris.readiness.ignore-severe-issues", "true"); - return config; - } - - @Override - public List testResources() { - return List.of(new TestResourceEntry(OpaTestResource.class)); - } - } - - /** OPA profile using a bearer token read from a file. */ - public static class FileToken implements QuarkusTestProfile { - // Exposed for tests that may need to inspect the created token file. - public static Path tokenFilePath; - - @Override - public Map getConfigOverrides() { - try { - tokenFilePath = Files.createTempFile("opa-test-token", ".txt"); - Files.writeString(tokenFilePath, "test-opa-bearer-token-from-file"); - - Map config = new HashMap<>(); - config.put("polaris.authorization.type", "opa"); - config.put("polaris.authorization.opa.auth.type", "bearer"); - config.put( - "polaris.authorization.opa.auth.bearer.file-based.path", tokenFilePath.toString()); - config.put("polaris.authorization.opa.auth.bearer.file-based.refresh-interval", "PT1S"); - config.put("polaris.features.\"SUPPORTED_CATALOG_STORAGE_TYPES\"", "[\"FILE\"]"); - config.put("polaris.features.\"ALLOW_INSECURE_STORAGE_TYPES\"", "true"); - config.put("polaris.readiness.ignore-severe-issues", "true"); - return config; - } catch (IOException e) { - throw new RuntimeException("Failed to create test token file", e); - } - } - - @Override - public List testResources() { - return List.of(new TestResourceEntry(OpaTestResource.class)); - } - } -} diff --git a/extensions/auth/ranger/impl/README.md b/extensions/auth/ranger/README.md similarity index 100% rename from extensions/auth/ranger/impl/README.md rename to extensions/auth/ranger/README.md diff --git a/extensions/auth/ranger/build.gradle.kts b/extensions/auth/ranger/build.gradle.kts new file mode 100644 index 00000000000..46343917b64 --- /dev/null +++ b/extensions/auth/ranger/build.gradle.kts @@ -0,0 +1,140 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +import org.gradle.api.attributes.java.TargetJvmVersion +import org.gradle.api.plugins.jvm.JvmTestSuite + +plugins { + id("polaris-server") + id("org.kordamp.gradle.jandex") + id("polaris-server-test-runner") +} + +val intTestJvmVersion = 21 + +dependencies { + polarisServer(project(path = ":polaris-server", configuration = "quarkusRunner")) + + implementation(project(":polaris-core")) + + implementation(libs.ranger.authz.embedded) { + exclude("org.apache.ranger", "ranger-audit-dest-hdfs") + exclude("org.slf4j", "slf4j-reload4j") + exclude("ch.qos.reload4j", "reload4j") + exclude("io.dropwizard.metrics", "metrics-core") + } + + implementation(libs.commons.lang3) + implementation(libs.guava) + + // Iceberg dependency for ForbiddenException + implementation(platform(libs.iceberg.bom)) + implementation("org.apache.iceberg:iceberg-api") + + compileOnly(libs.jspecify) + compileOnly(libs.jakarta.enterprise.cdi.api) + compileOnly(libs.jakarta.inject.api) + compileOnly(libs.smallrye.config.core) + compileOnly(project(":polaris-immutables")) + + runtimeOnly(libs.graalvm.js.js.scriptengine) + runtimeOnly(libs.graalvm.polyglot.js) + runtimeOnly(libs.graalvm.polyglot.polyglot) +} + +testing { + suites { + @Suppress("UnstableApiUsage") + register("intTest") { + dependencies { + implementation(platform(libs.quarkus.bom)) + implementation("io.rest-assured:rest-assured") + implementation(project(":polaris-tests")) + implementation(project(":polaris-runtime-test-common")) + implementation(project(":polaris-api-management-model")) + implementation(platform(libs.iceberg.bom)) + implementation("org.apache.iceberg:iceberg-api") + implementation("org.apache.iceberg:iceberg-core") + + implementation(platform(libs.testcontainers.bom)) + implementation("org.testcontainers:testcontainers-junit-jupiter") + implementation(project(":polaris-container-spec-helper")) + } + targets { + all { + val buildDir = project.layout.buildDirectory + val policyDir = + project.layout.projectDirectory.dir("src/intTest/resources/authz_it_tests") + testTask.configure { + environment( + "AWS_REGION", + providers.environmentVariable("AWS_REGION").getOrElse("us-west-2"), + ) + environment(mapOf("POLARIS_BOOTSTRAP_CREDENTIALS" to "POLARIS,test-admin,test-secret")) + val apiVersion = providers.environmentVariable("DOCKER_API_VERSION").getOrElse("1.44") + systemProperty("api.version", apiVersion) + jvmArgs("--add-exports", "java.base/sun.nio.ch=ALL-UNNAMED") + systemProperty("java.security.manager", "allow") + maxParallelForks = 1 + + val buildDirFile = buildDir.get().asFile + val logsDir = buildDirFile.resolve("logs") + + doFirst { + logsDir.deleteRecursively() + buildDirFile.resolve("quarkus.log").delete() + } + + withPolarisServer(configurations.polarisServer) { + environment.put( + "AWS_REGION", + providers.environmentVariable("AWS_REGION").orElse("us-west-2"), + ) + environment.putAll( + mapOf("POLARIS_BOOTSTRAP_CREDENTIALS" to "POLARIS,test-admin,test-secret") + ) + systemProperties.putAll( + mapOf( + "quarkus.log.file.path" to logsDir.resolve("polaris.log").absolutePath, + "polaris.authorization.type" to "ranger", + "polaris.authorization.ranger.service-name" to "dev_polaris", + "polaris.authorization.ranger.authz.default.policy.source.impl" to + "org.apache.ranger.admin.client.LocalFolderPolicySource", + "polaris.authorization.ranger.authz.default.enable.implicit.userstore.enricher" to + "true", + "polaris.authorization.ranger.authz.default.policy.source.local_folder.path" to + policyDir.asFile.absolutePath, + "polaris.features.\"SUPPORTED_CATALOG_STORAGE_TYPES\"" to "[\"FILE\"]", + "polaris.features.\"ALLOW_INSECURE_STORAGE_TYPES\"" to "true", + "polaris.readiness.ignore-severe-issues" to "true", + ) + ) + } + } + } + } + } + } +} + +listOf("intTestCompileClasspath", "intTestRuntimeClasspath").forEach { + configurations.named(it).configure { + attributes.attribute(TargetJvmVersion.TARGET_JVM_VERSION_ATTRIBUTE, intTestJvmVersion) + } +} diff --git a/extensions/auth/ranger/impl/build.gradle.kts b/extensions/auth/ranger/impl/build.gradle.kts deleted file mode 100644 index c4437ce2335..00000000000 --- a/extensions/auth/ranger/impl/build.gradle.kts +++ /dev/null @@ -1,51 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -plugins { - id("polaris-server") - id("org.kordamp.gradle.jandex") -} - -dependencies { - implementation(project(":polaris-core")) - - implementation(libs.ranger.authz.embedded) { - exclude("org.apache.ranger", "ranger-audit-dest-hdfs") - exclude("org.slf4j", "slf4j-reload4j") - exclude("ch.qos.reload4j", "reload4j") - exclude("io.dropwizard.metrics", "metrics-core") - } - - implementation(libs.commons.lang3) - implementation(libs.guava) - - // Iceberg dependency for ForbiddenException - implementation(platform(libs.iceberg.bom)) - implementation("org.apache.iceberg:iceberg-api") - - compileOnly(libs.jspecify) - compileOnly(libs.jakarta.enterprise.cdi.api) - compileOnly(libs.jakarta.inject.api) - compileOnly(libs.smallrye.config.core) - compileOnly(project(":polaris-immutables")) - - runtimeOnly(libs.graalvm.js.js.scriptengine) - runtimeOnly(libs.graalvm.polyglot.js) - runtimeOnly(libs.graalvm.polyglot.polyglot) -} diff --git a/extensions/auth/ranger/tests/src/intTest/java/org/apache/polaris/extension/auth/ranger/test/RangerAdminServiceIT.java b/extensions/auth/ranger/src/intTest/java/org/apache/polaris/extension/auth/ranger/test/RangerAdminServiceIT.java similarity index 98% rename from extensions/auth/ranger/tests/src/intTest/java/org/apache/polaris/extension/auth/ranger/test/RangerAdminServiceIT.java rename to extensions/auth/ranger/src/intTest/java/org/apache/polaris/extension/auth/ranger/test/RangerAdminServiceIT.java index 1bfcd1c3062..7b62a27e863 100644 --- a/extensions/auth/ranger/tests/src/intTest/java/org/apache/polaris/extension/auth/ranger/test/RangerAdminServiceIT.java +++ b/extensions/auth/ranger/src/intTest/java/org/apache/polaris/extension/auth/ranger/test/RangerAdminServiceIT.java @@ -20,8 +20,6 @@ import static io.restassured.RestAssured.given; -import io.quarkus.test.junit.QuarkusTest; -import io.quarkus.test.junit.TestProfile; import io.restassured.http.ContentType; import java.net.URI; import java.nio.file.Files; @@ -38,8 +36,6 @@ import org.junit.jupiter.api.Test; import org.junit.jupiter.api.io.TempDir; -@QuarkusTest -@TestProfile(RangerTestProfiles.EmbeddedPolicy.class) public class RangerAdminServiceIT extends RangerIntegrationTestBase { private @TempDir Path baseCatalogLocationPath; diff --git a/extensions/auth/ranger/tests/src/intTest/java/org/apache/polaris/extension/auth/ranger/test/RangerGenericTableHandlerIT.java b/extensions/auth/ranger/src/intTest/java/org/apache/polaris/extension/auth/ranger/test/RangerGenericTableHandlerIT.java similarity index 96% rename from extensions/auth/ranger/tests/src/intTest/java/org/apache/polaris/extension/auth/ranger/test/RangerGenericTableHandlerIT.java rename to extensions/auth/ranger/src/intTest/java/org/apache/polaris/extension/auth/ranger/test/RangerGenericTableHandlerIT.java index 54a83bdf302..f5bace0a9c7 100644 --- a/extensions/auth/ranger/tests/src/intTest/java/org/apache/polaris/extension/auth/ranger/test/RangerGenericTableHandlerIT.java +++ b/extensions/auth/ranger/src/intTest/java/org/apache/polaris/extension/auth/ranger/test/RangerGenericTableHandlerIT.java @@ -20,8 +20,6 @@ import static io.restassured.RestAssured.given; -import io.quarkus.test.junit.QuarkusTest; -import io.quarkus.test.junit.TestProfile; import io.restassured.http.ContentType; import java.nio.file.Files; import java.nio.file.Path; @@ -32,8 +30,6 @@ import org.junit.jupiter.api.Test; import org.junit.jupiter.api.io.TempDir; -@QuarkusTest -@TestProfile(RangerTestProfiles.EmbeddedPolicy.class) public class RangerGenericTableHandlerIT extends RangerIntegrationTestBase { private String catalogName; diff --git a/extensions/auth/ranger/tests/src/intTest/java/org/apache/polaris/extension/auth/ranger/test/RangerIcebergCatalogHandlerIT.java b/extensions/auth/ranger/src/intTest/java/org/apache/polaris/extension/auth/ranger/test/RangerIcebergCatalogHandlerIT.java similarity index 97% rename from extensions/auth/ranger/tests/src/intTest/java/org/apache/polaris/extension/auth/ranger/test/RangerIcebergCatalogHandlerIT.java rename to extensions/auth/ranger/src/intTest/java/org/apache/polaris/extension/auth/ranger/test/RangerIcebergCatalogHandlerIT.java index 18399ff57e7..d2c8d5a6b1c 100644 --- a/extensions/auth/ranger/tests/src/intTest/java/org/apache/polaris/extension/auth/ranger/test/RangerIcebergCatalogHandlerIT.java +++ b/extensions/auth/ranger/src/intTest/java/org/apache/polaris/extension/auth/ranger/test/RangerIcebergCatalogHandlerIT.java @@ -20,8 +20,6 @@ import static io.restassured.RestAssured.given; -import io.quarkus.test.junit.QuarkusTest; -import io.quarkus.test.junit.TestProfile; import io.restassured.http.ContentType; import java.net.URI; import java.nio.file.Files; @@ -38,8 +36,6 @@ import org.junit.jupiter.api.Test; import org.junit.jupiter.api.io.TempDir; -@QuarkusTest -@TestProfile(RangerTestProfiles.EmbeddedPolicy.class) public class RangerIcebergCatalogHandlerIT extends RangerIntegrationTestBase { private String catalogName; diff --git a/extensions/auth/ranger/tests/src/intTest/java/org/apache/polaris/extension/auth/ranger/test/RangerIntegrationTest.java b/extensions/auth/ranger/src/intTest/java/org/apache/polaris/extension/auth/ranger/test/RangerIntegrationTest.java similarity index 95% rename from extensions/auth/ranger/tests/src/intTest/java/org/apache/polaris/extension/auth/ranger/test/RangerIntegrationTest.java rename to extensions/auth/ranger/src/intTest/java/org/apache/polaris/extension/auth/ranger/test/RangerIntegrationTest.java index a808e238cc5..e9e10b11bf7 100644 --- a/extensions/auth/ranger/tests/src/intTest/java/org/apache/polaris/extension/auth/ranger/test/RangerIntegrationTest.java +++ b/extensions/auth/ranger/src/intTest/java/org/apache/polaris/extension/auth/ranger/test/RangerIntegrationTest.java @@ -20,8 +20,6 @@ import static io.restassured.RestAssured.given; -import io.quarkus.test.junit.QuarkusTest; -import io.quarkus.test.junit.TestProfile; import io.restassured.http.ContentType; import java.nio.file.Path; import java.util.List; @@ -30,8 +28,6 @@ import org.junit.jupiter.api.Test; import org.junit.jupiter.api.io.TempDir; -@QuarkusTest -@TestProfile(RangerTestProfiles.EmbeddedPolicy.class) public class RangerIntegrationTest extends RangerIntegrationTestBase { @Test diff --git a/extensions/auth/ranger/tests/src/intTest/java/org/apache/polaris/extension/auth/ranger/test/RangerIntegrationTestBase.java b/extensions/auth/ranger/src/intTest/java/org/apache/polaris/extension/auth/ranger/test/RangerIntegrationTestBase.java similarity index 95% rename from extensions/auth/ranger/tests/src/intTest/java/org/apache/polaris/extension/auth/ranger/test/RangerIntegrationTestBase.java rename to extensions/auth/ranger/src/intTest/java/org/apache/polaris/extension/auth/ranger/test/RangerIntegrationTestBase.java index 89403eea084..b4f150b59ad 100644 --- a/extensions/auth/ranger/tests/src/intTest/java/org/apache/polaris/extension/auth/ranger/test/RangerIntegrationTestBase.java +++ b/extensions/auth/ranger/src/intTest/java/org/apache/polaris/extension/auth/ranger/test/RangerIntegrationTestBase.java @@ -23,6 +23,7 @@ import com.fasterxml.jackson.databind.JsonNode; import com.fasterxml.jackson.databind.json.JsonMapper; +import io.restassured.RestAssured; import io.restassured.http.ContentType; import java.io.UncheckedIOException; import java.util.ArrayList; @@ -30,7 +31,9 @@ import java.util.HashMap; import java.util.List; import java.util.Map; +import org.junit.jupiter.api.AfterAll; import org.junit.jupiter.api.AfterEach; +import org.junit.jupiter.api.BeforeAll; /** * Base class for Ranger integration tests providing common helpers for authentication and principal @@ -43,6 +46,17 @@ public abstract class RangerIntegrationTestBase { protected Map user2Token = new HashMap<>(); + @BeforeAll + static void configureRestAssured() { + RestAssured.baseURI = "http://localhost"; + RestAssured.port = Integer.getInteger("quarkus.http.test-port"); + } + + @AfterAll + static void resetRestAssured() { + RestAssured.reset(); + } + protected String toJson(Object value) { try { return mapper.writeValueAsString(value); diff --git a/extensions/auth/ranger/tests/src/intTest/java/org/apache/polaris/extension/auth/ranger/test/RangerPolicyCatalogHandlerIT.java b/extensions/auth/ranger/src/intTest/java/org/apache/polaris/extension/auth/ranger/test/RangerPolicyCatalogHandlerIT.java similarity index 97% rename from extensions/auth/ranger/tests/src/intTest/java/org/apache/polaris/extension/auth/ranger/test/RangerPolicyCatalogHandlerIT.java rename to extensions/auth/ranger/src/intTest/java/org/apache/polaris/extension/auth/ranger/test/RangerPolicyCatalogHandlerIT.java index 8994693f604..5b9cb74206b 100644 --- a/extensions/auth/ranger/tests/src/intTest/java/org/apache/polaris/extension/auth/ranger/test/RangerPolicyCatalogHandlerIT.java +++ b/extensions/auth/ranger/src/intTest/java/org/apache/polaris/extension/auth/ranger/test/RangerPolicyCatalogHandlerIT.java @@ -20,8 +20,6 @@ import static io.restassured.RestAssured.given; -import io.quarkus.test.junit.QuarkusTest; -import io.quarkus.test.junit.TestProfile; import io.restassured.http.ContentType; import java.nio.file.Files; import java.nio.file.Path; @@ -32,8 +30,6 @@ import org.junit.jupiter.api.Test; import org.junit.jupiter.api.io.TempDir; -@QuarkusTest -@TestProfile(RangerTestProfiles.EmbeddedPolicy.class) public class RangerPolicyCatalogHandlerIT extends RangerIntegrationTestBase { private String catalogName; diff --git a/extensions/auth/ranger/tests/src/intTest/java/org/apache/polaris/extension/auth/ranger/test/RangerPolicyConditionIT.java b/extensions/auth/ranger/src/intTest/java/org/apache/polaris/extension/auth/ranger/test/RangerPolicyConditionIT.java similarity index 91% rename from extensions/auth/ranger/tests/src/intTest/java/org/apache/polaris/extension/auth/ranger/test/RangerPolicyConditionIT.java rename to extensions/auth/ranger/src/intTest/java/org/apache/polaris/extension/auth/ranger/test/RangerPolicyConditionIT.java index 2cdb05af415..a0e2167c6e9 100644 --- a/extensions/auth/ranger/tests/src/intTest/java/org/apache/polaris/extension/auth/ranger/test/RangerPolicyConditionIT.java +++ b/extensions/auth/ranger/src/intTest/java/org/apache/polaris/extension/auth/ranger/test/RangerPolicyConditionIT.java @@ -20,13 +20,9 @@ import static io.restassured.RestAssured.given; -import io.quarkus.test.junit.QuarkusTest; -import io.quarkus.test.junit.TestProfile; import io.restassured.http.ContentType; import org.junit.jupiter.api.Test; -@QuarkusTest -@TestProfile(RangerTestProfiles.EmbeddedPolicy.class) public class RangerPolicyConditionIT extends RangerIntegrationTestBase { @Test diff --git a/extensions/auth/ranger/src/intTest/resources/META-INF/services/org.apache.polaris.service.it.ext.PolarisServerManager b/extensions/auth/ranger/src/intTest/resources/META-INF/services/org.apache.polaris.service.it.ext.PolarisServerManager new file mode 100644 index 00000000000..6aa2dac9e0c --- /dev/null +++ b/extensions/auth/ranger/src/intTest/resources/META-INF/services/org.apache.polaris.service.it.ext.PolarisServerManager @@ -0,0 +1,20 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# + +org.apache.polaris.service.it.ext.ExternalPolarisServerManager diff --git a/extensions/auth/ranger/tests/src/intTest/resources/authz_it_tests/dev_polaris.json b/extensions/auth/ranger/src/intTest/resources/authz_it_tests/dev_polaris.json similarity index 100% rename from extensions/auth/ranger/tests/src/intTest/resources/authz_it_tests/dev_polaris.json rename to extensions/auth/ranger/src/intTest/resources/authz_it_tests/dev_polaris.json diff --git a/extensions/auth/ranger/tests/src/intTest/resources/authz_it_tests/dev_polaris_roles.json b/extensions/auth/ranger/src/intTest/resources/authz_it_tests/dev_polaris_roles.json similarity index 100% rename from extensions/auth/ranger/tests/src/intTest/resources/authz_it_tests/dev_polaris_roles.json rename to extensions/auth/ranger/src/intTest/resources/authz_it_tests/dev_polaris_roles.json diff --git a/extensions/auth/ranger/tests/src/intTest/resources/authz_it_tests/dev_polaris_userstore.json b/extensions/auth/ranger/src/intTest/resources/authz_it_tests/dev_polaris_userstore.json similarity index 100% rename from extensions/auth/ranger/tests/src/intTest/resources/authz_it_tests/dev_polaris_userstore.json rename to extensions/auth/ranger/src/intTest/resources/authz_it_tests/dev_polaris_userstore.json diff --git a/extensions/auth/ranger/impl/src/main/java/org/apache/polaris/extension/auth/ranger/RangerPolarisAuthorizer.java b/extensions/auth/ranger/src/main/java/org/apache/polaris/extension/auth/ranger/RangerPolarisAuthorizer.java similarity index 100% rename from extensions/auth/ranger/impl/src/main/java/org/apache/polaris/extension/auth/ranger/RangerPolarisAuthorizer.java rename to extensions/auth/ranger/src/main/java/org/apache/polaris/extension/auth/ranger/RangerPolarisAuthorizer.java diff --git a/extensions/auth/ranger/impl/src/main/java/org/apache/polaris/extension/auth/ranger/RangerPolarisAuthorizerConfig.java b/extensions/auth/ranger/src/main/java/org/apache/polaris/extension/auth/ranger/RangerPolarisAuthorizerConfig.java similarity index 100% rename from extensions/auth/ranger/impl/src/main/java/org/apache/polaris/extension/auth/ranger/RangerPolarisAuthorizerConfig.java rename to extensions/auth/ranger/src/main/java/org/apache/polaris/extension/auth/ranger/RangerPolarisAuthorizerConfig.java diff --git a/extensions/auth/ranger/impl/src/main/java/org/apache/polaris/extension/auth/ranger/RangerPolarisAuthorizerFactory.java b/extensions/auth/ranger/src/main/java/org/apache/polaris/extension/auth/ranger/RangerPolarisAuthorizerFactory.java similarity index 100% rename from extensions/auth/ranger/impl/src/main/java/org/apache/polaris/extension/auth/ranger/RangerPolarisAuthorizerFactory.java rename to extensions/auth/ranger/src/main/java/org/apache/polaris/extension/auth/ranger/RangerPolarisAuthorizerFactory.java diff --git a/extensions/auth/ranger/impl/src/main/java/org/apache/polaris/extension/auth/ranger/RangerPolarisOperationSemantics.java b/extensions/auth/ranger/src/main/java/org/apache/polaris/extension/auth/ranger/RangerPolarisOperationSemantics.java similarity index 100% rename from extensions/auth/ranger/impl/src/main/java/org/apache/polaris/extension/auth/ranger/RangerPolarisOperationSemantics.java rename to extensions/auth/ranger/src/main/java/org/apache/polaris/extension/auth/ranger/RangerPolarisOperationSemantics.java diff --git a/extensions/auth/ranger/impl/src/main/java/org/apache/polaris/extension/auth/ranger/utils/RangerUtils.java b/extensions/auth/ranger/src/main/java/org/apache/polaris/extension/auth/ranger/utils/RangerUtils.java similarity index 100% rename from extensions/auth/ranger/impl/src/main/java/org/apache/polaris/extension/auth/ranger/utils/RangerUtils.java rename to extensions/auth/ranger/src/main/java/org/apache/polaris/extension/auth/ranger/utils/RangerUtils.java diff --git a/extensions/auth/ranger/impl/src/test/java/org/apache/polaris/extension/auth/ranger/RangerPolarisAuthorizerFactoryTest.java b/extensions/auth/ranger/src/test/java/org/apache/polaris/extension/auth/ranger/RangerPolarisAuthorizerFactoryTest.java similarity index 100% rename from extensions/auth/ranger/impl/src/test/java/org/apache/polaris/extension/auth/ranger/RangerPolarisAuthorizerFactoryTest.java rename to extensions/auth/ranger/src/test/java/org/apache/polaris/extension/auth/ranger/RangerPolarisAuthorizerFactoryTest.java diff --git a/extensions/auth/ranger/impl/src/test/java/org/apache/polaris/extension/auth/ranger/RangerPolarisAuthorizerTest.java b/extensions/auth/ranger/src/test/java/org/apache/polaris/extension/auth/ranger/RangerPolarisAuthorizerTest.java similarity index 100% rename from extensions/auth/ranger/impl/src/test/java/org/apache/polaris/extension/auth/ranger/RangerPolarisAuthorizerTest.java rename to extensions/auth/ranger/src/test/java/org/apache/polaris/extension/auth/ranger/RangerPolarisAuthorizerTest.java diff --git a/extensions/auth/ranger/impl/src/test/java/org/apache/polaris/extension/auth/ranger/RangerTestUtils.java b/extensions/auth/ranger/src/test/java/org/apache/polaris/extension/auth/ranger/RangerTestUtils.java similarity index 100% rename from extensions/auth/ranger/impl/src/test/java/org/apache/polaris/extension/auth/ranger/RangerTestUtils.java rename to extensions/auth/ranger/src/test/java/org/apache/polaris/extension/auth/ranger/RangerTestUtils.java diff --git a/extensions/auth/ranger/impl/src/test/resources/authz_tests/dev_polaris.json b/extensions/auth/ranger/src/test/resources/authz_tests/dev_polaris.json similarity index 100% rename from extensions/auth/ranger/impl/src/test/resources/authz_tests/dev_polaris.json rename to extensions/auth/ranger/src/test/resources/authz_tests/dev_polaris.json diff --git a/extensions/auth/ranger/impl/src/test/resources/authz_tests/tests_authz_catalog.json b/extensions/auth/ranger/src/test/resources/authz_tests/tests_authz_catalog.json similarity index 100% rename from extensions/auth/ranger/impl/src/test/resources/authz_tests/tests_authz_catalog.json rename to extensions/auth/ranger/src/test/resources/authz_tests/tests_authz_catalog.json diff --git a/extensions/auth/ranger/impl/src/test/resources/authz_tests/tests_authz_namespace.json b/extensions/auth/ranger/src/test/resources/authz_tests/tests_authz_namespace.json similarity index 100% rename from extensions/auth/ranger/impl/src/test/resources/authz_tests/tests_authz_namespace.json rename to extensions/auth/ranger/src/test/resources/authz_tests/tests_authz_namespace.json diff --git a/extensions/auth/ranger/impl/src/test/resources/authz_tests/tests_authz_policy.json b/extensions/auth/ranger/src/test/resources/authz_tests/tests_authz_policy.json similarity index 100% rename from extensions/auth/ranger/impl/src/test/resources/authz_tests/tests_authz_policy.json rename to extensions/auth/ranger/src/test/resources/authz_tests/tests_authz_policy.json diff --git a/extensions/auth/ranger/impl/src/test/resources/authz_tests/tests_authz_principal.json b/extensions/auth/ranger/src/test/resources/authz_tests/tests_authz_principal.json similarity index 100% rename from extensions/auth/ranger/impl/src/test/resources/authz_tests/tests_authz_principal.json rename to extensions/auth/ranger/src/test/resources/authz_tests/tests_authz_principal.json diff --git a/extensions/auth/ranger/impl/src/test/resources/authz_tests/tests_authz_root.json b/extensions/auth/ranger/src/test/resources/authz_tests/tests_authz_root.json similarity index 100% rename from extensions/auth/ranger/impl/src/test/resources/authz_tests/tests_authz_root.json rename to extensions/auth/ranger/src/test/resources/authz_tests/tests_authz_root.json diff --git a/extensions/auth/ranger/impl/src/test/resources/authz_tests/tests_authz_table.json b/extensions/auth/ranger/src/test/resources/authz_tests/tests_authz_table.json similarity index 100% rename from extensions/auth/ranger/impl/src/test/resources/authz_tests/tests_authz_table.json rename to extensions/auth/ranger/src/test/resources/authz_tests/tests_authz_table.json diff --git a/extensions/auth/ranger/impl/src/test/resources/authz_tests/tests_authz_unsupported.json b/extensions/auth/ranger/src/test/resources/authz_tests/tests_authz_unsupported.json similarity index 100% rename from extensions/auth/ranger/impl/src/test/resources/authz_tests/tests_authz_unsupported.json rename to extensions/auth/ranger/src/test/resources/authz_tests/tests_authz_unsupported.json diff --git a/extensions/auth/ranger/impl/src/test/resources/logback-test.xml b/extensions/auth/ranger/src/test/resources/logback-test.xml similarity index 100% rename from extensions/auth/ranger/impl/src/test/resources/logback-test.xml rename to extensions/auth/ranger/src/test/resources/logback-test.xml diff --git a/extensions/auth/ranger/tests/build.gradle.kts b/extensions/auth/ranger/tests/build.gradle.kts deleted file mode 100644 index 418d0c7fbaa..00000000000 --- a/extensions/auth/ranger/tests/build.gradle.kts +++ /dev/null @@ -1,78 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -plugins { - alias(libs.plugins.quarkus) - id("org.kordamp.gradle.jandex") - id("polaris-runtime") -} - -dependencies { - implementation(platform(libs.quarkus.bom)) - implementation("io.quarkus:quarkus-rest-jackson") - - implementation(project(":polaris-extensions-auth-ranger")) - implementation(project(":polaris-runtime-service")) - - testImplementation(project(":polaris-runtime-test-common")) - - intTestImplementation("io.quarkus:quarkus-junit") - intTestImplementation("io.rest-assured:rest-assured") - intTestImplementation(project(":polaris-api-management-model")) - intTestImplementation(platform(libs.iceberg.bom)) - intTestImplementation("org.apache.iceberg:iceberg-api") - intTestImplementation("org.apache.iceberg:iceberg-core") - - intTestImplementation(platform(libs.testcontainers.bom)) - intTestImplementation("org.testcontainers:testcontainers-junit-jupiter") - intTestImplementation(project(":polaris-container-spec-helper")) -} - -sourceSets.named("intTest") { - resources.srcDir( - project(":polaris-extensions-auth-ranger").layout.projectDirectory.dir("src/test/resources") - ) -} - -tasks.named("javadoc") { dependsOn("jandex") } - -tasks.withType { - if (System.getenv("AWS_REGION") == null) { - environment("AWS_REGION", "us-west-2") - } - environment("POLARIS_BOOTSTRAP_CREDENTIALS", "POLARIS,test-admin,test-secret") - val apiVersion = System.getenv("DOCKER_API_VERSION") ?: "1.44" - systemProperty("api.version", apiVersion) - jvmArgs("--add-exports", "java.base/sun.nio.ch=ALL-UNNAMED") - systemProperty("java.security.manager", "allow") - maxParallelForks = 1 - - val logsDir = project.layout.buildDirectory.get().asFile.resolve("logs") - - jvmArgumentProviders.add( - CommandLineArgumentProvider { - listOf("-Dquarkus.log.file.path=${logsDir.resolve("polaris.log").absolutePath}") - } - ) - - doFirst { - logsDir.deleteRecursively() - project.layout.buildDirectory.get().asFile.resolve("quarkus.log").delete() - } -} diff --git a/extensions/auth/ranger/tests/src/intTest/java/org/apache/polaris/extension/auth/ranger/test/RangerTestProfiles.java b/extensions/auth/ranger/tests/src/intTest/java/org/apache/polaris/extension/auth/ranger/test/RangerTestProfiles.java deleted file mode 100644 index d91b6d4f174..00000000000 --- a/extensions/auth/ranger/tests/src/intTest/java/org/apache/polaris/extension/auth/ranger/test/RangerTestProfiles.java +++ /dev/null @@ -1,51 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ -package org.apache.polaris.extension.auth.ranger.test; - -import io.quarkus.test.junit.QuarkusTestProfile; -import java.util.HashMap; -import java.util.Map; - -/** Quarkus test profiles for Ranger integration tests using embedded policy fixtures. */ -public final class RangerTestProfiles { - - private RangerTestProfiles() {} - - /** Ranger authorizer with policies loaded from classpath ({@code /authz_tests}). */ - public static class EmbeddedPolicy implements QuarkusTestProfile { - @Override - public Map getConfigOverrides() { - Map config = new HashMap<>(); - config.put("polaris.authorization.type", "ranger"); - config.put("polaris.authorization.ranger.service-name", "dev_polaris"); - config.put( - "polaris.authorization.ranger.authz.default.policy.source.impl", - "org.apache.ranger.admin.client.EmbeddedResourcePolicySource"); - config.put( - "polaris.authorization.ranger.authz.default.enable.implicit.userstore.enricher", "true"); - config.put( - "polaris.authorization.ranger.authz.default.policy.source.embedded_resource.path", - "/authz_it_tests"); - config.put("polaris.features.\"SUPPORTED_CATALOG_STORAGE_TYPES\"", "[\"FILE\"]"); - config.put("polaris.features.\"ALLOW_INSECURE_STORAGE_TYPES\"", "true"); - config.put("polaris.readiness.ignore-severe-issues", "true"); - return config; - } - } -} diff --git a/gradle/projects.main.properties b/gradle/projects.main.properties index 322510c5578..cae568b92e2 100644 --- a/gradle/projects.main.properties +++ b/gradle/projects.main.properties @@ -47,10 +47,8 @@ polaris-misc-types=tools/misc-types polaris-extensions-federation-hadoop=extensions/federation/hadoop polaris-extensions-federation-hive=extensions/federation/hive polaris-extensions-federation-bigquery=extensions/federation/bigquery -polaris-extensions-auth-opa=extensions/auth/opa/impl -polaris-extensions-auth-opa-tests=extensions/auth/opa/tests -polaris-extensions-auth-ranger=extensions/auth/ranger/impl -polaris-extensions-auth-ranger-tests=extensions/auth/ranger/tests +polaris-extensions-auth-opa=extensions/auth/opa +polaris-extensions-auth-ranger=extensions/auth/ranger polaris-config-docs-annotations=tools/config-docs/annotations polaris-config-docs-generator=tools/config-docs/generator diff --git a/gradle/server-test-runner/build.gradle.kts b/gradle/server-test-runner/build.gradle.kts new file mode 100644 index 00000000000..504ebff8fce --- /dev/null +++ b/gradle/server-test-runner/build.gradle.kts @@ -0,0 +1,43 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +plugins { + `kotlin-dsl` + `java-gradle-plugin` +} + +gradlePlugin { + plugins { + create("polarisServerTestRunner") { + id = "polaris-server-test-runner" + implementationClass = "org.apache.polaris.server.test.runner.PolarisServerTestRunnerPlugin" + } + } +} + +testing { suites { named("test") { useJUnitJupiter() } } } + +dependencies { + testImplementation(gradleTestKit()) + testImplementation(platform(libs.junit.bom)) + testImplementation("org.junit.jupiter:junit-jupiter") + testImplementation(libs.assertj.core) +} + +group = "org.apache.polaris.server-test-runner" diff --git a/gradle/server-test-runner/gradle/libs.versions.toml b/gradle/server-test-runner/gradle/libs.versions.toml new file mode 100644 index 00000000000..a8e5324ce7b --- /dev/null +++ b/gradle/server-test-runner/gradle/libs.versions.toml @@ -0,0 +1,22 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# + +[libraries] +assertj-core = { module = "org.assertj:assertj-core", version = "3.27.7" } +junit-bom = { module = "org.junit:junit-bom", version = "6.1.0" } diff --git a/runtime/admin/src/test/java/org/apache/polaris/admintool/nosql/NoSqlInMemoryProfile.java b/gradle/server-test-runner/settings.gradle.kts similarity index 68% rename from runtime/admin/src/test/java/org/apache/polaris/admintool/nosql/NoSqlInMemoryProfile.java rename to gradle/server-test-runner/settings.gradle.kts index 024631403da..0f3d5c54557 100644 --- a/runtime/admin/src/test/java/org/apache/polaris/admintool/nosql/NoSqlInMemoryProfile.java +++ b/gradle/server-test-runner/settings.gradle.kts @@ -16,15 +16,9 @@ * specific language governing permissions and limitations * under the License. */ -package org.apache.polaris.admintool.nosql; -import io.quarkus.test.junit.QuarkusTestProfile; -import java.util.Map; +pluginManagement { repositories { gradlePluginPortal() } } -public class NoSqlInMemoryProfile implements QuarkusTestProfile { - @Override - public Map getConfigOverrides() { - return Map.of( - "polaris.persistence.type", "nosql", "polaris.persistence.nosql.backend", "InMemory"); - } -} +dependencyResolutionManagement { repositories { mavenCentral() } } + +rootProject.name = "polaris-server-test-runner" diff --git a/gradle/server-test-runner/src/main/java/org/apache/polaris/server/test/runner/spi/PolarisServerStartupAction.java b/gradle/server-test-runner/src/main/java/org/apache/polaris/server/test/runner/spi/PolarisServerStartupAction.java new file mode 100644 index 00000000000..4301579999c --- /dev/null +++ b/gradle/server-test-runner/src/main/java/org/apache/polaris/server/test/runner/spi/PolarisServerStartupAction.java @@ -0,0 +1,36 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.polaris.server.test.runner.spi; + +/** + * Isolated startup action that can prepare external services before the Polaris server process + * starts. + * + *

    Implementations are loaded from a child classloader configured by the build and may mutate the + * provided context to pass task-specific system properties or environment variables to the Polaris + * server. Implementations are closed after the decorated test task finishes, or when startup fails. + */ +public interface PolarisServerStartupAction extends AutoCloseable { + /** Starts required services and updates the Polaris server startup context. */ + void start(PolarisServerStartupContext context) throws Exception; + + /** Stops resources owned by this action. */ + @Override + default void close() throws Exception {} +} diff --git a/runtime/admin/src/test/java/org/apache/polaris/admintool/nosql/NoSqlMongoProfile.java b/gradle/server-test-runner/src/main/java/org/apache/polaris/server/test/runner/spi/PolarisServerStartupContext.java similarity index 58% rename from runtime/admin/src/test/java/org/apache/polaris/admintool/nosql/NoSqlMongoProfile.java rename to gradle/server-test-runner/src/main/java/org/apache/polaris/server/test/runner/spi/PolarisServerStartupContext.java index 4b6f9959d11..8c1233aee4a 100644 --- a/runtime/admin/src/test/java/org/apache/polaris/admintool/nosql/NoSqlMongoProfile.java +++ b/gradle/server-test-runner/src/main/java/org/apache/polaris/server/test/runner/spi/PolarisServerStartupContext.java @@ -16,22 +16,18 @@ * specific language governing permissions and limitations * under the License. */ -package org.apache.polaris.admintool.nosql; +package org.apache.polaris.server.test.runner.spi; -import io.quarkus.test.junit.QuarkusTestProfile; -import java.util.List; import java.util.Map; -import org.apache.polaris.admintool.MongoTestResourceLifecycleManager; -public class NoSqlMongoProfile implements QuarkusTestProfile { - @Override - public Map getConfigOverrides() { - return Map.of( - "polaris.persistence.type", "nosql", "polaris.persistence.nosql.backend", "MongoDb"); - } +/** Context passed to an isolated {@link PolarisServerStartupAction}. */ +public interface PolarisServerStartupContext { + /** Build-configured action parameters. */ + Map getParameters(); - @Override - public List testResources() { - return List.of(new TestResourceEntry(MongoTestResourceLifecycleManager.class)); - } + /** Mutable system properties that will be passed to the Polaris server JVM. */ + Map getSystemProperties(); + + /** Mutable environment variables that will be passed to the Polaris server process. */ + Map getEnvironment(); } diff --git a/gradle/server-test-runner/src/main/kotlin/PolarisServerTestRunnerExtensions.kt b/gradle/server-test-runner/src/main/kotlin/PolarisServerTestRunnerExtensions.kt new file mode 100644 index 00000000000..e994a4c4754 --- /dev/null +++ b/gradle/server-test-runner/src/main/kotlin/PolarisServerTestRunnerExtensions.kt @@ -0,0 +1,107 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +import org.apache.polaris.server.test.runner.PolarisServerTestRunnerExtension +import org.apache.polaris.server.test.runner.PolarisServerTestRunnerPlugin +import org.apache.polaris.server.test.runner.configurePolarisServer +import org.apache.polaris.server.test.runner.conventionFrom +import org.gradle.api.Action +import org.gradle.api.file.FileCollection +import org.gradle.api.provider.Provider +import org.gradle.api.tasks.testing.Test +import org.gradle.process.CommandLineArgumentProvider + +/** + * Runs a Polaris server from [server] for the duration of this [Test] task. + * + * The file collection must resolve to exactly one runnable jar. The server is started only when the + * test task executes, not during configuration. The discovered HTTP and management ports are passed + * to the test JVM using the default property names from [PolarisServerTestRunnerExtension]. + */ +fun Test.withPolarisServer(server: FileCollection) { + withPolarisServer(server) {} +} + +/** + * Runs a Polaris server from the file collection provided by [server] for the duration of this + * [Test] task. + * + * This overload is convenient for lazily named configurations such as + * `withPolarisServer(configurations.polarisServer)`. + */ +fun Test.withPolarisServer(server: Provider) { + withPolarisServer(server) {} +} + +/** + * Runs a Polaris server from [server] for the duration of this [Test] task and customizes its + * process settings using [configure]. + * + * Use this overload when the server artifact is task-specific, for example + * `withPolarisServer(configurations.polarisServer) { ... }`. Dynamic port values are passed via a + * [CommandLineArgumentProvider] so they do not become test task inputs. + */ +fun Test.withPolarisServer( + server: FileCollection, + configure: Action, +) { + val extension = polarisServerTestRunnerExtension() + extension.server.setFrom(server) + configure.execute(extension) + configurePolarisServer(extension) +} + +/** + * Runs a Polaris server from the file collection provided by [server] for the duration of this + * [Test] task and customizes its process settings using [configure]. + * + * Use this overload when the server artifact is exposed through a lazy Gradle provider, for example + * `withPolarisServer(configurations.polarisServer) { ... }`. + */ +fun Test.withPolarisServer( + server: Provider, + configure: Action, +) { + val extension = polarisServerTestRunnerExtension() + extension.server.setFrom(server) + configure.execute(extension) + configurePolarisServer(extension) +} + +/** + * Runs a Polaris server for the duration of this [Test] task using the plugin extension's + * configured [PolarisServerTestRunnerExtension.server] artifact. + * + * Use this overload when the `polarisServerTestRunner` extension is configured globally and the + * test task only needs to opt into the server lifecycle. + */ +fun Test.withPolarisServer(configure: Action) { + val extension = polarisServerTestRunnerExtension() + configure.execute(extension) + configurePolarisServer(extension) +} + +private fun Test.polarisServerTestRunnerExtension(): PolarisServerTestRunnerExtension { + val projectExtension = + project.extensions.getByName(PolarisServerTestRunnerPlugin.POLARIS_SERVER_EXTENSION) + as PolarisServerTestRunnerExtension + return project.objects + .newInstance(PolarisServerTestRunnerExtension::class.java, project, projectExtension.service) + .apply { conventionFrom(projectExtension) } +} diff --git a/gradle/server-test-runner/src/main/kotlin/org/apache/polaris/server/test/runner/PolarisServerTestRunnerExtension.kt b/gradle/server-test-runner/src/main/kotlin/org/apache/polaris/server/test/runner/PolarisServerTestRunnerExtension.kt new file mode 100644 index 00000000000..24452ec92a1 --- /dev/null +++ b/gradle/server-test-runner/src/main/kotlin/org/apache/polaris/server/test/runner/PolarisServerTestRunnerExtension.kt @@ -0,0 +1,136 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.polaris.server.test.runner + +import java.time.Duration +import javax.inject.Inject +import org.gradle.api.Project +import org.gradle.api.file.ConfigurableFileCollection +import org.gradle.api.file.DirectoryProperty +import org.gradle.api.provider.ListProperty +import org.gradle.api.provider.MapProperty +import org.gradle.api.provider.Property +import org.gradle.api.provider.Provider + +/** + * Configuration for running a Polaris server process for Gradle test tasks. + * + * The plugin starts the configured server artifact before a decorated + * [Test][org.gradle.api.tasks.testing.Test] task executes and stops it when the task's root test + * suite completes. The discovered Quarkus HTTP and management ports are passed to the test JVM + * using the property names configured here. + */ +abstract class PolarisServerTestRunnerExtension +@Inject +constructor(project: Project, internal val service: Provider) { + /** Server artifact to execute. This must resolve to exactly one runnable `java -jar` artifact. */ + val server: ConfigurableFileCollection = project.objects.fileCollection() + + /** + * Classpath used to load the optional startup action in an isolated child classloader. + * + * The startup action can prepare external services before the Polaris server starts and can add + * dynamic server system properties or environment variables through the startup context. + */ + val startupActionClasspath: ConfigurableFileCollection = project.objects.fileCollection() + + /** + * Fully qualified implementation class name of the optional startup action. + * + * The class must implement [org.apache.polaris.server.test.runner.spi.PolarisServerStartupAction] + * and have a public no-argument constructor. + */ + val startupActionClass: Property = project.objects.property(String::class.java) + + /** String parameters passed to the optional startup action. */ + val startupActionParameters: MapProperty = + project.objects.mapProperty(String::class.java, String::class.java) + + /** Working directory for the server process. Defaults to `build/polaris-server`. */ + val workingDirectory: DirectoryProperty = + project.objects + .directoryProperty() + .convention(project.layout.buildDirectory.dir("polaris-server")) + + /** Maximum time to wait for the server process to print its Quarkus listen ports. */ + val startupTimeout: Property = + project.objects.property(Duration::class.java).convention(Duration.ofSeconds(30)) + + /** Maximum time to wait for the server process to stop gracefully before it is killed. */ + val stopTimeout: Property = + project.objects.property(Duration::class.java).convention(Duration.ofSeconds(15)) + + /** Test JVM system property name that receives the discovered HTTP port. */ + val httpPortProperty: Property = + project.objects.property(String::class.java).convention("quarkus.http.test-port") + + /** Test JVM system property name that receives the discovered management port, when present. */ + val managementPortProperty: Property = + project.objects.property(String::class.java).convention("quarkus.management.test-port") + + /** + * System properties passed to the server JVM. + * + * These properties are also declared as inputs of the decorated test task. + */ + val systemProperties: MapProperty = + project.objects.mapProperty(String::class.java, String::class.java) + + /** + * Environment variables passed to the server process. + * + * These variables are also declared as inputs of the decorated test task. + */ + val environment: MapProperty = + project.objects.mapProperty(String::class.java, String::class.java) + + /** + * JVM arguments passed before `-jar` when starting the server. + * + * These arguments are also declared as inputs of the decorated test task. + */ + val jvmArguments: ListProperty = + project.objects.listProperty(String::class.java).convention(emptyList()) + + /** + * Application arguments passed after the server jar path. + * + * These arguments are also declared as inputs of the decorated test task. + */ + val arguments: ListProperty = + project.objects.listProperty(String::class.java).convention(emptyList()) +} + +internal fun PolarisServerTestRunnerExtension.conventionFrom( + other: PolarisServerTestRunnerExtension +) { + server.setFrom(other.server) + startupActionClasspath.setFrom(other.startupActionClasspath) + startupActionClass.convention(other.startupActionClass) + startupActionParameters.convention(other.startupActionParameters) + workingDirectory.convention(other.workingDirectory) + startupTimeout.convention(other.startupTimeout) + stopTimeout.convention(other.stopTimeout) + httpPortProperty.convention(other.httpPortProperty) + managementPortProperty.convention(other.managementPortProperty) + systemProperties.convention(other.systemProperties) + environment.convention(other.environment) + jvmArguments.convention(other.jvmArguments) + arguments.convention(other.arguments) +} diff --git a/gradle/server-test-runner/src/main/kotlin/org/apache/polaris/server/test/runner/PolarisServerTestRunnerPlugin.kt b/gradle/server-test-runner/src/main/kotlin/org/apache/polaris/server/test/runner/PolarisServerTestRunnerPlugin.kt new file mode 100644 index 00000000000..6dfeba8dbc5 --- /dev/null +++ b/gradle/server-test-runner/src/main/kotlin/org/apache/polaris/server/test/runner/PolarisServerTestRunnerPlugin.kt @@ -0,0 +1,62 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.polaris.server.test.runner + +import org.gradle.api.Plugin +import org.gradle.api.Project + +class PolarisServerTestRunnerPlugin : Plugin { + override fun apply(project: Project) { + val serverConfiguration = + project.configurations.create(POLARIS_SERVER_CONFIGURATION) { + isCanBeConsumed = false + isCanBeResolved = true + isTransitive = false + description = "Runnable Polaris server artifact used by integration test tasks." + } + + // We need one service per project to prevent classloader isolation issues. + // Build services are global to the current Gradle build invocation and shared + // across projects. As plugins are applied to projects, those use a "per project" + // class loader, which can then cause classic `ClassCastException` with a cause + // that `PolarisServerTestService$Inject_` from class loader A cannot be cast to + // `PolarisServerTestService` from class loader B. + // Using a project-scoped service name prevents this issue. + val serviceName = "polarisServerTestRunner-${project.path}" + val service = + project.gradle.sharedServices.registerIfAbsent( + serviceName, + PolarisServerTestService::class.java, + ) {} + + val extension = + project.extensions.create( + POLARIS_SERVER_EXTENSION, + PolarisServerTestRunnerExtension::class.java, + project, + service, + ) + extension.server.convention(serverConfiguration) + } + + companion object { + const val POLARIS_SERVER_CONFIGURATION = "polarisServer" + const val POLARIS_SERVER_EXTENSION = "polarisServerTestRunner" + } +} diff --git a/gradle/server-test-runner/src/main/kotlin/org/apache/polaris/server/test/runner/PolarisServerTestService.kt b/gradle/server-test-runner/src/main/kotlin/org/apache/polaris/server/test/runner/PolarisServerTestService.kt new file mode 100644 index 00000000000..cae1aae865e --- /dev/null +++ b/gradle/server-test-runner/src/main/kotlin/org/apache/polaris/server/test/runner/PolarisServerTestService.kt @@ -0,0 +1,54 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.polaris.server.test.runner + +import org.gradle.api.Task +import org.gradle.api.logging.Logging +import org.gradle.api.services.BuildService +import org.gradle.api.services.BuildServiceParameters + +abstract class PolarisServerTestService : BuildService, AutoCloseable { + private val servers = linkedMapOf() + private val logger = Logging.getLogger(PolarisServerTestService::class.java) + + internal fun register(task: Task, server: RunningPolarisServer) { + synchronized(servers) { + if (servers.containsKey(task.path)) { + throw IllegalStateException("Server already registered for task ${task.path}") + } + servers[task.path] = server + } + } + + fun finished(task: Task) { + val server = synchronized(servers) { servers.remove(task.path) } + server?.stop(task.logger) + } + + override fun close() { + val remaining = synchronized(servers) { servers.values.toList().also { servers.clear() } } + if (remaining.isNotEmpty()) { + logger.warn( + "Stopping {} Polaris server process(es) during build service cleanup", + remaining.size, + ) + } + remaining.forEach { it.stop(logger) } + } +} diff --git a/gradle/server-test-runner/src/main/kotlin/org/apache/polaris/server/test/runner/PolarisServerTestTaskConfigurer.kt b/gradle/server-test-runner/src/main/kotlin/org/apache/polaris/server/test/runner/PolarisServerTestTaskConfigurer.kt new file mode 100644 index 00000000000..2478341c015 --- /dev/null +++ b/gradle/server-test-runner/src/main/kotlin/org/apache/polaris/server/test/runner/PolarisServerTestTaskConfigurer.kt @@ -0,0 +1,194 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.polaris.server.test.runner + +import java.io.File +import java.net.URLClassLoader +import org.apache.polaris.server.test.runner.spi.PolarisServerStartupAction +import org.apache.polaris.server.test.runner.spi.PolarisServerStartupContext +import org.gradle.api.GradleException +import org.gradle.api.tasks.Internal +import org.gradle.api.tasks.PathSensitivity +import org.gradle.api.tasks.testing.Test +import org.gradle.api.tasks.testing.TestDescriptor +import org.gradle.api.tasks.testing.TestListener +import org.gradle.api.tasks.testing.TestResult +import org.gradle.process.CommandLineArgumentProvider + +internal fun Test.configurePolarisServer(extension: PolarisServerTestRunnerExtension) { + val dynamicArguments = PolarisServerArguments() + var startupAction: AutoCloseable? = null + jvmArgumentProviders.add(dynamicArguments) + inputs.files(extension.server).withPathSensitivity(PathSensitivity.RELATIVE) + inputs.files(extension.startupActionClasspath).withPathSensitivity(PathSensitivity.RELATIVE) + inputs.property("polarisServerStartupActionClass", extension.startupActionClass.orNull ?: "") + inputs.properties(extension.startupActionParameters.get()) + inputs.properties(extension.systemProperties.get()) + inputs.properties(extension.environment.get()) + inputs.property("polarisServerJvmArguments", extension.jvmArguments.get()) + inputs.property("polarisServerArguments", extension.arguments.get()) + usesService(extension.service) + addTestListener( + object : TestListener { + override fun beforeSuite(suite: TestDescriptor) {} + + override fun beforeTest(testDescriptor: TestDescriptor) {} + + override fun afterTest(testDescriptor: TestDescriptor, result: TestResult) {} + + override fun afterSuite(suite: TestDescriptor, result: TestResult) { + if (suite.parent == null) { + extension.service.get().finished(this@configurePolarisServer) + startupAction?.closeQuietly() + startupAction = null + } + } + } + ) + + doFirst { + val files = extension.server.files + if (files.size != 1) { + throw GradleException( + "Expected exactly one Polaris server artifact, but found ${files.size}: $files" + ) + } + val jar = files.single() + val javaExecutable = javaLauncher.get().executablePath.asFile.absolutePath + val systemProperties = extension.systemProperties.get().toMutableMap() + val environment = extension.environment.get().toMutableMap() + try { + startupAction = + extension.startupActionClass.orNull?.let { + IsolatedPolarisServerStartupAction.start( + implementationClass = it, + classpath = extension.startupActionClasspath.files, + parameters = extension.startupActionParameters.get(), + systemProperties = systemProperties, + environment = environment, + ) + } + val started = + RunningPolarisServer.start( + javaExecutable = javaExecutable, + jar = jar, + workingDirectory = extension.workingDirectory.get().asFile, + startupTimeout = extension.startupTimeout.get(), + stopTimeout = extension.stopTimeout.get(), + jvmArguments = extension.jvmArguments.get(), + systemProperties = systemProperties, + environment = environment, + arguments = extension.arguments.get(), + logger = logger, + ) + extension.service.get().register(this, started.server) + dynamicArguments.arguments = buildList { + add("-D${extension.httpPortProperty.get()}=${started.ports.httpPort}") + started.ports.managementPort?.let { add("-D${extension.managementPortProperty.get()}=$it") } + } + } catch (e: Throwable) { + startupAction?.closeQuietly() + startupAction = null + throw e + } + } +} + +private class PolarisServerArguments : CommandLineArgumentProvider { + @get:Internal var arguments: List = emptyList() + + override fun asArguments(): Iterable = arguments +} + +private class DefaultPolarisServerStartupContext( + private val parameters: Map, + private val systemProperties: MutableMap, + private val environment: MutableMap, +) : PolarisServerStartupContext { + override fun getParameters(): Map = parameters + + override fun getSystemProperties(): MutableMap = systemProperties + + override fun getEnvironment(): MutableMap = environment +} + +private class IsolatedPolarisServerStartupAction( + private val classLoader: URLClassLoader, + private val delegate: PolarisServerStartupAction, +) : AutoCloseable { + override fun close() { + withContextClassLoader(classLoader) { delegate.close() } + classLoader.close() + } + + companion object { + fun start( + implementationClass: String, + classpath: Set, + parameters: Map, + systemProperties: MutableMap, + environment: MutableMap, + ): IsolatedPolarisServerStartupAction { + val urls = classpath.map { it.toURI().toURL() }.toTypedArray() + val classLoader = URLClassLoader(urls, PolarisServerStartupAction::class.java.classLoader) + val action = + try { + withContextClassLoader(classLoader) { + val type = Class.forName(implementationClass, true, classLoader) + type.getDeclaredConstructor().newInstance() as PolarisServerStartupAction + } + } catch (e: Throwable) { + classLoader.close() + throw GradleException( + "Failed to load Polaris server startup action $implementationClass", + e, + ) + } + val isolated = IsolatedPolarisServerStartupAction(classLoader, action) + try { + withContextClassLoader(classLoader) { + action.start( + DefaultPolarisServerStartupContext(parameters, systemProperties, environment) + ) + } + } catch (e: Throwable) { + isolated.closeQuietly() + throw GradleException("Failed to run Polaris server startup action $implementationClass", e) + } + return isolated + } + } +} + +private fun AutoCloseable.closeQuietly() { + try { + close() + } catch (_: Exception) {} +} + +private fun withContextClassLoader(classLoader: ClassLoader, action: () -> T): T { + val thread = Thread.currentThread() + val previous = thread.contextClassLoader + thread.contextClassLoader = classLoader + try { + return action() + } finally { + thread.contextClassLoader = previous + } +} diff --git a/gradle/server-test-runner/src/main/kotlin/org/apache/polaris/server/test/runner/RunningPolarisServer.kt b/gradle/server-test-runner/src/main/kotlin/org/apache/polaris/server/test/runner/RunningPolarisServer.kt new file mode 100644 index 00000000000..2470a89be70 --- /dev/null +++ b/gradle/server-test-runner/src/main/kotlin/org/apache/polaris/server/test/runner/RunningPolarisServer.kt @@ -0,0 +1,167 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.polaris.server.test.runner + +import java.io.BufferedReader +import java.io.File +import java.io.InputStreamReader +import java.net.URI +import java.time.Duration +import java.util.concurrent.CountDownLatch +import java.util.concurrent.TimeUnit +import java.util.concurrent.atomic.AtomicReference +import org.gradle.api.GradleException +import org.gradle.api.logging.Logger + +internal class RunningPolarisServer( + private val process: Process, + private val outputThread: Thread, + private val stopTimeout: Duration, +) { + fun stop(logger: Logger) { + if (!process.isAlive) { + joinOutputThread(logger) + return + } + + process.destroy() + if (!process.waitFor(stopTimeout.toMillis(), TimeUnit.MILLISECONDS)) { + process.destroyForcibly() + process.waitFor(stopTimeout.toMillis(), TimeUnit.MILLISECONDS) + } + joinOutputThread(logger) + } + + private fun joinOutputThread(logger: Logger) { + try { + outputThread.join(stopTimeout.toMillis()) + } catch (e: InterruptedException) { + Thread.currentThread().interrupt() + logger.warn("Interrupted while waiting for Polaris server output thread to finish", e) + } + } + + companion object { + private val listenPattern = + Regex( + "^.*Listening on: (https?://\\S+?)(?:[.] Management interface listening on (https?://\\S+?)[.])?\\s*$" + ) + + fun start( + javaExecutable: String, + jar: File, + workingDirectory: File, + startupTimeout: Duration, + stopTimeout: Duration, + jvmArguments: List, + systemProperties: Map, + environment: Map, + arguments: List, + logger: Logger, + ): StartedServer { + workingDirectory.mkdirs() + val command = buildList { + add(javaExecutable) + addAll(jvmArguments) + add("-Dquarkus.http.port=0") + add("-Dquarkus.management.port=0") + systemProperties.forEach { (name, value) -> add("-D$name=$value") } + add("-jar") + add(jar.absolutePath) + addAll(arguments) + } + + logger.info("Starting Polaris server process: {}", command) + val processBuilder = + ProcessBuilder(command).directory(workingDirectory).redirectErrorStream(true) + processBuilder.environment().putAll(environment) + val process = processBuilder.start() + val detected = AtomicReference() + val failed = AtomicReference() + val ready = CountDownLatch(1) + val capturedOutput = mutableListOf() + + val outputThread = + Thread( + { + try { + BufferedReader(InputStreamReader(process.inputStream)).useLines { lines -> + lines.forEach { line -> + logger.info("[polaris-server] {}", line) + synchronized(capturedOutput) { capturedOutput.add(line) } + if (detected.get() == null) { + val match = listenPattern.matchEntire(line) + if (match != null) { + detected.set( + ListenUrls( + match.groupValues[1], + match.groupValues.getOrNull(2).orEmpty().ifEmpty { null }, + ) + ) + ready.countDown() + } + } + } + } + } catch (e: Throwable) { + failed.set(e) + ready.countDown() + } finally { + ready.countDown() + } + }, + "polaris-server-output", + ) + outputThread.isDaemon = true + outputThread.start() + + if (!ready.await(startupTimeout.toMillis(), TimeUnit.MILLISECONDS)) { + process.destroyForcibly() + throw GradleException( + "Polaris server did not emit a listen URL within $startupTimeout. Captured output:\n${capturedOutput(capturedOutput)}" + ) + } + + failed.get()?.let { throw GradleException("Failed while reading Polaris server output", it) } + val urls = detected.get() + if (urls == null) { + val exit = + if (process.isAlive) "still running" else "exited with code ${process.exitValue()}" + process.destroyForcibly() + throw GradleException( + "Polaris server $exit before emitting a listen URL. Captured output:\n${capturedOutput(capturedOutput)}" + ) + } + + return StartedServer(RunningPolarisServer(process, outputThread, stopTimeout), urls.toPorts()) + } + + private fun capturedOutput(lines: MutableList): String = + synchronized(lines) { lines.takeLast(200).joinToString("\n") }.ifBlank { "" } + } +} + +internal data class StartedServer(val server: RunningPolarisServer, val ports: ServerPorts) + +private data class ListenUrls(val httpUrl: String, val managementUrl: String?) { + fun toPorts(): ServerPorts = + ServerPorts(URI.create(httpUrl).port, managementUrl?.let { URI.create(it).port }) +} + +internal data class ServerPorts(val httpPort: Int, val managementPort: Int?) diff --git a/gradle/server-test-runner/src/test/kotlin/org/apache/polaris/server/test/runner/PolarisServerTestRunnerPluginTest.kt b/gradle/server-test-runner/src/test/kotlin/org/apache/polaris/server/test/runner/PolarisServerTestRunnerPluginTest.kt new file mode 100644 index 00000000000..c9781bbf8bd --- /dev/null +++ b/gradle/server-test-runner/src/test/kotlin/org/apache/polaris/server/test/runner/PolarisServerTestRunnerPluginTest.kt @@ -0,0 +1,243 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.polaris.server.test.runner + +import java.nio.file.Path +import kotlin.io.path.createDirectories +import kotlin.io.path.writeText +import org.assertj.core.api.Assertions.assertThat +import org.gradle.testkit.runner.GradleRunner +import org.gradle.testkit.runner.TaskOutcome +import org.junit.jupiter.api.Test +import org.junit.jupiter.api.io.TempDir + +class PolarisServerTestRunnerPluginTest { + @TempDir lateinit var projectDir: Path + + @Test + fun `starts server for test task and passes discovered ports`() { + writeSettings() + writeBuild( + """ + import org.gradle.api.tasks.compile.JavaCompile + import org.gradle.jvm.tasks.Jar + + plugins { + java + id("polaris-server-test-runner") + } + + repositories { mavenCentral() } + + dependencies { + testImplementation(platform("org.junit:junit-bom:6.1.0")) + testImplementation("org.junit.jupiter:junit-jupiter") + testRuntimeOnly("org.junit.platform:junit-platform-launcher") + } + + val serverRuntime by configurations.creating + + dependencies { + serverRuntime(files(tasks.named("jar"))) + } + + val startupActionSourceDir = layout.buildDirectory.dir("startup-action-src") + val startupActionClassesDir = layout.buildDirectory.dir("startup-action-classes") + + val writeStartupAction by tasks.registering { + val sourceFile = startupActionSourceDir.map { it.file("test/StartupAction.java") } + outputs.file(sourceFile) + doLast { + sourceFile.get().asFile.apply { + parentFile.mkdirs() + writeText( + ${"\"\"\""} + package test; + + import org.apache.polaris.server.test.runner.spi.PolarisServerStartupAction; + import org.apache.polaris.server.test.runner.spi.PolarisServerStartupContext; + + public class StartupAction implements PolarisServerStartupAction { + @Override + public void start(PolarisServerStartupContext context) { + context.getSystemProperties().put( + "startup.value", context.getParameters().get("value")); + } + } + ${"\"\"\""}.trimIndent() + ) + } + } + } + + val compileStartupAction by tasks.registering(JavaCompile::class) { + dependsOn(writeStartupAction) + source(startupActionSourceDir) + destinationDirectory.set(startupActionClassesDir) + classpath = + files( + org.apache.polaris.server.test.runner.spi.PolarisServerStartupAction::class.java + .protectionDomain + .codeSource + .location + ) + } + + val startupActionJar by tasks.registering(Jar::class) { + dependsOn(compileStartupAction) + from(startupActionClassesDir) + archiveBaseName.set("startup-action") + } + + tasks.named("jar") { + manifest { attributes("Main-Class" to "test.FakeServer") } + } + + tasks.test { + useJUnitPlatform() + withPolarisServer(configurations.named("serverRuntime")) { + arguments.add(layout.buildDirectory.file("server-stopped.txt").get().asFile.absolutePath) + arguments.add(layout.buildDirectory.file("server-started.txt").get().asFile.absolutePath) + startupActionClasspath.from(startupActionJar) + startupActionClass.set("test.StartupAction") + startupActionParameters.put("value", "started") + } + } + """ + .trimIndent() + ) + writeFakeServer() + writePropertyTest() + + val result = gradleRunner("test").build() + + assertThat(result.task(":test")?.outcome).isEqualTo(TaskOutcome.SUCCESS) + assertThat(projectDir.resolve("build/server-stopped.txt")).exists() + assertThat(projectDir.resolve("build/server-started.txt")).hasContent("started") + } + + @Test + fun `fails when server artifact is missing`() { + writeSettings() + writeBuild( + """ + plugins { + java + id("polaris-server-test-runner") + } + + repositories { mavenCentral() } + + dependencies { + testImplementation(platform("org.junit:junit-bom:6.1.0")) + testImplementation("org.junit.jupiter:junit-jupiter") + testRuntimeOnly("org.junit.platform:junit-platform-launcher") + } + + tasks.test { + useJUnitPlatform() + withPolarisServer(files()) + } + """ + .trimIndent() + ) + writePropertyTest() + + val result = gradleRunner("test").buildAndFail() + + assertThat(result.output).contains("Expected exactly one Polaris server artifact") + } + + private fun writeSettings() { + projectDir.resolve("settings.gradle.kts").writeText("rootProject.name = \"fixture\"\n") + } + + private fun writeBuild(content: String) { + projectDir.resolve("build.gradle.kts").writeText(content) + } + + private fun writeFakeServer() { + val sourceDir = projectDir.resolve("src/main/java/test").createDirectories() + sourceDir + .resolve("FakeServer.java") + .writeText( + """ + package test; + + import java.nio.file.Files; + import java.nio.file.Path; + import java.util.concurrent.CountDownLatch; + + public final class FakeServer { + public static void main(String[] args) throws Exception { + Path stopped = Path.of(args[0]); + if (args.length > 1) { + String startupValue = System.getProperty("startup.value"); + if (!"started".equals(startupValue)) { + throw new IllegalStateException("Unexpected startup value: " + startupValue); + } + Files.writeString(Path.of(args[1]), startupValue); + } + Runtime.getRuntime().addShutdownHook(new Thread(() -> { + try { + Files.writeString(stopped, "stopped"); + } catch (Exception e) { + throw new RuntimeException(e); + } + })); + System.out.println("Listening on: http://0.0.0.0:12345. Management interface listening on http://0.0.0.0:12346."); + new CountDownLatch(1).await(); + } + } + """ + .trimIndent() + ) + } + + private fun writePropertyTest() { + val testDir = projectDir.resolve("src/test/java/test").createDirectories() + testDir + .resolve("ServerPropertiesTest.java") + .writeText( + """ + package test; + + import static org.junit.jupiter.api.Assertions.assertEquals; + + import org.junit.jupiter.api.Test; + + public class ServerPropertiesTest { + @Test + void receivesServerProperties() { + assertEquals("12345", System.getProperty("quarkus.http.test-port")); + assertEquals("12346", System.getProperty("quarkus.management.test-port")); + } + } + """ + .trimIndent() + ) + } + + private fun gradleRunner(vararg arguments: String): GradleRunner = + GradleRunner.create() + .withProjectDir(projectDir.toFile()) + .withPluginClasspath() + .withArguments(*arguments, "--stacktrace") + .forwardOutput() +} diff --git a/gradlew b/gradlew index 47acbf2081a..5da0b6bbcaa 100755 --- a/gradlew +++ b/gradlew @@ -88,6 +88,7 @@ APP_BASE_NAME=${0##*/} # Discard cd standard output in case $CDPATH is set (https://github.com/gradle/gradle/issues/25036) APP_HOME=$( cd -P "${APP_HOME:-./}" > /dev/null && printf '%s\n' "$PWD" ) || exit +[ -f "${APP_HOME}/.env" ] && . "${APP_HOME}/.env" . "${APP_HOME}/gradle/gradlew-include.sh" # Use the maximum available, or set MAX_FD != -1 to use that value. diff --git a/integration-tests/src/main/java/org/apache/polaris/service/it/ext/ExternalPolarisServerManager.java b/integration-tests/src/main/java/org/apache/polaris/service/it/ext/ExternalPolarisServerManager.java new file mode 100644 index 00000000000..276ef797ce5 --- /dev/null +++ b/integration-tests/src/main/java/org/apache/polaris/service/it/ext/ExternalPolarisServerManager.java @@ -0,0 +1,81 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.polaris.service.it.ext; + +import java.net.URI; +import org.apache.polaris.service.it.env.ClientCredentials; +import org.apache.polaris.service.it.env.ClientPrincipal; +import org.apache.polaris.service.it.env.Server; +import org.junit.jupiter.api.extension.ExtensionContext; + +/** + * {@link PolarisServerManager} implementation for tests whose Polaris server lifecycle is managed + * outside JUnit, for example by Gradle. + */ +public class ExternalPolarisServerManager implements PolarisServerManager { + + public static final String HTTP_PORT_PROPERTY = "quarkus.http.test-port"; + public static final String HOST_PROPERTY = "polaris.test.server.host"; + + @Override + public Server serverForContext(ExtensionContext context) { + URI baseUri = URI.create(String.format("http://%s:%d", host(), httpPort())); + return new Server() { + @Override + public URI baseUri() { + return baseUri; + } + + @Override + public ClientPrincipal adminCredentials() { + return new ClientPrincipal("root", new ClientCredentials("test-admin", "test-secret")); + } + + @Override + public void close() { + // The external process owner is responsible for server lifecycle. + } + }; + } + + private static String host() { + String host = System.getProperty(HOST_PROPERTY, "localhost"); + return host.isBlank() ? "localhost" : host; + } + + private static int httpPort() { + String port = System.getProperty(HTTP_PORT_PROPERTY); + if (port == null || port.isBlank()) { + throw new IllegalStateException( + String.format("System property '%s' must be set", HTTP_PORT_PROPERTY)); + } + try { + int parsed = Integer.parseInt(port); + if (parsed <= 0 || parsed > 65535) { + throw new IllegalArgumentException("Port out of range: " + parsed); + } + return parsed; + } catch (IllegalArgumentException e) { + throw new IllegalStateException( + String.format( + "System property '%s' must be a valid TCP port: %s", HTTP_PORT_PROPERTY, port), + e); + } + } +} diff --git a/integration-tests/src/test/java/org/apache/polaris/service/it/ext/ExternalPolarisServerManagerTest.java b/integration-tests/src/test/java/org/apache/polaris/service/it/ext/ExternalPolarisServerManagerTest.java new file mode 100644 index 00000000000..a6a05bf5b23 --- /dev/null +++ b/integration-tests/src/test/java/org/apache/polaris/service/it/ext/ExternalPolarisServerManagerTest.java @@ -0,0 +1,102 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.polaris.service.it.ext; + +import static org.assertj.core.api.Assertions.assertThat; +import static org.assertj.core.api.Assertions.assertThatThrownBy; + +import org.apache.polaris.service.it.env.Server; +import org.junit.jupiter.api.AfterEach; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; + +class ExternalPolarisServerManagerTest { + + private String originalHost; + private String originalPort; + + @BeforeEach + void captureProperties() { + originalHost = System.getProperty(ExternalPolarisServerManager.HOST_PROPERTY); + originalPort = System.getProperty(ExternalPolarisServerManager.HTTP_PORT_PROPERTY); + } + + @AfterEach + void restoreProperties() { + restore(ExternalPolarisServerManager.HOST_PROPERTY, originalHost); + restore(ExternalPolarisServerManager.HTTP_PORT_PROPERTY, originalPort); + } + + @Test + void serverForContextUsesConfiguredHttpPort() { + System.setProperty(ExternalPolarisServerManager.HTTP_PORT_PROPERTY, "12345"); + + Server server = new ExternalPolarisServerManager().serverForContext(null); + + assertThat(server.baseUri()).hasToString("http://localhost:12345"); + assertThat(server.adminCredentials().principalName()).isEqualTo("root"); + assertThat(server.adminCredentials().credentials().clientId()).isEqualTo("test-admin"); + assertThat(server.adminCredentials().credentials().clientSecret()).isEqualTo("test-secret"); + } + + @Test + void serverForContextUsesConfiguredHost() { + System.setProperty(ExternalPolarisServerManager.HTTP_PORT_PROPERTY, "12345"); + System.setProperty(ExternalPolarisServerManager.HOST_PROPERTY, "127.0.0.1"); + + Server server = new ExternalPolarisServerManager().serverForContext(null); + + assertThat(server.baseUri()).hasToString("http://127.0.0.1:12345"); + } + + @Test + void serverForContextRejectsMissingHttpPort() { + System.clearProperty(ExternalPolarisServerManager.HTTP_PORT_PROPERTY); + + assertThatThrownBy(() -> new ExternalPolarisServerManager().serverForContext(null)) + .isInstanceOf(IllegalStateException.class) + .hasMessageContaining(ExternalPolarisServerManager.HTTP_PORT_PROPERTY); + } + + @Test + void serverForContextRejectsInvalidHttpPort() { + System.setProperty(ExternalPolarisServerManager.HTTP_PORT_PROPERTY, "not-a-port"); + + assertThatThrownBy(() -> new ExternalPolarisServerManager().serverForContext(null)) + .isInstanceOf(IllegalStateException.class) + .hasMessageContaining(ExternalPolarisServerManager.HTTP_PORT_PROPERTY); + } + + @Test + void serverForContextRejectsOutOfRangeHttpPort() { + System.setProperty(ExternalPolarisServerManager.HTTP_PORT_PROPERTY, "65536"); + + assertThatThrownBy(() -> new ExternalPolarisServerManager().serverForContext(null)) + .isInstanceOf(IllegalStateException.class) + .hasMessageContaining(ExternalPolarisServerManager.HTTP_PORT_PROPERTY); + } + + private static void restore(String propertyName, String value) { + if (value == null) { + System.clearProperty(propertyName); + } else { + System.setProperty(propertyName, value); + } + } +} diff --git a/persistence/nosql/persistence/correctness/build.gradle.kts b/persistence/nosql/persistence/correctness/build.gradle.kts index e938e791ad0..d27bdf72493 100644 --- a/persistence/nosql/persistence/correctness/build.gradle.kts +++ b/persistence/nosql/persistence/correctness/build.gradle.kts @@ -66,7 +66,9 @@ testing { implementation(testFixtures(project(":polaris-persistence-nosql-$prj"))) } - targets.all { testTask.configure { systemProperty("polaris.testBackend.name", db) } } + targets.configureEach { + testTask.configure { systemProperty("polaris.testBackend.name", db) } + } } tasks.named("intTest") { dependsOn(dbTaskName) } @@ -81,7 +83,7 @@ testing { // Pass system properties starting with `polaris.` down to the manually executed test(s) so // they can setup the backend via // `o.a.p.persistence.api.BackendConfigurer.defaultBackendConfigurer` using smallrye-config. - targets.all { + targets.configureEach { testTask.configure { providers.systemPropertiesPrefixedBy("polaris").get().forEach { (k, v) -> systemProperty(k, v) diff --git a/plugins/spark/common/src/intTest/java/org/apache/polaris/spark/quarkus/it/SparkCatalogIcebergIT.java b/plugins/spark/common/src/intTest/java/org/apache/polaris/spark/quarkus/it/SparkCatalogIcebergIT.java index 812d8f19d53..36161a7b95a 100644 --- a/plugins/spark/common/src/intTest/java/org/apache/polaris/spark/quarkus/it/SparkCatalogIcebergIT.java +++ b/plugins/spark/common/src/intTest/java/org/apache/polaris/spark/quarkus/it/SparkCatalogIcebergIT.java @@ -18,11 +18,9 @@ */ package org.apache.polaris.spark.quarkus.it; -import io.quarkus.test.junit.QuarkusIntegrationTest; import org.apache.polaris.service.it.ext.SparkSessionBuilder; import org.apache.spark.sql.SparkSession; -@QuarkusIntegrationTest public class SparkCatalogIcebergIT extends SparkCatalogBaseIT { /** Initialize the spark catalog to use the iceberg spark catalog. */ @Override diff --git a/plugins/spark/common/src/intTest/java/org/apache/polaris/spark/quarkus/it/SparkCatalogPolarisIT.java b/plugins/spark/common/src/intTest/java/org/apache/polaris/spark/quarkus/it/SparkCatalogPolarisIT.java index 97a4c222db1..5e823524f1c 100644 --- a/plugins/spark/common/src/intTest/java/org/apache/polaris/spark/quarkus/it/SparkCatalogPolarisIT.java +++ b/plugins/spark/common/src/intTest/java/org/apache/polaris/spark/quarkus/it/SparkCatalogPolarisIT.java @@ -18,7 +18,5 @@ */ package org.apache.polaris.spark.quarkus.it; -import io.quarkus.test.junit.QuarkusIntegrationTest; -@QuarkusIntegrationTest public class SparkCatalogPolarisIT extends SparkCatalogBaseIT {} diff --git a/plugins/spark/common/src/intTest/java/org/apache/polaris/spark/quarkus/it/SparkDeltaIT.java b/plugins/spark/common/src/intTest/java/org/apache/polaris/spark/quarkus/it/SparkDeltaIT.java index 7beacb11415..cef29c36fb6 100644 --- a/plugins/spark/common/src/intTest/java/org/apache/polaris/spark/quarkus/it/SparkDeltaIT.java +++ b/plugins/spark/common/src/intTest/java/org/apache/polaris/spark/quarkus/it/SparkDeltaIT.java @@ -21,7 +21,6 @@ import static org.assertj.core.api.Assertions.assertThat; import static org.assertj.core.api.Assertions.assertThatThrownBy; -import io.quarkus.test.junit.QuarkusIntegrationTest; import java.io.File; import java.nio.file.Path; import java.util.Arrays; @@ -41,7 +40,6 @@ import org.junit.jupiter.api.Test; import org.junit.jupiter.api.io.TempDir; -@QuarkusIntegrationTest public class SparkDeltaIT extends SparkIntegrationBase { private String defaultNs; private String tableRootDir; diff --git a/plugins/spark/common/src/intTest/java/org/apache/polaris/spark/quarkus/it/SparkHudiIT.java b/plugins/spark/common/src/intTest/java/org/apache/polaris/spark/quarkus/it/SparkHudiIT.java index aec85f5d46f..30d5fa71a4a 100644 --- a/plugins/spark/common/src/intTest/java/org/apache/polaris/spark/quarkus/it/SparkHudiIT.java +++ b/plugins/spark/common/src/intTest/java/org/apache/polaris/spark/quarkus/it/SparkHudiIT.java @@ -21,7 +21,6 @@ import static org.assertj.core.api.Assertions.assertThat; import static org.assertj.core.api.Assertions.assertThatThrownBy; -import io.quarkus.test.junit.QuarkusIntegrationTest; import java.io.File; import java.nio.file.Path; import java.util.List; @@ -33,7 +32,6 @@ import org.junit.jupiter.api.Test; import org.junit.jupiter.api.io.TempDir; -@QuarkusIntegrationTest public class SparkHudiIT extends SparkIntegrationBase { @Override diff --git a/plugins/spark/common/src/intTest/java/org/apache/polaris/spark/quarkus/it/SparkIT.java b/plugins/spark/common/src/intTest/java/org/apache/polaris/spark/quarkus/it/SparkIT.java index a4e060a52fc..4850512728a 100644 --- a/plugins/spark/common/src/intTest/java/org/apache/polaris/spark/quarkus/it/SparkIT.java +++ b/plugins/spark/common/src/intTest/java/org/apache/polaris/spark/quarkus/it/SparkIT.java @@ -21,7 +21,6 @@ import static org.assertj.core.api.Assertions.assertThat; import static org.assertj.core.api.Assertions.assertThatThrownBy; -import io.quarkus.test.junit.QuarkusIntegrationTest; import java.io.File; import java.nio.file.Path; import java.util.List; @@ -30,7 +29,6 @@ import org.junit.jupiter.api.Test; import org.junit.jupiter.api.io.TempDir; -@QuarkusIntegrationTest public class SparkIT extends SparkIntegrationBase { @Test public void testNamespaces() { diff --git a/plugins/spark/common/src/intTest/resources/META-INF/services/org.apache.polaris.service.it.ext.PolarisServerManager b/plugins/spark/common/src/intTest/resources/META-INF/services/org.apache.polaris.service.it.ext.PolarisServerManager index b3dd7d7c069..6aa2dac9e0c 100644 --- a/plugins/spark/common/src/intTest/resources/META-INF/services/org.apache.polaris.service.it.ext.PolarisServerManager +++ b/plugins/spark/common/src/intTest/resources/META-INF/services/org.apache.polaris.service.it.ext.PolarisServerManager @@ -17,4 +17,4 @@ # under the License. # -org.apache.polaris.service.it.ServerManager +org.apache.polaris.service.it.ext.ExternalPolarisServerManager diff --git a/plugins/spark/v3.5/integration/build.gradle.kts b/plugins/spark/v3.5/integration/build.gradle.kts deleted file mode 100644 index 476c675fc31..00000000000 --- a/plugins/spark/v3.5/integration/build.gradle.kts +++ /dev/null @@ -1,171 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -plugins { - alias(libs.plugins.quarkus) - id("org.kordamp.gradle.jandex") - id("polaris-runtime") -} - -// get version information -val sparkMajorVersion = "3.5" -val scalaVersion = getAndUseScalaVersionForProject() -val icebergVersion = libs.versions.iceberg.get() -val spark35Version = libs.versions.spark35.get() -val scalaLibraryVersion = - if (scalaVersion == "2.12") { - libs.versions.scala212.get() - } else { - libs.versions.scala213.get() - } - -sourceSets { - named("intTest") { - java { srcDir("../../common/src/intTest/java") } - resources { srcDir("../../common/src/intTest/resources") } - } -} - -dependencies { - // must be enforced to get a consistent and validated set of dependencies - implementation(enforcedPlatform(libs.quarkus.bom)) { - exclude(group = "org.antlr", module = "antlr4-runtime") - exclude(group = "org.scala-lang", module = "scala-library") - exclude(group = "org.scala-lang", module = "scala-reflect") - } - - implementation(project(":polaris-runtime-service")) - - testImplementation( - "org.apache.iceberg:iceberg-spark-runtime-${sparkMajorVersion}_${scalaVersion}:${icebergVersion}" - ) - testImplementation(project(":polaris-spark-${sparkMajorVersion}_${scalaVersion}")) - - testImplementation(project(":polaris-api-management-model")) - - testImplementation(project(":polaris-runtime-test-common")) - - testImplementation("org.apache.spark:spark-sql_${scalaVersion}:${spark35Version}") { - // exclude log4j dependencies. Explicit dependencies for the log4j libraries are - // enforced below to ensure the version compatibility - exclude("org.apache.logging.log4j", "log4j-slf4j2-impl") - exclude("org.apache.logging.log4j", "log4j-1.2-api") - exclude("org.apache.logging.log4j", "log4j-core") - exclude("org.slf4j", "jul-to-slf4j") - } - - // Add spark-hive for Hudi integration - provides HiveExternalCatalog that Hudi needs - testRuntimeOnly("org.apache.spark:spark-hive_${scalaVersion}:${spark35Version}") { - // exclude log4j dependencies to match spark-sql exclusions - exclude("org.apache.logging.log4j", "log4j-slf4j2-impl") - exclude("org.apache.logging.log4j", "log4j-1.2-api") - exclude("org.apache.logging.log4j", "log4j-core") - exclude("org.slf4j", "jul-to-slf4j") - // exclude old slf4j 1.x to log4j 2.x bridge that conflicts with slf4j 2.x bridge - exclude("org.apache.logging.log4j", "log4j-slf4j-impl") - } - // enforce the usage of log4j 2.24.3. This is for the log4j-api compatibility - // of spark-sql dependency - testRuntimeOnly("org.apache.logging.log4j:log4j-core:2.26.0") - - testImplementation("io.delta:delta-spark_${scalaVersion}:3.3.1") - testImplementation("org.apache.hudi:hudi-spark3.5-bundle_${scalaVersion}:1.1.1") { - // exclude log4j dependencies to match spark-sql exclusions - // exclude log4j dependencies to match spark-sql exclusions and prevent version conflicts - exclude("org.apache.logging.log4j", "log4j-slf4j2-impl") - exclude("org.apache.logging.log4j", "log4j-1.2-api") - exclude("org.apache.logging.log4j", "log4j-core") - exclude("org.slf4j", "jul-to-slf4j") - exclude("org.slf4j", "slf4j-log4j12") - exclude("org.slf4j", "slf4j-reload4j") - exclude("ch.qos.reload4j", "reload4j") - exclude("log4j", "log4j") - // exclude old slf4j 1.x to log4j 2.x bridge that conflicts with slf4j 2.x bridge - exclude("org.apache.logging.log4j", "log4j-slf4j-impl") - } - - // The hudi-spark-bundle includes most Hive libraries but excludes hive-exec to keep size - // manageable - // This matches what Spark 3.5 distribution provides (hive-exec-2.3.10-core.jar) - testImplementation("org.apache.hive:hive-exec:2.3.10:core") { - // Exclude conflicting dependencies to use Spark's versions - exclude("org.apache.hadoop", "*") - exclude("org.apache.commons", "*") - exclude("org.slf4j", "*") - exclude("log4j", "*") - exclude("org.apache.logging.log4j", "*") - exclude("org.pentaho", "*") - exclude("org.apache.calcite", "*") - exclude("org.apache.tez", "*") - } - - testImplementation(platform(libs.jackson.bom)) - testImplementation("com.fasterxml.jackson.jakarta.rs:jackson-jakarta-rs-json-provider") - - testImplementation(testFixtures(project(":polaris-runtime-service"))) - - testImplementation(platform(libs.quarkus.bom)) - testImplementation("io.quarkus:quarkus-junit") - testImplementation("io.quarkus:quarkus-rest-client") - testImplementation("io.quarkus:quarkus-rest-client-jackson") - - testImplementation(platform(libs.awssdk.bom)) - testImplementation("software.amazon.awssdk:glue") - testImplementation("software.amazon.awssdk:kms") - testImplementation("software.amazon.awssdk:dynamodb") - - testImplementation(platform(libs.testcontainers.bom)) - testImplementation("org.testcontainers:testcontainers") - testImplementation(libs.s3mock.testcontainers) - - // Required for Spark integration tests - testImplementation(enforcedPlatform("org.scala-lang:scala-library:${scalaLibraryVersion}")) - testImplementation(enforcedPlatform("org.scala-lang:scala-reflect:${scalaLibraryVersion}")) - testImplementation(libs.javax.servlet.api) - testImplementation(libs.antlr4.runtime.spark35) -} - -tasks.named("intTest").configure { - if (System.getenv("AWS_REGION") == null) { - environment("AWS_REGION", "us-west-2") - } - // Note: the test secrets are referenced in - // org.apache.polaris.service.it.ServerManager - environment("POLARIS_BOOTSTRAP_CREDENTIALS", "POLARIS,test-admin,test-secret") - jvmArgs("--add-exports", "java.base/sun.nio.ch=ALL-UNNAMED") - // Need to allow a java security manager after Java 21, for Subject.getSubject to work - // "getSubject is supported only if a security manager is allowed". - systemProperty("java.security.manager", "allow") - // Same issue as above: allow a java security manager after Java 21 - // (this setting is for the application under test, while the setting above is for test code). - systemProperty("quarkus.test.arg-line", "-Djava.security.manager=allow") - val logsDir = project.layout.buildDirectory.get().asFile.resolve("logs") - // delete files from previous runs - doFirst { - // delete log files written by Polaris - logsDir.deleteRecursively() - // delete quarkus.log file (captured Polaris stdout/stderr) - project.layout.buildDirectory.get().asFile.resolve("quarkus.log").delete() - } - // This property is not honored in a per-profile application.properties file, - // so we need to set it here. - systemProperty("quarkus.log.file.path", logsDir.resolve("polaris.log").absolutePath) - // For Spark integration tests - addSparkJvmOptions() -} diff --git a/plugins/spark/v3.5/spark/build.gradle.kts b/plugins/spark/v3.5/spark/build.gradle.kts index e0397dbaf3d..4c8db6aa09b 100644 --- a/plugins/spark/v3.5/spark/build.gradle.kts +++ b/plugins/spark/v3.5/spark/build.gradle.kts @@ -18,8 +18,13 @@ */ import com.github.jengelman.gradle.plugins.shadow.tasks.ShadowJar +import org.gradle.api.attributes.java.TargetJvmVersion +import org.gradle.api.plugins.jvm.JvmTestSuite -plugins { id("polaris-client") } +plugins { + id("polaris-client") + id("polaris-server-test-runner") +} checkstyle { configProperties = @@ -29,11 +34,6 @@ checkstyle { ) } -sourceSets { - main { java { srcDir("../../common/src/main/java") } } - test { java { srcDir("../../common/src/test/java") } } -} - // get version information val sparkMajorVersion = "3.5" val scalaVersion = getAndUseScalaVersionForProject() @@ -46,8 +46,12 @@ val scalaLibraryVersion = } else { libs.versions.scala213.get() } +val errorProneAnnotationsVersion = libs.errorprone.get().versionConstraint.requiredVersion +val intTestJvmVersion = 21 dependencies { + polarisServer(project(path = ":polaris-server", configuration = "quarkusRunner")) + // TODO: extract a polaris-rest module as a thin layer for // client to depends on. implementation(project(":polaris-core")) { isTransitive = false } @@ -89,6 +93,157 @@ dependencies { } } +testing { + suites { + register("intTest") { + dependencies { + implementation( + "org.apache.iceberg:iceberg-spark-runtime-${sparkMajorVersion}_${scalaVersion}:${icebergVersion}" + ) + + implementation(project(":polaris-api-management-model")) + + implementation(project(":polaris-runtime-test-common")) + + implementation("org.apache.spark:spark-sql_${scalaVersion}:${spark35Version}") { + // exclude log4j dependencies. Explicit dependencies for the log4j libraries are + // enforced below to ensure the version compatibility + exclude("org.apache.logging.log4j", "log4j-slf4j2-impl") + exclude("org.apache.logging.log4j", "log4j-1.2-api") + exclude("org.apache.logging.log4j", "log4j-core") + exclude("org.slf4j", "jul-to-slf4j") + } + + // Add spark-hive for Hudi integration - provides HiveExternalCatalog that Hudi needs + runtimeOnly("org.apache.spark:spark-hive_${scalaVersion}:${spark35Version}") { + // exclude log4j dependencies to match spark-sql exclusions + exclude("org.apache.logging.log4j", "log4j-slf4j2-impl") + exclude("org.apache.logging.log4j", "log4j-1.2-api") + exclude("org.apache.logging.log4j", "log4j-core") + exclude("org.slf4j", "jul-to-slf4j") + // exclude old slf4j 1.x to log4j 2.x bridge that conflicts with slf4j 2.x bridge + exclude("org.apache.logging.log4j", "log4j-slf4j-impl") + } + // enforce the usage of log4j 2.24.3. This is for the log4j-api compatibility + // of spark-sql dependency + runtimeOnly("org.apache.logging.log4j:log4j-core:2.26.0") + + implementation("io.delta:delta-spark_${scalaVersion}:3.3.1") + implementation("org.apache.hudi:hudi-spark3.5-bundle_${scalaVersion}:1.1.1") { + // exclude log4j dependencies to match spark-sql exclusions + // exclude log4j dependencies to match spark-sql exclusions and prevent version conflicts + exclude("org.apache.logging.log4j", "log4j-slf4j2-impl") + exclude("org.apache.logging.log4j", "log4j-1.2-api") + exclude("org.apache.logging.log4j", "log4j-core") + exclude("org.slf4j", "jul-to-slf4j") + exclude("org.slf4j", "slf4j-log4j12") + exclude("org.slf4j", "slf4j-reload4j") + exclude("ch.qos.reload4j", "reload4j") + exclude("log4j", "log4j") + // exclude old slf4j 1.x to log4j 2.x bridge that conflicts with slf4j 2.x bridge + exclude("org.apache.logging.log4j", "log4j-slf4j-impl") + } + + // The hudi-spark-bundle includes most Hive libraries but excludes hive-exec to keep size + // manageable + // This matches what Spark 3.5 distribution provides (hive-exec-2.3.10-core.jar) + implementation("org.apache.hive:hive-exec:2.3.10:core") { + // Exclude conflicting dependencies to use Spark's versions + exclude("org.apache.hadoop", "*") + exclude("org.apache.commons", "*") + exclude("org.slf4j", "*") + exclude("log4j", "*") + exclude("org.apache.logging.log4j", "*") + exclude("org.pentaho", "*") + exclude("org.apache.calcite", "*") + exclude("org.apache.tez", "*") + } + + implementation(platform(libs.jackson.bom)) + implementation("com.fasterxml.jackson.jakarta.rs:jackson-jakarta-rs-json-provider") + implementation(libs.jakarta.ws.rs.api) + compileOnly("com.google.errorprone:error_prone_annotations:${errorProneAnnotationsVersion}") + + implementation(testFixtures(project(":polaris-runtime-service"))) + + implementation(platform(libs.awssdk.bom)) + implementation("software.amazon.awssdk:glue") + implementation("software.amazon.awssdk:kms") + implementation("software.amazon.awssdk:dynamodb") + + implementation(platform(libs.testcontainers.bom)) + implementation("org.testcontainers:testcontainers") + implementation(libs.s3mock.testcontainers) + + // Required for Spark integration tests + implementation(enforcedPlatform("org.scala-lang:scala-library:${scalaLibraryVersion}")) + implementation(enforcedPlatform("org.scala-lang:scala-reflect:${scalaLibraryVersion}")) + implementation(libs.javax.servlet.api) + implementation(libs.antlr4.runtime.spark35) + } + + targets.configureEach { + testTask.configure { + environment( + "AWS_REGION", + providers.environmentVariable("AWS_REGION").getOrElse("us-west-2"), + ) + jvmArgs("--add-exports", "java.base/sun.nio.ch=ALL-UNNAMED") + // Need to allow a java security manager after Java 21, for Subject.getSubject to work + // "getSubject is supported only if a security manager is allowed". + systemProperty("java.security.manager", "allow") + val logsDir = project.layout.buildDirectory.get().asFile.resolve("logs") + // delete files from previous runs + doFirst { + // delete log files written by Polaris + logsDir.deleteRecursively() + } + withPolarisServer(configurations.polarisServer) { + environment.put( + "AWS_REGION", + providers.environmentVariable("AWS_REGION").orElse("us-west-2"), + ) + environment.put( + "AWS_ACCESS_KEY_ID", + providers.environmentVariable("AWS_ACCESS_KEY_ID").orElse("ap1"), + ) + environment.put( + "AWS_SECRET_ACCESS_KEY", + providers.environmentVariable("AWS_SECRET_ACCESS_KEY").orElse("s3cr3t"), + ) + environment.put("AWS_EC2_METADATA_DISABLED", "true") + environment.put("POLARIS_BOOTSTRAP_CREDENTIALS", "POLARIS,test-admin,test-secret") + systemProperties.put("quarkus.profile", "it") + systemProperties.put( + "quarkus.log.file.path", + logsDir.resolve("polaris.log").absolutePath, + ) + } + // For Spark integration tests + addSparkJvmOptions() + } + } + } + } +} + +sourceSets { + main { java { srcDir("../../common/src/main/java") } } + test { java { srcDir("../../common/src/test/java") } } + named("intTest") { + java { srcDir("../../common/src/intTest/java") } + resources { srcDir("../../common/src/intTest/resources") } + } +} + +listOf("intTestCompileClasspath", "intTestRuntimeClasspath").forEach { + // :polaris-runtime-test-common and :polaris-runtime-service (testFixtures) require JVM 21 + // compatibility. + configurations.named(it).configure { + attributes.attribute(TargetJvmVersion.TARGET_JVM_VERSION_ATTRIBUTE, intTestJvmVersion) + } +} + tasks.register("createPolarisSparkJar") { archiveClassifier = "bundle" isZip64 = true diff --git a/plugins/spark/v3.5/integration/src/intTest/java/org/apache/polaris/spark/quarkus/it/SparkCatalogBaseIT.java b/plugins/spark/v3.5/spark/src/intTest/java/org/apache/polaris/spark/quarkus/it/SparkCatalogBaseIT.java similarity index 99% rename from plugins/spark/v3.5/integration/src/intTest/java/org/apache/polaris/spark/quarkus/it/SparkCatalogBaseIT.java rename to plugins/spark/v3.5/spark/src/intTest/java/org/apache/polaris/spark/quarkus/it/SparkCatalogBaseIT.java index 23f2f1f5fd1..b64f1e6766e 100644 --- a/plugins/spark/v3.5/integration/src/intTest/java/org/apache/polaris/spark/quarkus/it/SparkCatalogBaseIT.java +++ b/plugins/spark/v3.5/spark/src/intTest/java/org/apache/polaris/spark/quarkus/it/SparkCatalogBaseIT.java @@ -23,7 +23,6 @@ import com.google.common.base.Preconditions; import com.google.common.collect.Maps; -import io.quarkus.test.junit.QuarkusIntegrationTest; import java.util.Arrays; import java.util.Map; import org.apache.iceberg.exceptions.BadRequestException; @@ -49,7 +48,6 @@ * SparkCatalog operations, some operations like listNamespaces under a namespace can not be * triggered through a SQL interface directly with Spark. */ -@QuarkusIntegrationTest public abstract class SparkCatalogBaseIT extends SparkIntegrationBase { private static StructType schema = new StructType().add("id", "long").add("name", "string"); protected StagingTableCatalog tableCatalog = null; diff --git a/plugins/spark/v4.0/integration/build.gradle.kts b/plugins/spark/v4.0/integration/build.gradle.kts deleted file mode 100644 index ccc140fad3f..00000000000 --- a/plugins/spark/v4.0/integration/build.gradle.kts +++ /dev/null @@ -1,179 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -plugins { - alias(libs.plugins.quarkus) - id("org.kordamp.gradle.jandex") - id("polaris-runtime") -} - -// get version information -val sparkMajorVersion = "4.0" -val scalaVersion = getAndUseScalaVersionForProject() -val icebergVersion = libs.versions.iceberg.get() -val spark40Version = libs.versions.spark40.get() -val scalaLibraryVersion = libs.versions.scala213.get() - -sourceSets { - named("intTest") { - java { srcDir("../../common/src/intTest/java") } - resources { srcDir("../../common/src/intTest/resources") } - } -} - -dependencies { - // must be enforced to get a consistent and validated set of dependencies - implementation(enforcedPlatform(libs.quarkus.bom)) { - exclude(group = "org.antlr", module = "antlr4-runtime") - exclude(group = "org.scala-lang", module = "scala-library") - exclude(group = "org.scala-lang", module = "scala-reflect") - } - - // For test configurations, exclude jakarta.servlet-api from Quarkus BOM - // to allow Spark 4.0's version (5.0.0) which includes SingleThreadModel - testImplementation(platform(libs.quarkus.bom)) { - exclude(group = "jakarta.servlet", module = "jakarta.servlet-api") - } - - implementation(project(":polaris-runtime-service")) - - testImplementation( - "org.apache.iceberg:iceberg-spark-runtime-${sparkMajorVersion}_${scalaVersion}:${icebergVersion}" - ) - testImplementation(project(":polaris-spark-${sparkMajorVersion}_${scalaVersion}")) - - testImplementation(project(":polaris-api-management-model")) - - testImplementation(project(":polaris-runtime-test-common")) - - testImplementation("org.apache.spark:spark-sql_${scalaVersion}:${spark40Version}") { - // exclude log4j dependencies. Explicit dependencies for the log4j libraries are - // enforced below to ensure the version compatibility - exclude("org.apache.logging.log4j", "log4j-slf4j2-impl") - exclude("org.apache.logging.log4j", "log4j-1.2-api") - exclude("org.apache.logging.log4j", "log4j-core") - exclude("org.slf4j", "jul-to-slf4j") - } - - // Add spark-hive for Hudi integration - provides HiveExternalCatalog that Hudi needs - testRuntimeOnly("org.apache.spark:spark-hive_${scalaVersion}:${spark40Version}") { - // exclude log4j dependencies to match spark-sql exclusions - exclude("org.apache.logging.log4j", "log4j-slf4j2-impl") - exclude("org.apache.logging.log4j", "log4j-1.2-api") - exclude("org.apache.logging.log4j", "log4j-core") - exclude("org.slf4j", "jul-to-slf4j") - // exclude old slf4j 1.x to log4j 2.x bridge that conflicts with slf4j 2.x bridge - exclude("org.apache.logging.log4j", "log4j-slf4j-impl") - } - // enforce the usage of log4j 2.24.3. This is for the log4j-api compatibility - // of spark-sql dependency - testRuntimeOnly("org.apache.logging.log4j:log4j-core:2.26.0") - - testImplementation("io.delta:delta-spark_${scalaVersion}:4.0.1") - testImplementation("org.apache.hudi:hudi-spark4.0-bundle_${scalaVersion}:1.1.1") { - // exclude log4j dependencies to match spark-sql exclusions - // exclude log4j dependencies to match spark-sql exclusions and prevent version conflicts - exclude("org.apache.logging.log4j", "log4j-slf4j2-impl") - exclude("org.apache.logging.log4j", "log4j-1.2-api") - exclude("org.apache.logging.log4j", "log4j-core") - exclude("org.slf4j", "jul-to-slf4j") - exclude("org.slf4j", "slf4j-log4j12") - exclude("org.slf4j", "slf4j-reload4j") - exclude("ch.qos.reload4j", "reload4j") - exclude("log4j", "log4j") - // exclude old slf4j 1.x to log4j 2.x bridge that conflicts with slf4j 2.x bridge - exclude("org.apache.logging.log4j", "log4j-slf4j-impl") - } - - // The hudi-spark-bundle includes most Hive libraries but excludes hive-exec to keep size - // manageable - // This matches what Spark 4.0 distribution provides (hive-exec-2.3.10-core.jar) - testImplementation("org.apache.hive:hive-exec:2.3.10:core") { - // Exclude conflicting dependencies to use Spark's versions - exclude("org.apache.hadoop", "*") - exclude("org.apache.commons", "*") - exclude("org.slf4j", "*") - exclude("log4j", "*") - exclude("org.apache.logging.log4j", "*") - exclude("org.pentaho", "*") - exclude("org.apache.calcite", "*") - exclude("org.apache.tez", "*") - } - - testImplementation(platform(libs.jackson.bom)) - testImplementation("com.fasterxml.jackson.jakarta.rs:jackson-jakarta-rs-json-provider") - - testImplementation(testFixtures(project(":polaris-runtime-service"))) - - testImplementation(platform(libs.quarkus.bom)) - testImplementation("io.quarkus:quarkus-junit") - testImplementation("io.quarkus:quarkus-rest-client") - testImplementation("io.quarkus:quarkus-rest-client-jackson") - - testImplementation(platform(libs.awssdk.bom)) - testImplementation("software.amazon.awssdk:glue") - testImplementation("software.amazon.awssdk:kms") - testImplementation("software.amazon.awssdk:dynamodb") - - testImplementation(platform(libs.testcontainers.bom)) - testImplementation("org.testcontainers:testcontainers") - testImplementation(libs.s3mock.testcontainers) - - // Required for Spark integration tests - testImplementation(enforcedPlatform("org.scala-lang:scala-library:${scalaLibraryVersion}")) - testImplementation(enforcedPlatform("org.scala-lang:scala-reflect:${scalaLibraryVersion}")) - testImplementation(libs.javax.servlet.api) - testImplementation(libs.antlr4.runtime.spark40) -} - -// Force jakarta.servlet-api to 5.0.0 for Spark 4.0 compatibility -// Spark 4.0 requires version 5.0.0 which includes SingleThreadModel -// Quarkus BOM forces it to 6.x which removed SingleThreadModel -configurations.named("intTestRuntimeClasspath") { - resolutionStrategy { force("jakarta.servlet:jakarta.servlet-api:5.0.0") } -} - -tasks.named("intTest").configure { - if (System.getenv("AWS_REGION") == null) { - environment("AWS_REGION", "us-west-2") - } - // Note: the test secrets are referenced in - // org.apache.polaris.service.it.ServerManager - environment("POLARIS_BOOTSTRAP_CREDENTIALS", "POLARIS,test-admin,test-secret") - jvmArgs("--add-exports", "java.base/sun.nio.ch=ALL-UNNAMED") - // Need to allow a java security manager after Java 21, for Subject.getSubject to work - // "getSubject is supported only if a security manager is allowed". - systemProperty("java.security.manager", "allow") - // Same issue as above: allow a java security manager after Java 21 - // (this setting is for the application under test, while the setting above is for test code). - systemProperty("quarkus.test.arg-line", "-Djava.security.manager=allow") - val logsDir = project.layout.buildDirectory.get().asFile.resolve("logs") - // delete files from previous runs - doFirst { - // delete log files written by Polaris - logsDir.deleteRecursively() - // delete quarkus.log file (captured Polaris stdout/stderr) - project.layout.buildDirectory.get().asFile.resolve("quarkus.log").delete() - } - // This property is not honored in a per-profile application.properties file, - // so we need to set it here. - systemProperty("quarkus.log.file.path", logsDir.resolve("polaris.log").absolutePath) - // For Spark integration tests - addSparkJvmOptions() -} diff --git a/plugins/spark/v4.0/spark/build.gradle.kts b/plugins/spark/v4.0/spark/build.gradle.kts index d2437cd1809..6e998e88bc4 100644 --- a/plugins/spark/v4.0/spark/build.gradle.kts +++ b/plugins/spark/v4.0/spark/build.gradle.kts @@ -18,8 +18,13 @@ */ import com.github.jengelman.gradle.plugins.shadow.tasks.ShadowJar +import org.gradle.api.attributes.java.TargetJvmVersion +import org.gradle.api.plugins.jvm.JvmTestSuite -plugins { id("polaris-client") } +plugins { + id("polaris-client") + id("polaris-server-test-runner") +} checkstyle { configProperties = @@ -29,11 +34,6 @@ checkstyle { ) } -sourceSets { - main { java { srcDir("../../common/src/main/java") } } - test { java { srcDir("../../common/src/test/java") } } -} - // get version information val sparkMajorVersion = "4.0" val scalaVersion = getAndUseScalaVersionForProject() @@ -41,8 +41,12 @@ val icebergVersion = libs.versions.iceberg.get() val spark40Version = libs.versions.spark40.get() val scalaLibraryVersion = libs.versions.scala213.get() +val errorProneAnnotationsVersion = libs.errorprone.get().versionConstraint.requiredVersion +val intTestJvmVersion = 21 dependencies { + polarisServer(project(path = ":polaris-server", configuration = "quarkusRunner")) + // TODO: extract a polaris-rest module as a thin layer for // client to depends on. implementation(project(":polaris-core")) { isTransitive = false } @@ -85,6 +89,164 @@ dependencies { testImplementation("org.apache.logging.log4j:log4j-api:2.26.0") } +testing { + suites { + register("intTest") { + dependencies { + implementation( + "org.apache.iceberg:iceberg-spark-runtime-${sparkMajorVersion}_${scalaVersion}:${icebergVersion}" + ) + + implementation(project(":polaris-api-management-model")) + + implementation(project(":polaris-runtime-test-common")) + + implementation("org.apache.spark:spark-sql_${scalaVersion}:${spark40Version}") { + // exclude log4j dependencies. Explicit dependencies for the log4j libraries are + // enforced below to ensure the version compatibility + exclude("org.apache.logging.log4j", "log4j-slf4j2-impl") + exclude("org.apache.logging.log4j", "log4j-1.2-api") + exclude("org.apache.logging.log4j", "log4j-core") + exclude("org.slf4j", "jul-to-slf4j") + } + + // Add spark-hive for Hudi integration - provides HiveExternalCatalog that Hudi needs + runtimeOnly("org.apache.spark:spark-hive_${scalaVersion}:${spark40Version}") { + // exclude log4j dependencies to match spark-sql exclusions + exclude("org.apache.logging.log4j", "log4j-slf4j2-impl") + exclude("org.apache.logging.log4j", "log4j-1.2-api") + exclude("org.apache.logging.log4j", "log4j-core") + exclude("org.slf4j", "jul-to-slf4j") + // exclude old slf4j 1.x to log4j 2.x bridge that conflicts with slf4j 2.x bridge + exclude("org.apache.logging.log4j", "log4j-slf4j-impl") + } + // enforce the usage of log4j 2.24.3. This is for the log4j-api compatibility + // of spark-sql dependency + runtimeOnly("org.apache.logging.log4j:log4j-core:2.26.0") + + implementation("io.delta:delta-spark_${scalaVersion}:4.0.1") + implementation("org.apache.hudi:hudi-spark4.0-bundle_${scalaVersion}:1.1.1") { + // exclude log4j dependencies to match spark-sql exclusions + // exclude log4j dependencies to match spark-sql exclusions and prevent version conflicts + exclude("org.apache.logging.log4j", "log4j-slf4j2-impl") + exclude("org.apache.logging.log4j", "log4j-1.2-api") + exclude("org.apache.logging.log4j", "log4j-core") + exclude("org.slf4j", "jul-to-slf4j") + exclude("org.slf4j", "slf4j-log4j12") + exclude("org.slf4j", "slf4j-reload4j") + exclude("ch.qos.reload4j", "reload4j") + exclude("log4j", "log4j") + // exclude old slf4j 1.x to log4j 2.x bridge that conflicts with slf4j 2.x bridge + exclude("org.apache.logging.log4j", "log4j-slf4j-impl") + } + + // The hudi-spark-bundle includes most Hive libraries but excludes hive-exec to keep size + // manageable + // This matches what Spark 4.0 distribution provides (hive-exec-2.3.10-core.jar) + implementation("org.apache.hive:hive-exec:2.3.10:core") { + // Exclude conflicting dependencies to use Spark's versions + exclude("org.apache.hadoop", "*") + exclude("org.apache.commons", "*") + exclude("org.slf4j", "*") + exclude("log4j", "*") + exclude("org.apache.logging.log4j", "*") + exclude("org.pentaho", "*") + exclude("org.apache.calcite", "*") + exclude("org.apache.tez", "*") + } + + implementation(platform(libs.jackson.bom)) + implementation("com.fasterxml.jackson.jakarta.rs:jackson-jakarta-rs-json-provider") + implementation(libs.jakarta.ws.rs.api) + compileOnly("com.google.errorprone:error_prone_annotations:${errorProneAnnotationsVersion}") + + implementation(testFixtures(project(":polaris-runtime-service"))) + + implementation(platform(libs.awssdk.bom)) + implementation("software.amazon.awssdk:glue") + implementation("software.amazon.awssdk:kms") + implementation("software.amazon.awssdk:dynamodb") + + implementation(platform(libs.testcontainers.bom)) + implementation("org.testcontainers:testcontainers") + implementation(libs.s3mock.testcontainers) + + // Required for Spark integration tests + implementation(enforcedPlatform("org.scala-lang:scala-library:${scalaLibraryVersion}")) + implementation(enforcedPlatform("org.scala-lang:scala-reflect:${scalaLibraryVersion}")) + implementation(libs.javax.servlet.api) + implementation(libs.antlr4.runtime.spark40) + } + + targets.configureEach { + testTask.configure { + environment( + "AWS_REGION", + providers.environmentVariable("AWS_REGION").getOrElse("us-west-2"), + ) + jvmArgs("--add-exports", "java.base/sun.nio.ch=ALL-UNNAMED") + // Need to allow a java security manager after Java 21, for Subject.getSubject to work + // "getSubject is supported only if a security manager is allowed". + systemProperty("java.security.manager", "allow") + val logsDir = project.layout.buildDirectory.get().asFile.resolve("logs") + // delete files from previous runs + doFirst { + // delete log files written by Polaris + logsDir.deleteRecursively() + } + withPolarisServer(configurations.polarisServer) { + environment.put( + "AWS_REGION", + providers.environmentVariable("AWS_REGION").orElse("us-west-2"), + ) + environment.put( + "AWS_ACCESS_KEY_ID", + providers.environmentVariable("AWS_ACCESS_KEY_ID").orElse("ap1"), + ) + environment.put( + "AWS_SECRET_ACCESS_KEY", + providers.environmentVariable("AWS_SECRET_ACCESS_KEY").orElse("s3cr3t"), + ) + environment.put("AWS_EC2_METADATA_DISABLED", "true") + environment.put("POLARIS_BOOTSTRAP_CREDENTIALS", "POLARIS,test-admin,test-secret") + systemProperties.put("quarkus.profile", "it") + systemProperties.put( + "quarkus.log.file.path", + logsDir.resolve("polaris.log").absolutePath, + ) + } + // For Spark integration tests + addSparkJvmOptions() + } + } + } + } +} + +sourceSets { + main { java { srcDir("../../common/src/main/java") } } + test { java { srcDir("../../common/src/test/java") } } + named("intTest") { + java { srcDir("../../common/src/intTest/java") } + resources { srcDir("../../common/src/intTest/resources") } + } +} + +listOf("intTestCompileClasspath", "intTestRuntimeClasspath").forEach { + // :polaris-runtime-test-common and :polaris-runtime-service (testFixtures) require JVM 21 + // compatibility. + configurations.named(it).configure { + attributes.attribute(TargetJvmVersion.TARGET_JVM_VERSION_ATTRIBUTE, intTestJvmVersion) + } +} + +// Force jakarta.servlet-api to 5.0.0 for Spark 4.0 compatibility +// Spark 4.0 requires version 5.0.0 which includes SingleThreadModel +// Quarkus BOM forces it to 6.x which removed SingleThreadModel +configurations.named("intTestRuntimeClasspath") { + resolutionStrategy { force("jakarta.servlet:jakarta.servlet-api:5.0.0") } +} + tasks.register("createPolarisSparkJar") { archiveClassifier = "bundle" isZip64 = true diff --git a/plugins/spark/v4.0/integration/src/intTest/java/org/apache/polaris/spark/quarkus/it/SparkCatalogBaseIT.java b/plugins/spark/v4.0/spark/src/intTest/java/org/apache/polaris/spark/quarkus/it/SparkCatalogBaseIT.java similarity index 99% rename from plugins/spark/v4.0/integration/src/intTest/java/org/apache/polaris/spark/quarkus/it/SparkCatalogBaseIT.java rename to plugins/spark/v4.0/spark/src/intTest/java/org/apache/polaris/spark/quarkus/it/SparkCatalogBaseIT.java index 76b99d1c144..d328484cc1a 100644 --- a/plugins/spark/v4.0/integration/src/intTest/java/org/apache/polaris/spark/quarkus/it/SparkCatalogBaseIT.java +++ b/plugins/spark/v4.0/spark/src/intTest/java/org/apache/polaris/spark/quarkus/it/SparkCatalogBaseIT.java @@ -23,7 +23,6 @@ import com.google.common.base.Preconditions; import com.google.common.collect.Maps; -import io.quarkus.test.junit.QuarkusIntegrationTest; import java.util.Arrays; import java.util.Map; import org.apache.iceberg.exceptions.BadRequestException; @@ -50,7 +49,6 @@ * SparkCatalog operations, some operations like listNamespaces under a namespace can not be * triggered through a SQL interface directly with Spark. */ -@QuarkusIntegrationTest public abstract class SparkCatalogBaseIT extends SparkIntegrationBase { private static StructType schema = new StructType().add("id", "long").add("name", "string"); protected StagingTableCatalog tableCatalog = null; diff --git a/runtime/admin/build.gradle.kts b/runtime/admin/build.gradle.kts index e6bdeeb5bb3..5dae356cd81 100644 --- a/runtime/admin/build.gradle.kts +++ b/runtime/admin/build.gradle.kts @@ -58,6 +58,7 @@ dependencies { testImplementation(project(":polaris-runtime-test-common")) testFixturesApi(project(":polaris-core")) + testFixturesImplementation(project(":polaris-runtime-test-common")) testFixturesApi(enforcedPlatform(libs.quarkus.bom)) testFixturesApi("io.quarkus:quarkus-junit") @@ -106,3 +107,10 @@ artifacts { add("distributionElements", layout.buildDirectory.dir("quarkus-app")) { builtBy("quarkusBuild") } add("licenseNoticeElements", layout.projectDirectory.dir("distribution")) } + +tasks.withType().configureEach { + forkEvery = 0 + + // enlarge the max heap size to avoid out of memory error + maxHeapSize = "4g" +} diff --git a/runtime/admin/src/test/java/org/apache/polaris/admintool/BootstrapCommandTest.java b/runtime/admin/src/test/java/org/apache/polaris/admintool/BootstrapCommandTest.java new file mode 100644 index 00000000000..180ed1bb3ed --- /dev/null +++ b/runtime/admin/src/test/java/org/apache/polaris/admintool/BootstrapCommandTest.java @@ -0,0 +1,95 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.polaris.admintool; + +import static org.assertj.core.api.Assertions.assertThat; + +import java.io.PrintWriter; +import java.io.StringWriter; +import org.junit.jupiter.api.Test; +import picocli.CommandLine; + +class BootstrapCommandTest { + + @Test + void testBootstrapInvalidCredentials() { + CommandLine commandLine = new CommandLine(new BootstrapCommand()); + StringWriter err = new StringWriter(); + commandLine.setErr(new PrintWriter(err)); + + int exitCode = commandLine.execute("-r", "realm1", "-c", "invalid syntax"); + + assertThat(exitCode).isEqualTo(BaseCommand.EXIT_CODE_BOOTSTRAP_ERROR); + assertThat(err.toString()) + .contains("Invalid credentials format: invalid syntax") + .contains("Bootstrap encountered errors during operation."); + } + + @Test + void testBootstrapInvalidArguments() { + CommandLine commandLine = new CommandLine(new BootstrapCommand()); + StringWriter err = new StringWriter(); + commandLine.setErr(new PrintWriter(err)); + + int exitCode = commandLine.execute("-r", "realm1", "-f", "/irrelevant"); + + assertThat(exitCode).isEqualTo(BaseCommand.EXIT_CODE_USAGE); + assertThat(err.toString()) + .contains( + "Error: [-r= [-r=]... [-c=]... [-p]] and [[-f=]] are mutually exclusive (specify only one)"); + } + + @Test + void testBootstrapFromInvalidFile() { + CommandLine commandLine = new CommandLine(new BootstrapCommand()); + StringWriter err = new StringWriter(); + commandLine.setErr(new PrintWriter(err)); + + int exitCode = commandLine.execute("-f", "/non/existing/file"); + + assertThat(exitCode).isEqualTo(BaseCommand.EXIT_CODE_BOOTSTRAP_ERROR); + assertThat(err.toString()) + .contains("Failed to read credentials from file:///non/existing/file") + .contains("Bootstrap encountered errors during operation."); + } + + @Test + void testNoPrintCredentialsSystemGenerated() { + CommandLine commandLine = new CommandLine(new BootstrapCommand()); + StringWriter err = new StringWriter(); + commandLine.setErr(new PrintWriter(err)); + + int exitCode = commandLine.execute("-r", "realm1"); + + assertThat(exitCode).isEqualTo(BaseCommand.EXIT_CODE_BOOTSTRAP_ERROR); + assertThat(err.toString()).contains("--credentials").contains("--print-credentials"); + } + + @Test + void testBootstrapInvalidArg() { + CommandLine commandLine = new CommandLine(new BootstrapCommand()); + StringWriter err = new StringWriter(); + commandLine.setErr(new PrintWriter(err)); + + int exitCode = commandLine.execute("-r", "realm1", "--not-real-arg"); + + assertThat(exitCode).isEqualTo(BaseCommand.EXIT_CODE_USAGE); + assertThat(err.toString()).contains("Unknown option: '--not-real-arg'").contains("Usage:"); + } +} diff --git a/runtime/admin/src/test/java/org/apache/polaris/admintool/BootstrapCommandTestBase.java b/runtime/admin/src/test/java/org/apache/polaris/admintool/BootstrapCommandTestBase.java index 9286c53b727..45a401c4f6c 100644 --- a/runtime/admin/src/test/java/org/apache/polaris/admintool/BootstrapCommandTestBase.java +++ b/runtime/admin/src/test/java/org/apache/polaris/admintool/BootstrapCommandTestBase.java @@ -18,8 +18,6 @@ */ package org.apache.polaris.admintool; -import static org.apache.polaris.admintool.BaseCommand.EXIT_CODE_BOOTSTRAP_ERROR; -import static org.apache.polaris.admintool.BaseCommand.EXIT_CODE_USAGE; import static org.assertj.core.api.Assertions.assertThat; import io.quarkus.test.junit.main.Launch; @@ -68,32 +66,6 @@ public void testBootstrapFromCommandLineArguments(LaunchResult result) { .contains("Bootstrap completed successfully."); } - @Test - @Launch( - value = { - "bootstrap", - "-r", - "realm1", - "-c", - "invalid syntax", - }, - exitCode = EXIT_CODE_BOOTSTRAP_ERROR) - public void testBootstrapInvalidCredentials(LaunchResult result) { - assertThat(result.getErrorOutput()) - .contains("Invalid credentials format: invalid syntax") - .contains("Bootstrap encountered errors during operation."); - } - - @Test - @Launch( - value = {"bootstrap", "-r", "realm1", "-f", "/irrelevant"}, - exitCode = EXIT_CODE_USAGE) - public void testBootstrapInvalidArguments(LaunchResult result) { - assertThat(result.getErrorOutput()) - .contains( - "Error: [-r= [-r=]... [-c=]... [-p]] and [[-f=]] are mutually exclusive (specify only one)"); - } - @Test public void testBootstrapFromValidJsonFile(QuarkusMainLauncher launcher) { LaunchResult result = launcher.launch("bootstrap", "-f", json.toString()); @@ -114,15 +86,6 @@ public void testBootstrapFromValidYamlFile(QuarkusMainLauncher launcher) { .contains("Bootstrap completed successfully."); } - @Test - public void testBootstrapFromInvalidFile(QuarkusMainLauncher launcher) { - LaunchResult result = launcher.launch("bootstrap", "-f", "/non/existing/file"); - assertThat(result.exitCode()).isEqualTo(EXIT_CODE_BOOTSTRAP_ERROR); - assertThat(result.getErrorOutput()) - .contains("Failed to read credentials from file:///non/existing/file") - .contains("Bootstrap encountered errors during operation."); - } - @Test @Launch( value = {"bootstrap", "-r", "realm1", "-c", "realm1,client1d,s3cr3t", "--print-credentials"}) @@ -138,30 +101,6 @@ public void testPrintCredentialsSystemGenerated(LaunchResult result) { assertThat(result.getOutput()).contains("realm: realm1 root principal credentials: "); } - @Test - @Launch( - value = {"bootstrap", "-r", "realm1"}, - exitCode = EXIT_CODE_BOOTSTRAP_ERROR) - public void testNoPrintCredentialsSystemGenerated(LaunchResult result) { - assertThat(result.getErrorOutput()).contains("--credentials"); - assertThat(result.getErrorOutput()).contains("--print-credentials"); - } - - @Test - @Launch( - value = { - "bootstrap", - "-r", - "realm1", - "--not-real-arg", - }, - exitCode = EXIT_CODE_USAGE) - public void testBootstrapInvalidArg(LaunchResult result) { - assertThat(result.getErrorOutput()) - .contains("Unknown option: '--not-real-arg'") - .contains("Usage:"); - } - private static Path copyResource(Path temp, String resource) throws IOException { URL source = Objects.requireNonNull(BootstrapCommandTestBase.class.getResource(resource)); Path dest = temp.resolve(resource); diff --git a/runtime/admin/src/test/java/org/apache/polaris/admintool/PurgeCommandTest.java b/runtime/admin/src/test/java/org/apache/polaris/admintool/PurgeCommandTest.java index e2cf48972df..a9cd643b6fb 100644 --- a/runtime/admin/src/test/java/org/apache/polaris/admintool/PurgeCommandTest.java +++ b/runtime/admin/src/test/java/org/apache/polaris/admintool/PurgeCommandTest.java @@ -38,6 +38,33 @@ class PurgeCommandTest { + @Test + void testPurgeFailure() { + String realm = "missing-realm"; + PurgeCommand command = new PurgeCommand(); + command.metaStoreManagerFactory = + new FakeMetaStoreManagerFactory( + Map.of( + realm, + new BaseResult(BaseResult.ReturnStatus.ENTITY_NOT_FOUND, "Not bootstrapped"))); + + CommandLine commandLine = new CommandLine(command); + StringWriter out = new StringWriter(); + StringWriter err = new StringWriter(); + commandLine.setOut(new PrintWriter(out)); + commandLine.setErr(new PrintWriter(err)); + + int exitCode = commandLine.execute("-r", realm); + + assertThat(exitCode).isEqualTo(BaseCommand.EXIT_CODE_PURGE_ERROR); + assertThat(out.toString()) + .contains( + "Realm " + + realm + + " is not bootstrapped, could not load root principal. Please run Bootstrap command."); + assertThat(err.toString()).contains("Purge encountered errors during operation."); + } + /** * When {@code purgeRealms} fails (e.g. database connection failure, permission denied), the * command must surface the underlying exception to stderr so operators can diagnose it, in @@ -48,7 +75,7 @@ void testPurgeFailurePrintsStackTrace() { String failureMessage = "simulated database connection failure"; PurgeCommand command = new PurgeCommand(); command.metaStoreManagerFactory = - new ThrowingMetaStoreManagerFactory(new RuntimeException(failureMessage)); + new FakeMetaStoreManagerFactory(new RuntimeException(failureMessage)); CommandLine commandLine = new CommandLine(command); StringWriter err = new StringWriter(); @@ -63,17 +90,27 @@ void testPurgeFailurePrintsStackTrace() { .contains("Purge encountered errors during operation."); } - /** Minimal {@link MetaStoreManagerFactory} whose {@code purgeRealms} always throws. */ - private static final class ThrowingMetaStoreManagerFactory implements MetaStoreManagerFactory { + /** Minimal {@link MetaStoreManagerFactory} for testing purge command results and failures. */ + private static final class FakeMetaStoreManagerFactory implements MetaStoreManagerFactory { + private final Map results; private final RuntimeException failure; - private ThrowingMetaStoreManagerFactory(RuntimeException failure) { + private FakeMetaStoreManagerFactory(Map results) { + this.results = results; + this.failure = null; + } + + private FakeMetaStoreManagerFactory(RuntimeException failure) { + this.results = null; this.failure = failure; } @Override public Map purgeRealms(Iterable realms) { - throw failure; + if (failure != null) { + throw failure; + } + return results; } @Override diff --git a/runtime/admin/src/test/java/org/apache/polaris/admintool/PurgeCommandTestBase.java b/runtime/admin/src/test/java/org/apache/polaris/admintool/PurgeCommandTestBase.java index 758f41d68db..2d84f5b02b2 100644 --- a/runtime/admin/src/test/java/org/apache/polaris/admintool/PurgeCommandTestBase.java +++ b/runtime/admin/src/test/java/org/apache/polaris/admintool/PurgeCommandTestBase.java @@ -28,50 +28,20 @@ import java.util.List; import org.apache.polaris.core.persistence.MetaStoreManagerFactory; import org.apache.polaris.core.persistence.bootstrap.RootCredentialsSet; -import org.assertj.core.api.SoftAssertions; -import org.eclipse.microprofile.config.inject.ConfigProperty; -import org.junit.jupiter.api.AfterEach; -import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; @QuarkusMainTest public abstract class PurgeCommandTestBase { - protected SoftAssertions soft; + private static final String REALM1 = "purge-test-realm1"; + private static final String REALM2 = "purge-test-realm2"; - @BeforeEach - void setup() { - soft = new SoftAssertions(); - } - - @AfterEach - void after() { - soft.assertAll(); - } - - void preBootstrap( - @Observes StartupEvent event, - @ConfigProperty(name = "pre-bootstrap", defaultValue = "false") boolean preBootstrap, - MetaStoreManagerFactory metaStoreManagerFactory) { - if (preBootstrap) { - metaStoreManagerFactory.bootstrapRealms( - List.of("realm1", "realm2"), RootCredentialsSet.EMPTY); - } + void preBootstrap(@Observes StartupEvent event, MetaStoreManagerFactory metaStoreManagerFactory) { + metaStoreManagerFactory.bootstrapRealms(List.of(REALM1, REALM2), RootCredentialsSet.EMPTY); } @Test - @Launch(value = {"purge", "-r", "realm1", "-r", "realm2"}) + @Launch(value = {"purge", "-r", REALM1, "-r", REALM2}) public void testPurge(LaunchResult result) { assertThat(result.getOutput()).contains("Purge completed successfully."); } - - @Test - @Launch( - value = {"purge", "-r", "realm3"}, - exitCode = BaseCommand.EXIT_CODE_PURGE_ERROR) - public void testPurgeFailure(LaunchResult result) { - soft.assertThat(result.getOutput()) - .contains( - "Realm realm3 is not bootstrapped, could not load root principal. Please run Bootstrap command."); - soft.assertThat(result.getErrorOutput()).contains("Purge encountered errors during operation."); - } } diff --git a/runtime/admin/src/test/java/org/apache/polaris/admintool/nosql/NoSqlInMemoryBootstrapCommandTest.java b/runtime/admin/src/test/java/org/apache/polaris/admintool/nosql/NoSqlInMemoryBootstrapCommandTest.java index 5f9d6f6c1f3..50a8649ee5a 100644 --- a/runtime/admin/src/test/java/org/apache/polaris/admintool/nosql/NoSqlInMemoryBootstrapCommandTest.java +++ b/runtime/admin/src/test/java/org/apache/polaris/admintool/nosql/NoSqlInMemoryBootstrapCommandTest.java @@ -19,7 +19,8 @@ package org.apache.polaris.admintool.nosql; import io.quarkus.test.junit.TestProfile; +import org.apache.polaris.admintool.AdminProfiles; import org.apache.polaris.admintool.BootstrapCommandTestBase; -@TestProfile(NoSqlInMemoryProfile.class) +@TestProfile(AdminProfiles.NoSqlInMemory.class) class NoSqlInMemoryBootstrapCommandTest extends BootstrapCommandTestBase {} diff --git a/runtime/admin/src/test/java/org/apache/polaris/admintool/nosql/NoSqlInMemoryPurgeCommandTest.java b/runtime/admin/src/test/java/org/apache/polaris/admintool/nosql/NoSqlInMemoryPurgeCommandTest.java index ecfacf5c319..e338477066b 100644 --- a/runtime/admin/src/test/java/org/apache/polaris/admintool/nosql/NoSqlInMemoryPurgeCommandTest.java +++ b/runtime/admin/src/test/java/org/apache/polaris/admintool/nosql/NoSqlInMemoryPurgeCommandTest.java @@ -18,21 +18,9 @@ */ package org.apache.polaris.admintool.nosql; -import com.google.common.collect.ImmutableMap; import io.quarkus.test.junit.TestProfile; -import java.util.Map; +import org.apache.polaris.admintool.AdminProfiles; import org.apache.polaris.admintool.PurgeCommandTestBase; -@TestProfile(NoSqlInMemoryPurgeCommandTest.Profile.class) -class NoSqlInMemoryPurgeCommandTest extends PurgeCommandTestBase { - - public static class Profile extends NoSqlInMemoryProfile { - @Override - public Map getConfigOverrides() { - return ImmutableMap.builder() - .putAll(super.getConfigOverrides()) - .put("pre-bootstrap", "true") - .build(); - } - } -} +@TestProfile(AdminProfiles.NoSqlInMemory.class) +class NoSqlInMemoryPurgeCommandTest extends PurgeCommandTestBase {} diff --git a/runtime/admin/src/test/java/org/apache/polaris/admintool/nosql/NoSqlMongoBootstrapCommandTest.java b/runtime/admin/src/test/java/org/apache/polaris/admintool/nosql/NoSqlMongoBootstrapCommandTest.java index 22aa507b3d7..14b335f3814 100644 --- a/runtime/admin/src/test/java/org/apache/polaris/admintool/nosql/NoSqlMongoBootstrapCommandTest.java +++ b/runtime/admin/src/test/java/org/apache/polaris/admintool/nosql/NoSqlMongoBootstrapCommandTest.java @@ -19,7 +19,8 @@ package org.apache.polaris.admintool.nosql; import io.quarkus.test.junit.TestProfile; +import org.apache.polaris.admintool.AdminProfiles; import org.apache.polaris.admintool.BootstrapCommandTestBase; -@TestProfile(NoSqlMongoProfile.class) +@TestProfile(AdminProfiles.NoSqlMongo.class) class NoSqlMongoBootstrapCommandTest extends BootstrapCommandTestBase {} diff --git a/runtime/admin/src/test/java/org/apache/polaris/admintool/nosql/NoSqlMongoMaintenanceCommandTest.java b/runtime/admin/src/test/java/org/apache/polaris/admintool/nosql/NoSqlMongoMaintenanceCommandTest.java index 595160853de..beacbe5e915 100644 --- a/runtime/admin/src/test/java/org/apache/polaris/admintool/nosql/NoSqlMongoMaintenanceCommandTest.java +++ b/runtime/admin/src/test/java/org/apache/polaris/admintool/nosql/NoSqlMongoMaintenanceCommandTest.java @@ -24,9 +24,10 @@ import io.quarkus.test.junit.main.Launch; import io.quarkus.test.junit.main.LaunchResult; import io.quarkus.test.junit.main.QuarkusMainTest; +import org.apache.polaris.admintool.AdminProfiles; import org.junit.jupiter.api.Test; -@TestProfile(NoSqlMongoProfile.class) +@TestProfile(AdminProfiles.NoSqlMongo.class) @QuarkusMainTest class NoSqlMongoMaintenanceCommandTest { @Test diff --git a/runtime/admin/src/test/java/org/apache/polaris/admintool/nosql/NoSqlMongoPurgeCommandTest.java b/runtime/admin/src/test/java/org/apache/polaris/admintool/nosql/NoSqlMongoPurgeCommandTest.java index 81deb680cc1..124849b9a5d 100644 --- a/runtime/admin/src/test/java/org/apache/polaris/admintool/nosql/NoSqlMongoPurgeCommandTest.java +++ b/runtime/admin/src/test/java/org/apache/polaris/admintool/nosql/NoSqlMongoPurgeCommandTest.java @@ -18,21 +18,9 @@ */ package org.apache.polaris.admintool.nosql; -import com.google.common.collect.ImmutableMap; import io.quarkus.test.junit.TestProfile; -import java.util.Map; +import org.apache.polaris.admintool.AdminProfiles; import org.apache.polaris.admintool.PurgeCommandTestBase; -@TestProfile(NoSqlMongoPurgeCommandTest.Profile.class) -class NoSqlMongoPurgeCommandTest extends PurgeCommandTestBase { - - public static class Profile extends NoSqlMongoProfile { - @Override - public Map getConfigOverrides() { - return ImmutableMap.builder() - .putAll(super.getConfigOverrides()) - .put("pre-bootstrap", "true") - .build(); - } - } -} +@TestProfile(AdminProfiles.NoSqlMongo.class) +class NoSqlMongoPurgeCommandTest extends PurgeCommandTestBase {} diff --git a/runtime/admin/src/test/java/org/apache/polaris/admintool/relational/jdbc/CockroachJdbcAdminProfile.java b/runtime/admin/src/test/java/org/apache/polaris/admintool/relational/jdbc/CockroachJdbcAdminProfile.java deleted file mode 100644 index 2b2e1d4d970..00000000000 --- a/runtime/admin/src/test/java/org/apache/polaris/admintool/relational/jdbc/CockroachJdbcAdminProfile.java +++ /dev/null @@ -1,47 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ -package org.apache.polaris.admintool.relational.jdbc; - -import java.util.List; -import java.util.Map; -import org.apache.polaris.test.commons.CockroachRelationalJdbcLifeCycleManagement; -import org.apache.polaris.test.commons.RelationalJdbcProfile; - -public class CockroachJdbcAdminProfile extends RelationalJdbcProfile { - @Override - public Map getConfigOverrides() { - return Map.of( - "polaris.persistence.type", - "relational-jdbc", - "polaris.persistence.relational.jdbc.database-type", - "cockroachdb", - // These two options are required to "trigger" the enablement of JDBC data sources in - // admin-tool tests, but do not harm other tests. - "quarkus.datasource.active", - "true", - "quarkus.datasource.db-kind", - "postgresql"); - } - - @Override - public List testResources() { - return List.of( - new TestResourceEntry(CockroachRelationalJdbcLifeCycleManagement.class, Map.of())); - } -} diff --git a/runtime/admin/src/test/java/org/apache/polaris/admintool/relational/jdbc/CockroachJdbcBootstrapCommandTest.java b/runtime/admin/src/test/java/org/apache/polaris/admintool/relational/jdbc/CockroachJdbcBootstrapCommandTest.java index c9fd056d6fa..2e030d70986 100644 --- a/runtime/admin/src/test/java/org/apache/polaris/admintool/relational/jdbc/CockroachJdbcBootstrapCommandTest.java +++ b/runtime/admin/src/test/java/org/apache/polaris/admintool/relational/jdbc/CockroachJdbcBootstrapCommandTest.java @@ -18,25 +18,8 @@ */ package org.apache.polaris.admintool.relational.jdbc; -import static org.assertj.core.api.Assertions.assertThat; - import io.quarkus.test.junit.TestProfile; -import io.quarkus.test.junit.main.LaunchResult; -import io.quarkus.test.junit.main.QuarkusMainLauncher; -import org.junit.jupiter.api.Test; - -@TestProfile(CockroachJdbcAdminProfile.class) -public class CockroachJdbcBootstrapCommandTest extends RelationalJdbcBootstrapCommandTest { +import org.apache.polaris.admintool.AdminProfiles; - @Override - @Test - public void testBootstrapFailsWhenAddingRealmWithDifferentSchemaVersion( - QuarkusMainLauncher launcher) { - // CockroachDB only has schema v4 (no v1, v2 or v3 schemas exist). - // Override to bootstrap with v4, which is the only version available for CockroachDB. - LaunchResult result1 = - launcher.launch("bootstrap", "-v", "4", "-r", "realm1", "-c", "realm1,root,s3cr3t"); - assertThat(result1.exitCode()).isEqualTo(0); - assertThat(result1.getOutput()).contains("Bootstrap completed successfully."); - } -} +@TestProfile(AdminProfiles.CockroachJdbc.class) +public class CockroachJdbcBootstrapCommandTest extends RelationalJdbcBootstrapCommandTest {} diff --git a/runtime/admin/src/test/java/org/apache/polaris/admintool/relational/jdbc/CockroachJdbcPurgeCommandTest.java b/runtime/admin/src/test/java/org/apache/polaris/admintool/relational/jdbc/CockroachJdbcPurgeCommandTest.java index 550eb779f46..c3556ec0b94 100644 --- a/runtime/admin/src/test/java/org/apache/polaris/admintool/relational/jdbc/CockroachJdbcPurgeCommandTest.java +++ b/runtime/admin/src/test/java/org/apache/polaris/admintool/relational/jdbc/CockroachJdbcPurgeCommandTest.java @@ -18,20 +18,9 @@ */ package org.apache.polaris.admintool.relational.jdbc; -import com.google.common.collect.ImmutableMap; import io.quarkus.test.junit.TestProfile; -import java.util.Map; +import org.apache.polaris.admintool.AdminProfiles; import org.apache.polaris.admintool.PurgeCommandTestBase; -@TestProfile(CockroachJdbcPurgeCommandTest.Profile.class) -public class CockroachJdbcPurgeCommandTest extends PurgeCommandTestBase { - public static class Profile extends CockroachJdbcAdminProfile { - @Override - public Map getConfigOverrides() { - return ImmutableMap.builder() - .putAll(super.getConfigOverrides()) - .put("pre-bootstrap", "true") - .build(); - } - } -} +@TestProfile(AdminProfiles.CockroachJdbc.class) +public class CockroachJdbcPurgeCommandTest extends PurgeCommandTestBase {} diff --git a/runtime/admin/src/test/java/org/apache/polaris/admintool/relational/jdbc/RelationalJdbcAdminProfile.java b/runtime/admin/src/test/java/org/apache/polaris/admintool/relational/jdbc/RelationalJdbcAdminProfile.java deleted file mode 100644 index 07d2e29d364..00000000000 --- a/runtime/admin/src/test/java/org/apache/polaris/admintool/relational/jdbc/RelationalJdbcAdminProfile.java +++ /dev/null @@ -1,45 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ -package org.apache.polaris.admintool.relational.jdbc; - -import java.util.List; -import java.util.Map; -import org.apache.polaris.test.commons.PostgresRelationalJdbcLifeCycleManagement; -import org.apache.polaris.test.commons.RelationalJdbcProfile; - -public class RelationalJdbcAdminProfile extends RelationalJdbcProfile { - @Override - public Map getConfigOverrides() { - return Map.of( - "polaris.persistence.type", - "relational-jdbc", - // These two options are required to "trigger" the enablement of JDBC data sources in - // admin-tool tests, but do not harm other tests. - "quarkus.datasource.active", - "true", - "quarkus.datasource.db-kind", - "postgresql"); - } - - @Override - public List testResources() { - return List.of( - new TestResourceEntry(PostgresRelationalJdbcLifeCycleManagement.class, Map.of())); - } -} diff --git a/runtime/admin/src/test/java/org/apache/polaris/admintool/relational/jdbc/RelationalJdbcBootstrapCommandTest.java b/runtime/admin/src/test/java/org/apache/polaris/admintool/relational/jdbc/RelationalJdbcBootstrapCommandTest.java index 5feb36d660e..68f81073b3f 100644 --- a/runtime/admin/src/test/java/org/apache/polaris/admintool/relational/jdbc/RelationalJdbcBootstrapCommandTest.java +++ b/runtime/admin/src/test/java/org/apache/polaris/admintool/relational/jdbc/RelationalJdbcBootstrapCommandTest.java @@ -23,26 +23,19 @@ import io.quarkus.test.junit.TestProfile; import io.quarkus.test.junit.main.LaunchResult; import io.quarkus.test.junit.main.QuarkusMainLauncher; +import org.apache.polaris.admintool.AdminProfiles; import org.apache.polaris.admintool.BootstrapCommandTestBase; import org.junit.jupiter.api.Test; -@TestProfile(RelationalJdbcAdminProfile.class) +@TestProfile(AdminProfiles.RelationalJdbc.class) public class RelationalJdbcBootstrapCommandTest extends BootstrapCommandTestBase { @Test - public void testBootstrapFailsWhenAddingRealmWithDifferentSchemaVersion( - QuarkusMainLauncher launcher) { - // First, bootstrap the schema to version 1 + public void testBootstrapWithExplicitSchemaVersion(QuarkusMainLauncher launcher) { LaunchResult result1 = - launcher.launch("bootstrap", "-v", "1", "-r", "realm1", "-c", "realm1,root,s3cr3t"); + launcher.launch("bootstrap", "-v", "4", "-r", "realm1", "-c", "realm1,root,s3cr3t"); assertThat(result1.exitCode()).isEqualTo(0); assertThat(result1.getOutput()).contains("Bootstrap completed successfully."); - - // TODO: enable this once we enable postgres container reuse in the same test. - // LaunchResult result2 = launcher.launch("bootstrap", "-v", "2", "-r", "realm2", "-c", - // "realm2,root,s3cr3t"); - // assertThat(result2.exitCode()).isEqualTo(EXIT_CODE_BOOTSTRAP_ERROR); - // assertThat(result2.getOutput()).contains("Cannot bootstrap due to schema version mismatch."); } @Test diff --git a/runtime/admin/src/test/java/org/apache/polaris/admintool/relational/jdbc/RelationalJdbcPurgeCommandTest.java b/runtime/admin/src/test/java/org/apache/polaris/admintool/relational/jdbc/RelationalJdbcPurgeCommandTest.java index 096708643cb..7fabdab8bc5 100644 --- a/runtime/admin/src/test/java/org/apache/polaris/admintool/relational/jdbc/RelationalJdbcPurgeCommandTest.java +++ b/runtime/admin/src/test/java/org/apache/polaris/admintool/relational/jdbc/RelationalJdbcPurgeCommandTest.java @@ -18,20 +18,9 @@ */ package org.apache.polaris.admintool.relational.jdbc; -import com.google.common.collect.ImmutableMap; import io.quarkus.test.junit.TestProfile; -import java.util.Map; +import org.apache.polaris.admintool.AdminProfiles; import org.apache.polaris.admintool.PurgeCommandTestBase; -@TestProfile(RelationalJdbcPurgeCommandTest.Profile.class) -public class RelationalJdbcPurgeCommandTest extends PurgeCommandTestBase { - public static class Profile extends RelationalJdbcAdminProfile { - @Override - public Map getConfigOverrides() { - return ImmutableMap.builder() - .putAll(super.getConfigOverrides()) - .put("pre-bootstrap", "true") - .build(); - } - } -} +@TestProfile(AdminProfiles.RelationalJdbc.class) +public class RelationalJdbcPurgeCommandTest extends PurgeCommandTestBase {} diff --git a/runtime/admin/src/testFixtures/java/org/apache/polaris/admintool/AdminProfiles.java b/runtime/admin/src/testFixtures/java/org/apache/polaris/admintool/AdminProfiles.java new file mode 100644 index 00000000000..3abe2b906cc --- /dev/null +++ b/runtime/admin/src/testFixtures/java/org/apache/polaris/admintool/AdminProfiles.java @@ -0,0 +1,95 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.polaris.admintool; + +import io.quarkus.test.junit.QuarkusTestProfile; +import java.util.List; +import java.util.Map; +import org.apache.polaris.test.commons.CockroachRelationalJdbcLifeCycleManagement; +import org.apache.polaris.test.commons.PostgresRelationalJdbcLifeCycleManagement; + +public final class AdminProfiles { + + private AdminProfiles() {} + + public static class NoSqlInMemory implements QuarkusTestProfile { + @Override + public Map getConfigOverrides() { + return Map.of( + "polaris.persistence.type", "nosql", "polaris.persistence.nosql.backend", "InMemory"); + } + } + + public static class NoSqlMongo implements QuarkusTestProfile { + @Override + public Map getConfigOverrides() { + return Map.of( + "polaris.persistence.type", "nosql", "polaris.persistence.nosql.backend", "MongoDb"); + } + + @Override + public List testResources() { + return List.of(new TestResourceEntry(MongoTestResourceLifecycleManager.class)); + } + } + + public static class RelationalJdbc implements QuarkusTestProfile { + @Override + public Map getConfigOverrides() { + return Map.of( + "polaris.persistence.type", + "relational-jdbc", + // These two options are required to "trigger" the enablement of JDBC data sources in + // admin-tool tests, but do not harm other tests. + "quarkus.datasource.active", + "true", + "quarkus.datasource.db-kind", + "postgresql"); + } + + @Override + public List testResources() { + return List.of( + new TestResourceEntry(PostgresRelationalJdbcLifeCycleManagement.class, Map.of())); + } + } + + public static class CockroachJdbc implements QuarkusTestProfile { + @Override + public Map getConfigOverrides() { + return Map.of( + "polaris.persistence.type", + "relational-jdbc", + "polaris.persistence.relational.jdbc.database-type", + "cockroachdb", + // These two options are required to "trigger" the enablement of JDBC data sources in + // admin-tool tests, but do not harm other tests. + "quarkus.datasource.active", + "true", + "quarkus.datasource.db-kind", + "postgresql"); + } + + @Override + public List testResources() { + return List.of( + new TestResourceEntry(CockroachRelationalJdbcLifeCycleManagement.class, Map.of())); + } + } +} diff --git a/runtime/server/build.gradle.kts b/runtime/server/build.gradle.kts index 87989965a0a..461b55b795d 100644 --- a/runtime/server/build.gradle.kts +++ b/runtime/server/build.gradle.kts @@ -40,11 +40,11 @@ dependencies { runtimeOnly(project(":polaris-extensions-auth-opa")) runtimeOnly(project(":polaris-extensions-auth-ranger")) - if ((project.findProperty("NonRESTCatalogs") as String?)?.contains("HIVE") == true) { + val nonRestCatalogs = providers.gradleProperty("NonRESTCatalogs").orNull + if (nonRestCatalogs?.contains("HIVE") == true) { runtimeOnly(project(":polaris-extensions-federation-hive")) } - - if ((project.findProperty("NonRESTCatalogs") as String?)?.contains("BIGQUERY") == true) { + if (nonRestCatalogs?.contains("BIGQUERY") == true) { runtimeOnly(project(":polaris-extensions-federation-bigquery")) } diff --git a/runtime/service/build.gradle.kts b/runtime/service/build.gradle.kts index 27d13eccec7..60ae5380df0 100644 --- a/runtime/service/build.gradle.kts +++ b/runtime/service/build.gradle.kts @@ -211,19 +211,6 @@ dependencies { tasks.named("javadoc") { dependsOn("jandex") } -tasks.withType(Test::class.java).configureEach { - if (System.getenv("AWS_REGION") == null) { - environment("AWS_REGION", "us-west-2") - } - // Note: the test secrets are referenced in - // org.apache.polaris.service.it.ServerManager - environment("POLARIS_BOOTSTRAP_CREDENTIALS", "POLARIS,test-admin,test-secret") - jvmArgs("--add-exports", "java.base/sun.nio.ch=ALL-UNNAMED") - // Need to allow a java security manager after Java 21, for Subject.getSubject to work - // "getSubject is supported only if a security manager is allowed". - systemProperty("java.security.manager", "allow") -} - val buildDir = project.layout.buildDirectory // Same issue as above: allow a java security manager after Java 21 @@ -243,27 +230,40 @@ val quarkusTestArgLine = val quarkusProperties = providers.systemPropertiesPrefixedBy("quarkus.") -listOf("intTest", "cloudTest").forEach { - tasks.named(it).configure(IntTestConfig(buildDir, quarkusProperties, quarkusTestArgLine)) -} +tasks.withType(Test::class.java).configureEach { + environment("AWS_REGION", providers.environmentVariable("AWS_REGION").getOrElse("us-west-2")) + // Note: the test secrets are referenced in + // org.apache.polaris.service.it.ServerManager + environment("POLARIS_BOOTSTRAP_CREDENTIALS", "POLARIS,test-admin,test-secret") + jvmArgs("--add-exports", "java.base/sun.nio.ch=ALL-UNNAMED") + // Need to allow a java security manager after Java 21, for Subject.getSubject to work + // "getSubject is supported only if a security manager is allowed". + systemProperty("java.security.manager", "allow") -class IntTestConfig( - @get:Internal val buildDir: DirectoryProperty, - @get:Input val quarkusProperties: Provider>, - @get:Input val quarkusTestArgLine: String, -) : Action { - override fun execute(t: Test) { - t.maxParallelForks = 1 + // Quarkus tests run "in isolated class loaders", which means that class-statically active + // resources pile up used JVM, as those classes cannot be GC'd. + // Examples of those statically held active resources are: + // - Iceberg's worker pools (thread pools, executors, etc.) + // - Hadoop's stats-cleaner (org.apache.hadoop.fs.FileSystem.Statistics.STATS_DATA_CLEANER) + // - Guava's 'MoreExecutors' (via Iceberg `ThreadPools`)` + // Forcing a new JVM after each test class works around this issue. + if ("test" == name) { + forkEvery = 1 + + // enlarge the max heap size to avoid out of memory error + maxHeapSize = "4g" + } + if ("intTest" == name || "cloutTest" == name) { val logsDir = buildDir.map { b -> b.asFile.resolve("logs") } // JVM arguments provider does not interfere with Gradle's cache keys - t.jvmArgumentProviders.add( + jvmArgumentProviders.add( IntTestArgumentProvider(logsDir, quarkusProperties, quarkusTestArgLine) ) // delete files from previous runs - t.doFirst(IntTestCleanup(logsDir, buildDir)) + doFirst(IntTestCleanup(logsDir, buildDir)) } } diff --git a/runtime/service/src/intTest/java/org/apache/polaris/service/it/PolarisRestCatalogMinIOIT.java b/runtime/service/src/intTest/java/org/apache/polaris/service/it/PolarisRestCatalogMinIOIT.java index ffccea4c778..50d7b8e8115 100644 --- a/runtime/service/src/intTest/java/org/apache/polaris/service/it/PolarisRestCatalogMinIOIT.java +++ b/runtime/service/src/intTest/java/org/apache/polaris/service/it/PolarisRestCatalogMinIOIT.java @@ -22,9 +22,10 @@ import static org.apache.polaris.test.commons.MinioRustProfile.SECRET_KEY; import com.google.common.collect.ImmutableMap; +import io.quarkus.test.common.QuarkusTestResource; +import io.quarkus.test.common.ResourceArg; import io.quarkus.test.junit.QuarkusIntegrationTest; import io.quarkus.test.junit.TestProfile; -import java.net.URI; import java.util.List; import java.util.Map; import org.apache.iceberg.aws.s3.S3FileIOProperties; @@ -36,35 +37,42 @@ import org.apache.polaris.test.commons.MinioRustProfile; import org.apache.polaris.test.minio.Minio; import org.apache.polaris.test.minio.MinioAccess; -import org.apache.polaris.test.minio.MinioExtension; -import org.junit.jupiter.api.BeforeAll; +import org.apache.polaris.test.minio.MinioConditionExtension; +import org.apache.polaris.test.minio.MinioTestResource; +import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.extension.ExtendWith; @QuarkusIntegrationTest @TestProfile(MinioRustProfile.class) -@ExtendWith(MinioExtension.class) +@QuarkusTestResource( + value = MinioTestResource.class, + initArgs = { + @ResourceArg(name = "accessKey", value = ACCESS_KEY), + @ResourceArg(name = "secretKey", value = SECRET_KEY) + }) +@ExtendWith(MinioConditionExtension.class) @ExtendWith(PolarisIntegrationTestExtension.class) @RestCatalogConfig({"header.X-Iceberg-Access-Delegation", "vended-credentials"}) public class PolarisRestCatalogMinIOIT extends PolarisRestCatalogIntegrationBase { protected static final String BUCKET_URI_PREFIX = "/minio-test-polaris"; - private static URI storageBase; - private static String endpoint; + @Minio static MinioAccess minioAccess; private static Map s3Properties; - @BeforeAll - static void setup( - @Minio(accessKey = ACCESS_KEY, secretKey = SECRET_KEY) MinioAccess minioAccess) { - storageBase = minioAccess.s3BucketUri(BUCKET_URI_PREFIX); - endpoint = minioAccess.s3endpoint(); + @BeforeEach + void setup() { s3Properties = Map.of( - S3FileIOProperties.ENDPOINT, endpoint, - S3FileIOProperties.PATH_STYLE_ACCESS, "true", - S3FileIOProperties.ACCESS_KEY_ID, ACCESS_KEY, - S3FileIOProperties.SECRET_ACCESS_KEY, SECRET_KEY); + S3FileIOProperties.ENDPOINT, + minioAccess.s3endpoint(), + S3FileIOProperties.PATH_STYLE_ACCESS, + "true", + S3FileIOProperties.ACCESS_KEY_ID, + ACCESS_KEY, + S3FileIOProperties.SECRET_ACCESS_KEY, + SECRET_KEY); } @Override @@ -78,8 +86,8 @@ protected StorageConfigInfo getStorageConfigInfo() { AwsStorageConfigInfo.builder() .setStorageType(StorageConfigInfo.StorageTypeEnum.S3) .setPathStyleAccess(true) - .setEndpoint(endpoint) - .setAllowedLocations(List.of(storageBase.toString())); + .setEndpoint(minioAccess.s3endpoint()) + .setAllowedLocations(List.of(minioAccess.s3BucketUri(BUCKET_URI_PREFIX).toString())); return storageConfig.build(); } diff --git a/runtime/service/src/intTest/java/org/apache/polaris/service/it/PolarisRestCatalogRustFSIT.java b/runtime/service/src/intTest/java/org/apache/polaris/service/it/PolarisRestCatalogRustFSIT.java index 5c3b833806a..f8c2f3626bc 100644 --- a/runtime/service/src/intTest/java/org/apache/polaris/service/it/PolarisRestCatalogRustFSIT.java +++ b/runtime/service/src/intTest/java/org/apache/polaris/service/it/PolarisRestCatalogRustFSIT.java @@ -22,9 +22,10 @@ import static org.apache.polaris.test.commons.MinioRustProfile.SECRET_KEY; import com.google.common.collect.ImmutableMap; +import io.quarkus.test.common.QuarkusTestResource; +import io.quarkus.test.common.ResourceArg; import io.quarkus.test.junit.QuarkusIntegrationTest; import io.quarkus.test.junit.TestProfile; -import java.net.URI; import java.util.List; import java.util.Map; import org.apache.polaris.core.admin.model.AwsStorageConfigInfo; @@ -35,33 +36,31 @@ import org.apache.polaris.test.commons.MinioRustProfile; import org.apache.polaris.test.rustfs.Rustfs; import org.apache.polaris.test.rustfs.RustfsAccess; -import org.apache.polaris.test.rustfs.RustfsExtension; -import org.junit.jupiter.api.BeforeAll; +import org.apache.polaris.test.rustfs.RustfsConditionExtension; +import org.apache.polaris.test.rustfs.RustfsTestResource; import org.junit.jupiter.api.TestInfo; import org.junit.jupiter.api.extension.ExtendWith; @QuarkusIntegrationTest @TestProfile(MinioRustProfile.class) -@ExtendWith(RustfsExtension.class) +@QuarkusTestResource( + value = RustfsTestResource.class, + initArgs = { + @ResourceArg(name = "accessKey", value = ACCESS_KEY), + @ResourceArg(name = "secretKey", value = SECRET_KEY) + }) +@ExtendWith(RustfsConditionExtension.class) @ExtendWith(PolarisIntegrationTestExtension.class) public class PolarisRestCatalogRustFSIT extends PolarisRestCatalogIntegrationBase { protected static final String BUCKET_URI_PREFIX = "/rustfs-test-polaris"; - private static URI storageBase; - private static String endpoint; - - @BeforeAll - static void setup( - @Rustfs(accessKey = ACCESS_KEY, secretKey = SECRET_KEY) RustfsAccess rustfsAccess) { - storageBase = rustfsAccess.s3BucketUri(BUCKET_URI_PREFIX); - endpoint = rustfsAccess.s3endpoint(); - } + @Rustfs static RustfsAccess rustfsAccess; @Override protected ImmutableMap.Builder clientFileIOProperties() { return super.clientFileIOProperties() - .put(StorageAccessProperty.AWS_ENDPOINT.getPropertyName(), endpoint) + .put(StorageAccessProperty.AWS_ENDPOINT.getPropertyName(), rustfsAccess.s3endpoint()) .put(StorageAccessProperty.AWS_PATH_STYLE_ACCESS.getPropertyName(), "true") .put(StorageAccessProperty.AWS_KEY_ID.getPropertyName(), ACCESS_KEY) .put(StorageAccessProperty.AWS_SECRET_KEY.getPropertyName(), SECRET_KEY); @@ -73,8 +72,8 @@ protected StorageConfigInfo getStorageConfigInfo() { AwsStorageConfigInfo.builder() .setStorageType(StorageConfigInfo.StorageTypeEnum.S3) .setPathStyleAccess(true) - .setEndpoint(endpoint) - .setAllowedLocations(List.of(storageBase.toString())); + .setEndpoint(rustfsAccess.s3endpoint()) + .setAllowedLocations(List.of(rustfsAccess.s3BucketUri(BUCKET_URI_PREFIX).toString())); return storageConfig.build(); } @@ -82,7 +81,7 @@ protected StorageConfigInfo getStorageConfigInfo() { @Override protected Map extraCatalogProperties(TestInfo testInfo) { return ImmutableMap.builder() - .put(StorageAccessProperty.AWS_ENDPOINT.getPropertyName(), endpoint) + .put(StorageAccessProperty.AWS_ENDPOINT.getPropertyName(), rustfsAccess.s3endpoint()) .put(StorageAccessProperty.AWS_PATH_STYLE_ACCESS.getPropertyName(), "true") .put(StorageAccessProperty.AWS_KEY_ID.getPropertyName(), ACCESS_KEY) .put(StorageAccessProperty.AWS_SECRET_KEY.getPropertyName(), SECRET_KEY) diff --git a/runtime/service/src/intTest/java/org/apache/polaris/service/it/RestCatalogMinIOSpecialIT.java b/runtime/service/src/intTest/java/org/apache/polaris/service/it/RestCatalogMinIOSpecialIT.java index 86e8b34e75d..7153150e6e7 100644 --- a/runtime/service/src/intTest/java/org/apache/polaris/service/it/RestCatalogMinIOSpecialIT.java +++ b/runtime/service/src/intTest/java/org/apache/polaris/service/it/RestCatalogMinIOSpecialIT.java @@ -36,6 +36,8 @@ import static org.assertj.core.api.Assertions.assertThatThrownBy; import com.google.common.collect.ImmutableMap; +import io.quarkus.test.common.QuarkusTestResource; +import io.quarkus.test.common.ResourceArg; import io.quarkus.test.junit.QuarkusIntegrationTest; import io.quarkus.test.junit.TestProfile; import java.io.IOException; @@ -44,6 +46,7 @@ import java.util.List; import java.util.Map; import java.util.Optional; +import java.util.concurrent.atomic.AtomicBoolean; import org.apache.iceberg.DataFile; import org.apache.iceberg.DataFiles; import org.apache.iceberg.FileFormat; @@ -76,10 +79,10 @@ import org.apache.polaris.test.commons.MinioRustProfile; import org.apache.polaris.test.minio.Minio; import org.apache.polaris.test.minio.MinioAccess; -import org.apache.polaris.test.minio.MinioExtension; +import org.apache.polaris.test.minio.MinioConditionExtension; +import org.apache.polaris.test.minio.MinioTestResource; import org.junit.jupiter.api.AfterAll; import org.junit.jupiter.api.AfterEach; -import org.junit.jupiter.api.BeforeAll; import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; import org.junit.jupiter.api.TestInfo; @@ -98,7 +101,13 @@ */ @QuarkusIntegrationTest @TestProfile(MinioRustProfile.class) -@ExtendWith(MinioExtension.class) +@QuarkusTestResource( + value = MinioTestResource.class, + initArgs = { + @ResourceArg(name = "accessKey", value = ACCESS_KEY), + @ResourceArg(name = "secretKey", value = SECRET_KEY) + }) +@ExtendWith(MinioConditionExtension.class) @ExtendWith(PolarisIntegrationTestExtension.class) public class RestCatalogMinIOSpecialIT { @@ -112,6 +121,9 @@ public class RestCatalogMinIOSpecialIT { required(1, "id", Types.IntegerType.get(), "doc"), optional(2, "data", Types.StringType.get())); + @Minio static MinioAccess minioAccess; + + private static final AtomicBoolean initialized = new AtomicBoolean(false); private static PolarisApiEndpoints endpoints; private static PolarisClient client; private static ManagementApi managementApi; @@ -124,27 +136,24 @@ public class RestCatalogMinIOSpecialIT { private PrincipalWithCredentials principalCredentials; private String catalogName; - @BeforeAll - static void setup( - PolarisApiEndpoints apiEndpoints, - @Minio(accessKey = ACCESS_KEY, secretKey = SECRET_KEY) MinioAccess minioAccess, - ClientCredentials credentials) { - s3Client = minioAccess.s3Client(); - endpoints = apiEndpoints; - client = polarisClient(endpoints); - adminToken = client.obtainToken(credentials); - managementApi = client.managementApi(adminToken); - storageBase = minioAccess.s3BucketUri(BUCKET_URI_PREFIX); - endpoint = minioAccess.s3endpoint(); - } - @AfterAll static void close() throws Exception { client.close(); } @BeforeEach - public void before(TestInfo testInfo) { + public void before( + TestInfo testInfo, PolarisApiEndpoints apiEndpoints, ClientCredentials credentials) { + if (initialized.compareAndSet(false, true)) { + endpoints = apiEndpoints; + s3Client = minioAccess.s3Client(); + client = polarisClient(endpoints); + adminToken = client.obtainToken(credentials); + managementApi = client.managementApi(adminToken); + storageBase = minioAccess.s3BucketUri(BUCKET_URI_PREFIX); + endpoint = minioAccess.s3endpoint(); + } + String principalName = client.newEntityName("test-user"); principalRoleName = client.newEntityName("test-admin"); principalCredentials = managementApi.createPrincipalWithRole(principalName, principalRoleName); diff --git a/runtime/service/src/intTest/java/org/apache/polaris/service/it/RestCatalogRustFSSpecialIT.java b/runtime/service/src/intTest/java/org/apache/polaris/service/it/RestCatalogRustFSSpecialIT.java index 569c68d6acd..71ca79f080d 100644 --- a/runtime/service/src/intTest/java/org/apache/polaris/service/it/RestCatalogRustFSSpecialIT.java +++ b/runtime/service/src/intTest/java/org/apache/polaris/service/it/RestCatalogRustFSSpecialIT.java @@ -36,6 +36,8 @@ import static org.assertj.core.api.Assertions.assertThatThrownBy; import com.google.common.collect.ImmutableMap; +import io.quarkus.test.common.QuarkusTestResource; +import io.quarkus.test.common.ResourceArg; import io.quarkus.test.junit.QuarkusIntegrationTest; import io.quarkus.test.junit.TestProfile; import java.io.IOException; @@ -44,6 +46,7 @@ import java.util.List; import java.util.Map; import java.util.Optional; +import java.util.concurrent.atomic.AtomicBoolean; import org.apache.iceberg.DataFile; import org.apache.iceberg.DataFiles; import org.apache.iceberg.FileFormat; @@ -76,10 +79,10 @@ import org.apache.polaris.test.commons.MinioRustProfile; import org.apache.polaris.test.rustfs.Rustfs; import org.apache.polaris.test.rustfs.RustfsAccess; -import org.apache.polaris.test.rustfs.RustfsExtension; +import org.apache.polaris.test.rustfs.RustfsConditionExtension; +import org.apache.polaris.test.rustfs.RustfsTestResource; import org.junit.jupiter.api.AfterAll; import org.junit.jupiter.api.AfterEach; -import org.junit.jupiter.api.BeforeAll; import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; import org.junit.jupiter.api.TestInfo; @@ -98,7 +101,13 @@ */ @QuarkusIntegrationTest @TestProfile(MinioRustProfile.class) -@ExtendWith(RustfsExtension.class) +@QuarkusTestResource( + value = RustfsTestResource.class, + initArgs = { + @ResourceArg(name = "accessKey", value = ACCESS_KEY), + @ResourceArg(name = "secretKey", value = SECRET_KEY) + }) +@ExtendWith(RustfsConditionExtension.class) @ExtendWith(PolarisIntegrationTestExtension.class) public class RestCatalogRustFSSpecialIT { @@ -112,6 +121,9 @@ public class RestCatalogRustFSSpecialIT { required(1, "id", Types.IntegerType.get(), "doc"), optional(2, "data", Types.StringType.get())); + @Rustfs static RustfsAccess rustfsAccess; + + private static final AtomicBoolean initialized = new AtomicBoolean(false); private static PolarisApiEndpoints endpoints; private static PolarisClient client; private static ManagementApi managementApi; @@ -124,27 +136,24 @@ public class RestCatalogRustFSSpecialIT { private PrincipalWithCredentials principalCredentials; private String catalogName; - @BeforeAll - static void setup( - PolarisApiEndpoints apiEndpoints, - @Rustfs(accessKey = ACCESS_KEY, secretKey = SECRET_KEY) RustfsAccess rustfsAccess, - ClientCredentials credentials) { - s3Client = rustfsAccess.s3Client(); - endpoints = apiEndpoints; - client = polarisClient(endpoints); - adminToken = client.obtainToken(credentials); - managementApi = client.managementApi(adminToken); - storageBase = rustfsAccess.s3BucketUri(BUCKET_URI_PREFIX); - endpoint = rustfsAccess.s3endpoint(); - } - @AfterAll static void close() throws Exception { client.close(); } @BeforeEach - public void before(TestInfo testInfo) { + public void before( + TestInfo testInfo, PolarisApiEndpoints apiEndpoints, ClientCredentials credentials) { + if (initialized.compareAndSet(false, true)) { + endpoints = apiEndpoints; + s3Client = rustfsAccess.s3Client(); + client = polarisClient(endpoints); + adminToken = client.obtainToken(credentials); + managementApi = client.managementApi(adminToken); + storageBase = rustfsAccess.s3BucketUri(BUCKET_URI_PREFIX); + endpoint = rustfsAccess.s3endpoint(); + } + String principalName = client.newEntityName("test-user"); principalRoleName = client.newEntityName("test-admin"); principalCredentials = managementApi.createPrincipalWithRole(principalName, principalRoleName); diff --git a/runtime/service/src/test/java/org/apache/polaris/service/Profiles.java b/runtime/service/src/test/java/org/apache/polaris/service/Profiles.java new file mode 100644 index 00000000000..db3476e9fa3 --- /dev/null +++ b/runtime/service/src/test/java/org/apache/polaris/service/Profiles.java @@ -0,0 +1,297 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.polaris.service; + +import com.google.common.collect.ImmutableMap; +import io.quarkus.test.junit.QuarkusTestProfile; +import java.util.Map; +import java.util.Set; +import org.apache.polaris.service.admin.PolarisAuthzTestBase; +import org.apache.polaris.test.commons.NoSqlInMemoryProfile; + +public final class Profiles { + + private Profiles() {} + + public static final Map DEFAULT_PROFILE_CONFIG_OVERRIDES = + Map.of( + "polaris.features.\"ALLOW_SPECIFYING_FILE_IO_IMPL\"", + "true", + "polaris.features.\"ALLOW_INSECURE_STORAGE_TYPES\"", + "true", + "polaris.features.\"SUPPORTED_CATALOG_STORAGE_TYPES\"", + "[\"FILE\",\"S3\"]", + "polaris.event-listener.type", + "test", + "polaris.readiness.ignore-severe-issues", + "true"); + + public static class DefaultProfile implements QuarkusTestProfile { + @Override + public Map getConfigOverrides() { + return DEFAULT_PROFILE_CONFIG_OVERRIDES; + } + } + + public static class ApplicationIntegrationProfile extends DefaultProfile { + @Override + public Map getConfigOverrides() { + return ImmutableMap.builder() + .putAll(super.getConfigOverrides()) + .put("quarkus.http.limits.max-body-size", "1000000") + .put("polaris.realm-context.realms", "POLARIS,OTHER") + .put("polaris.features.\"ALLOW_OVERLAPPING_CATALOG_URLS\"", "true") + .put("polaris.features.\"SKIP_CREDENTIAL_SUBSCOPING_INDIRECTION\"", "true") + .build(); + } + } + + public static class ManagementIntegrationProfile implements QuarkusTestProfile { + @Override + public Map getConfigOverrides() { + return Map.of( + "polaris.features.\"ALLOW_OVERLAPPING_CATALOG_URLS\"", + "true", + "polaris.features.\"ENFORCE_PRINCIPAL_CREDENTIAL_ROTATION_REQUIRED_CHECKING\"", + "true", + "polaris.storage.gcp.token", + "token", + "polaris.storage.gcp.lifespan", + "PT1H"); + } + } + + public static class RestCatalogFileIntegrationProfile extends DefaultProfile { + @Override + public Map getConfigOverrides() { + return ImmutableMap.builder() + .putAll(super.getConfigOverrides()) + .put("polaris.features.\"ALLOW_EXTERNAL_CATALOG_CREDENTIAL_VENDING\"", "false") + .put("polaris.features.\"ALLOW_NAMESPACE_CUSTOM_LOCATION\"", "true") + .build(); + } + } + + public static class RestCatalogViewFileIntegrationProfile implements QuarkusTestProfile { + @Override + public Map getConfigOverrides() { + return Map.of( + "polaris.features.\"ALLOW_INSECURE_STORAGE_TYPES\"", + "true", + "polaris.features.\"SUPPORTED_CATALOG_STORAGE_TYPES\"", + "[\"FILE\"]", + "polaris.readiness.ignore-severe-issues", + "true"); + } + } + + public static class DefaultNoSqlProfile extends DefaultProfile { + @Override + public Map getConfigOverrides() { + return ImmutableMap.builder() + .putAll(super.getConfigOverrides()) + .putAll(NoSqlInMemoryProfile.NOSQL_PERSISTENCE) + .build(); + } + } + + public static class DefaultIcebergCatalogProfile extends DefaultProfile { + @Override + public Map getConfigOverrides() { + return ImmutableMap.builder() + .putAll(super.getConfigOverrides()) + .put("polaris.features.\"ALLOW_TABLE_LOCATION_OVERLAP\"", "true") + .put("polaris.features.\"LIST_PAGINATION_ENABLED\"", "true") + .put("polaris.features.\"ALLOW_WILDCARD_LOCATION\"", "true") + .put("polaris.features.\"SKIP_CREDENTIAL_SUBSCOPING_INDIRECTION\"", "true") + .put("polaris.behavior-changes.\"ALLOW_NAMESPACE_CUSTOM_LOCATION\"", "true") + .put("polaris.test.rootAugmentor.enabled", "true") + .build(); + } + } + + public static class NoSqlIcebergCatalogProfile extends DefaultIcebergCatalogProfile { + @Override + public Map getConfigOverrides() { + return ImmutableMap.builder() + .putAll(super.getConfigOverrides()) + .putAll(NoSqlInMemoryProfile.NOSQL_PERSISTENCE) + .build(); + } + } + + public static class PolarisAuthzBaseProfile extends DefaultProfile { + @Override + public Set> getEnabledAlternatives() { + return Set.of(PolarisAuthzTestBase.TestPolarisLocalCatalogFactory.class); + } + + @Override + public Map getConfigOverrides() { + return ImmutableMap.builder() + .putAll(super.getConfigOverrides()) + .put("polaris.features.\"ALLOW_EXTERNAL_METADATA_FILE_LOCATION\"", "true") + .put("polaris.features.\"ENABLE_GENERIC_TABLES\"", "true") + .put( + "polaris.features.\"ENFORCE_PRINCIPAL_CREDENTIAL_ROTATION_REQUIRED_CHECKING\"", + "true") + .put("polaris.features.\"DROP_WITH_PURGE_ENABLED\"", "true") + .put("polaris.behavior-changes.\"ALLOW_NAMESPACE_CUSTOM_LOCATION\"", "true") + .put("polaris.features.\"ENABLE_CATALOG_FEDERATION\"", "true") + .build(); + } + } + + public static class IcebergCatalogHandlerNoSqlAuthzProfile extends PolarisAuthzBaseProfile { + @Override + public Map getConfigOverrides() { + return ImmutableMap.builder() + .putAll(super.getConfigOverrides()) + .putAll(NoSqlInMemoryProfile.NOSQL_PERSISTENCE) + .build(); + } + } + + static final Map IN_MEMORY_BUFFER_EVENT_LISTENER_BASE_CONFIG = + ImmutableMap.builder() + .put("polaris.realm-context.realms", "test1,test2") + .put("polaris.persistence.type", "relational-jdbc") + .put("polaris.persistence.auto-bootstrap-types", "relational-jdbc") + .put("quarkus.datasource.db-kind", "h2") + .put( + "quarkus.datasource.jdbc.url", + "jdbc:h2:mem:test;DB_CLOSE_DELAY=-1;MODE=PostgreSQL;DATABASE_TO_LOWER=TRUE") + .put("polaris.event-listener.type", "persistence-in-memory-buffer") + .put( + "quarkus.fault-tolerance.\"org.apache.polaris.service.events.listeners.inmemory.InMemoryBufferEventListener/flush\".retry.max-retries", + "1") + .put( + "quarkus.fault-tolerance.\"org.apache.polaris.service.events.listeners.inmemory.InMemoryBufferEventListener/flush\".retry.delay", + "10") + .build(); + + public static class InMemoryBufferEventListenerBufferSizeProfile implements QuarkusTestProfile { + @Override + public Map getConfigOverrides() { + return ImmutableMap.builder() + .putAll(IN_MEMORY_BUFFER_EVENT_LISTENER_BASE_CONFIG) + .put("polaris.event-listener.persistence-in-memory-buffer.buffer-time", "60s") + .put("polaris.event-listener.persistence-in-memory-buffer.max-buffer-size", "10") + .build(); + } + } + + public static class InMemoryBufferEventListenerBufferTimeProfile implements QuarkusTestProfile { + @Override + public Map getConfigOverrides() { + return ImmutableMap.builder() + .putAll(IN_MEMORY_BUFFER_EVENT_LISTENER_BASE_CONFIG) + .put("polaris.event-listener.persistence-in-memory-buffer.buffer-time", "100ms") + .put("polaris.event-listener.persistence-in-memory-buffer.max-buffer-size", "1000") + .build(); + } + } + + public static class RealmIdTagEnabledMetricsProfile implements QuarkusTestProfile { + @Override + public Map getConfigOverrides() { + return Map.of( + "polaris.metrics.tags.environment", + "prod", + "polaris.realm-context.type", + "test", + "polaris.metrics.realm-id-tag.enable-in-api-metrics", + "true", + "polaris.metrics.realm-id-tag.enable-in-http-metrics", + "true", + "polaris.metrics.user-principal-tag.enable-in-api-metrics", + "false"); + } + } + + public static class UserPrincipalTagEnabledMetricsProfile implements QuarkusTestProfile { + @Override + public Map getConfigOverrides() { + return Map.of( + "polaris.metrics.tags.environment", + "prod", + "polaris.metrics.user-principal-tag.enable-in-api-metrics", + "true", + "polaris.metrics.realm-id-tag.enable-in-api-metrics", + "false", + "polaris.metrics.realm-id-tag.enable-in-http-metrics", + "false"); + } + } + + public static class TagsDisabledMetricsProfile implements QuarkusTestProfile { + @Override + public Map getConfigOverrides() { + return Map.of( + "polaris.metrics.tags.environment", "prod", "polaris.realm-context.type", "test"); + } + } + + public static class PolarisEventListenersTestProfile implements QuarkusTestProfile { + @Override + public Map getConfigOverrides() { + return ImmutableMap.builder() + .put( + "polaris.event-listener.types", + "after-send-listener,before-send-listener,consume-all-listener,consume-all-listener-2,consume-only-catalog-listener,consume-catalog-and-after-notification-listener") + .put( + "polaris.event-listener.after-send-listener.enabled-event-types", + "AFTER_SEND_NOTIFICATION") + .put( + "polaris.event-listener.before-send-listener.enabled-event-types", + "BEFORE_SEND_NOTIFICATION") + .put( + "polaris.event-listener.consume-only-catalog-listener.enabled-event-categories", + "CATALOG") + .put( + "polaris.event-listener.consume-catalog-and-after-notification-listener.enabled-event-categories", + "CATALOG") + .put( + "polaris.event-listener.consume-catalog-and-after-notification-listener.enabled-event-types", + "AFTER_SEND_NOTIFICATION") + .build(); + } + } + + public static class InMemoryBufferEventListenerIntegrationProfile implements QuarkusTestProfile { + @Override + public Map getConfigOverrides() { + return ImmutableMap.builder() + .put("polaris.persistence.type", "relational-jdbc") + .put("polaris.persistence.auto-bootstrap-types", "relational-jdbc") + .put("quarkus.datasource.db-kind", "h2") + .put("quarkus.otel.sdk.disabled", "false") + .put( + "quarkus.datasource.jdbc.url", + "jdbc:h2:mem:test;DB_CLOSE_DELAY=-1;MODE=PostgreSQL;DATABASE_TO_LOWER=TRUE") + .put("polaris.event-listener.type", "persistence-in-memory-buffer") + .put("polaris.event-listener.persistence-in-memory-buffer.buffer-time", "100ms") + .put("polaris.features.\"ALLOW_INSECURE_STORAGE_TYPES\"", "true") + .put("polaris.features.\"SUPPORTED_CATALOG_STORAGE_TYPES\"", "[\"FILE\",\"S3\"]") + .put("polaris.readiness.ignore-severe-issues", "true") + .build(); + } + } +} diff --git a/runtime/service/src/test/java/org/apache/polaris/service/admin/PolarisAdminServiceAuthzTest.java b/runtime/service/src/test/java/org/apache/polaris/service/admin/PolarisAdminServiceAuthzTest.java index eadd33aff7d..77866abbc9c 100644 --- a/runtime/service/src/test/java/org/apache/polaris/service/admin/PolarisAdminServiceAuthzTest.java +++ b/runtime/service/src/test/java/org/apache/polaris/service/admin/PolarisAdminServiceAuthzTest.java @@ -34,11 +34,12 @@ import org.apache.polaris.core.entity.PolarisPrivilege; import org.apache.polaris.core.entity.PrincipalEntity; import org.apache.polaris.core.entity.PrincipalRoleEntity; +import org.apache.polaris.service.Profiles; import org.junit.jupiter.api.DynamicNode; import org.junit.jupiter.api.TestFactory; @QuarkusTest -@TestProfile(PolarisAuthzTestBase.Profile.class) +@TestProfile(Profiles.PolarisAuthzBaseProfile.class) public class PolarisAdminServiceAuthzTest extends PolarisAuthzTestBase { private PolarisAdminService newTestAdminService() { return newTestAdminService(Set.of()); diff --git a/runtime/service/src/test/java/org/apache/polaris/service/admin/PolarisAuthzTestBase.java b/runtime/service/src/test/java/org/apache/polaris/service/admin/PolarisAuthzTestBase.java index 23dd67f428e..5a5a69015ee 100644 --- a/runtime/service/src/test/java/org/apache/polaris/service/admin/PolarisAuthzTestBase.java +++ b/runtime/service/src/test/java/org/apache/polaris/service/admin/PolarisAuthzTestBase.java @@ -74,7 +74,6 @@ import org.apache.polaris.core.secrets.UserSecretsManager; import org.apache.polaris.core.storage.cache.StorageCredentialCache; import org.apache.polaris.service.catalog.PolarisPassthroughResolutionView; -import org.apache.polaris.service.catalog.Profiles; import org.apache.polaris.service.catalog.generic.PolarisGenericTableCatalog; import org.apache.polaris.service.catalog.iceberg.LocalIcebergCatalog; import org.apache.polaris.service.catalog.io.FileIOFactory; @@ -100,29 +99,6 @@ /** Base class for shared test setup logic used by various Polaris authz-related tests. */ public abstract class PolarisAuthzTestBase { - public static class Profile extends Profiles.DefaultProfile { - - @Override - public Set> getEnabledAlternatives() { - return Set.of(TestPolarisLocalCatalogFactory.class); - } - - @Override - public Map getConfigOverrides() { - return ImmutableMap.builder() - .putAll(super.getConfigOverrides()) - .put("polaris.features.\"ALLOW_EXTERNAL_METADATA_FILE_LOCATION\"", "true") - .put("polaris.features.\"ENABLE_GENERIC_TABLES\"", "true") - .put( - "polaris.features.\"ENFORCE_PRINCIPAL_CREDENTIAL_ROTATION_REQUIRED_CHECKING\"", - "true") - .put("polaris.features.\"DROP_WITH_PURGE_ENABLED\"", "true") - .put("polaris.behavior-changes.\"ALLOW_NAMESPACE_CUSTOM_LOCATION\"", "true") - .put("polaris.features.\"ENABLE_CATALOG_FEDERATION\"", "true") - .build(); - } - } - protected static final String CATALOG_NAME = "polaris-catalog"; protected static final String FEDERATED_CATALOG_NAME = "federated-polaris-catalog"; protected static final String PRINCIPAL_NAME = "snowman"; diff --git a/runtime/service/src/test/java/org/apache/polaris/service/catalog/Profiles.java b/runtime/service/src/test/java/org/apache/polaris/service/catalog/Profiles.java deleted file mode 100644 index 9c5411037dd..00000000000 --- a/runtime/service/src/test/java/org/apache/polaris/service/catalog/Profiles.java +++ /dev/null @@ -1,61 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -package org.apache.polaris.service.catalog; - -import com.google.common.collect.ImmutableMap; -import io.quarkus.test.junit.QuarkusTestProfile; -import java.util.Map; - -public final class Profiles { - private Profiles() {} - - public static final Map NOSQL_IN_MEM = - ImmutableMap.builder() - .put("polaris.persistence.type", "nosql") - .put("polaris.persistence.nosql.backend", "InMemory") - .build(); - - public static class DefaultProfile implements QuarkusTestProfile { - @Override - public Map getConfigOverrides() { - return Map.of( - "polaris.features.\"ALLOW_SPECIFYING_FILE_IO_IMPL\"", - "true", - "polaris.features.\"ALLOW_INSECURE_STORAGE_TYPES\"", - "true", - "polaris.features.\"SUPPORTED_CATALOG_STORAGE_TYPES\"", - "[\"FILE\",\"S3\"]", - "polaris.event-listener.type", - "test", - "polaris.readiness.ignore-severe-issues", - "true"); - } - } - - public static class DefaultNoSqlProfile extends DefaultProfile { - @Override - public Map getConfigOverrides() { - return ImmutableMap.builder() - .putAll(super.getConfigOverrides()) - .putAll(NOSQL_IN_MEM) - .build(); - } - } -} diff --git a/runtime/service/src/test/java/org/apache/polaris/service/catalog/generic/PolarisGenericTableCatalogHandlerAuthzTest.java b/runtime/service/src/test/java/org/apache/polaris/service/catalog/generic/PolarisGenericTableCatalogHandlerAuthzTest.java index 0dceebe8c2a..c68d1b1765d 100644 --- a/runtime/service/src/test/java/org/apache/polaris/service/catalog/generic/PolarisGenericTableCatalogHandlerAuthzTest.java +++ b/runtime/service/src/test/java/org/apache/polaris/service/catalog/generic/PolarisGenericTableCatalogHandlerAuthzTest.java @@ -27,12 +27,13 @@ import org.apache.iceberg.catalog.TableIdentifier; import org.apache.polaris.core.auth.PolarisPrincipal; import org.apache.polaris.core.entity.PolarisPrivilege; +import org.apache.polaris.service.Profiles; import org.apache.polaris.service.admin.PolarisAuthzTestBase; import org.junit.jupiter.api.DynamicNode; import org.junit.jupiter.api.TestFactory; @QuarkusTest -@TestProfile(PolarisAuthzTestBase.Profile.class) +@TestProfile(Profiles.PolarisAuthzBaseProfile.class) public class PolarisGenericTableCatalogHandlerAuthzTest extends PolarisAuthzTestBase { @Inject GenericTableCatalogHandlerFactory genericTableCatalogHandlerFactory; diff --git a/runtime/service/src/test/java/org/apache/polaris/service/catalog/generic/PolarisGenericTableCatalogNoSqlInMemTest.java b/runtime/service/src/test/java/org/apache/polaris/service/catalog/generic/PolarisGenericTableCatalogNoSqlInMemTest.java index a7caa216f4a..ecc6b8a717d 100644 --- a/runtime/service/src/test/java/org/apache/polaris/service/catalog/generic/PolarisGenericTableCatalogNoSqlInMemTest.java +++ b/runtime/service/src/test/java/org/apache/polaris/service/catalog/generic/PolarisGenericTableCatalogNoSqlInMemTest.java @@ -24,7 +24,7 @@ import java.util.List; import org.apache.polaris.core.persistence.MetaStoreManagerFactory; import org.apache.polaris.core.persistence.bootstrap.RootCredentialsSet; -import org.apache.polaris.service.catalog.Profiles; +import org.apache.polaris.service.Profiles; @QuarkusTest @TestProfile(Profiles.DefaultNoSqlProfile.class) diff --git a/runtime/service/src/test/java/org/apache/polaris/service/catalog/generic/PolarisGenericTableCatalogRelationalTest.java b/runtime/service/src/test/java/org/apache/polaris/service/catalog/generic/PolarisGenericTableCatalogRelationalTest.java index 54aef1843a9..e69b9f125f4 100644 --- a/runtime/service/src/test/java/org/apache/polaris/service/catalog/generic/PolarisGenericTableCatalogRelationalTest.java +++ b/runtime/service/src/test/java/org/apache/polaris/service/catalog/generic/PolarisGenericTableCatalogRelationalTest.java @@ -20,7 +20,7 @@ import io.quarkus.test.junit.QuarkusTest; import io.quarkus.test.junit.TestProfile; -import org.apache.polaris.service.catalog.Profiles; +import org.apache.polaris.service.Profiles; @QuarkusTest @TestProfile(Profiles.DefaultProfile.class) diff --git a/runtime/service/src/test/java/org/apache/polaris/service/catalog/iceberg/AbstractLocalIcebergCatalogTest.java b/runtime/service/src/test/java/org/apache/polaris/service/catalog/iceberg/AbstractLocalIcebergCatalogTest.java index 70f59eec0ce..a7f36a943eb 100644 --- a/runtime/service/src/test/java/org/apache/polaris/service/catalog/iceberg/AbstractLocalIcebergCatalogTest.java +++ b/runtime/service/src/test/java/org/apache/polaris/service/catalog/iceberg/AbstractLocalIcebergCatalogTest.java @@ -135,7 +135,6 @@ import org.apache.polaris.core.storage.cache.StorageCredentialCache; import org.apache.polaris.service.admin.PolarisAdminService; import org.apache.polaris.service.catalog.PolarisPassthroughResolutionView; -import org.apache.polaris.service.catalog.Profiles; import org.apache.polaris.service.catalog.io.ExceptionMappingFileIO; import org.apache.polaris.service.catalog.io.FileIOFactory; import org.apache.polaris.service.catalog.io.MeasuredFileIOFactory; @@ -201,20 +200,6 @@ public abstract class AbstractLocalIcebergCatalogTest extends CatalogTests getConfigOverrides() { - return ImmutableMap.builder() - .putAll(super.getConfigOverrides()) - .put("polaris.features.\"ALLOW_TABLE_LOCATION_OVERLAP\"", "true") - .put("polaris.features.\"LIST_PAGINATION_ENABLED\"", "true") - .put("polaris.behavior-changes.\"ALLOW_NAMESPACE_CUSTOM_LOCATION\"", "true") - .put("polaris.test.rootAugmentor.enabled", "true") - .put("polaris.event-listener.types", "test") - .build(); - } - } - private static final String VIEW_QUERY = "select * from ns1.layer1_table"; public static final String CATALOG_NAME = "polaris-catalog"; diff --git a/runtime/service/src/test/java/org/apache/polaris/service/catalog/iceberg/AbstractLocalIcebergCatalogViewTest.java b/runtime/service/src/test/java/org/apache/polaris/service/catalog/iceberg/AbstractLocalIcebergCatalogViewTest.java index aef84fc8f95..15c417d33a9 100644 --- a/runtime/service/src/test/java/org/apache/polaris/service/catalog/iceberg/AbstractLocalIcebergCatalogViewTest.java +++ b/runtime/service/src/test/java/org/apache/polaris/service/catalog/iceberg/AbstractLocalIcebergCatalogViewTest.java @@ -55,7 +55,6 @@ import org.apache.polaris.core.storage.cache.StorageCredentialCache; import org.apache.polaris.service.admin.PolarisAdminService; import org.apache.polaris.service.catalog.PolarisPassthroughResolutionView; -import org.apache.polaris.service.catalog.Profiles; import org.apache.polaris.service.catalog.io.FileIOFactory; import org.apache.polaris.service.catalog.io.StorageAccessConfigProvider; import org.apache.polaris.service.config.ReservedProperties; @@ -85,19 +84,6 @@ public abstract class AbstractLocalIcebergCatalogViewTest Assumptions.setPreferredAssumptionException(PreferredAssumptionException.JUNIT5); } - public static class Profile extends Profiles.DefaultProfile { - @Override - public Map getConfigOverrides() { - return ImmutableMap.builder() - .putAll(super.getConfigOverrides()) - .put("polaris.features.\"ALLOW_WILDCARD_LOCATION\"", "true") - .put("polaris.features.\"SKIP_CREDENTIAL_SUBSCOPING_INDIRECTION\"", "true") - .put("polaris.features.\"LIST_PAGINATION_ENABLED\"", "true") - .put("polaris.event-listener.types", "test") - .build(); - } - } - public static final String CATALOG_NAME = "polaris-catalog"; public static Map VIEW_PREFIXES = diff --git a/runtime/service/src/test/java/org/apache/polaris/service/catalog/iceberg/IcebergCatalogHandlerAuthzTest.java b/runtime/service/src/test/java/org/apache/polaris/service/catalog/iceberg/IcebergCatalogHandlerAuthzTest.java index b0caedce4ba..da406c97fea 100644 --- a/runtime/service/src/test/java/org/apache/polaris/service/catalog/iceberg/IcebergCatalogHandlerAuthzTest.java +++ b/runtime/service/src/test/java/org/apache/polaris/service/catalog/iceberg/IcebergCatalogHandlerAuthzTest.java @@ -20,8 +20,8 @@ import io.quarkus.test.junit.QuarkusTest; import io.quarkus.test.junit.TestProfile; -import org.apache.polaris.service.admin.PolarisAuthzTestBase; +import org.apache.polaris.service.Profiles; @QuarkusTest -@TestProfile(PolarisAuthzTestBase.Profile.class) +@TestProfile(Profiles.PolarisAuthzBaseProfile.class) public class IcebergCatalogHandlerAuthzTest extends AbstractIcebergCatalogHandlerAuthzTest {} diff --git a/runtime/service/src/test/java/org/apache/polaris/service/catalog/iceberg/IcebergCatalogHandlerFineGrainedDisabledTest.java b/runtime/service/src/test/java/org/apache/polaris/service/catalog/iceberg/IcebergCatalogHandlerFineGrainedDisabledTest.java index 6560184fe6f..e9955c07960 100644 --- a/runtime/service/src/test/java/org/apache/polaris/service/catalog/iceberg/IcebergCatalogHandlerFineGrainedDisabledTest.java +++ b/runtime/service/src/test/java/org/apache/polaris/service/catalog/iceberg/IcebergCatalogHandlerFineGrainedDisabledTest.java @@ -31,6 +31,7 @@ import org.apache.iceberg.rest.requests.UpdateTableRequest; import org.apache.polaris.core.auth.PolarisPrincipal; import org.apache.polaris.core.entity.PolarisPrivilege; +import org.apache.polaris.service.Profiles; import org.apache.polaris.service.admin.PolarisAuthzTestBase; import org.junit.jupiter.api.DynamicNode; import org.junit.jupiter.api.TestFactory; @@ -50,7 +51,7 @@ private IcebergCatalogHandler newHandler() { return icebergCatalogHandlerFactory.createHandler(CATALOG_NAME, authenticatedPrincipal); } - public static class Profile extends PolarisAuthzTestBase.Profile { + public static class Profile extends Profiles.PolarisAuthzBaseProfile { @Override public Map getConfigOverrides() { return ImmutableMap.builder() diff --git a/runtime/service/src/test/java/org/apache/polaris/service/catalog/iceberg/IcebergCatalogHandlerNoSqlAuthzTest.java b/runtime/service/src/test/java/org/apache/polaris/service/catalog/iceberg/IcebergCatalogHandlerNoSqlAuthzTest.java index aad67ce272d..afead49f5eb 100644 --- a/runtime/service/src/test/java/org/apache/polaris/service/catalog/iceberg/IcebergCatalogHandlerNoSqlAuthzTest.java +++ b/runtime/service/src/test/java/org/apache/polaris/service/catalog/iceberg/IcebergCatalogHandlerNoSqlAuthzTest.java @@ -18,32 +18,18 @@ */ package org.apache.polaris.service.catalog.iceberg; -import static org.apache.polaris.service.catalog.Profiles.NOSQL_IN_MEM; - -import com.google.common.collect.ImmutableMap; import io.quarkus.test.junit.QuarkusTest; import io.quarkus.test.junit.TestProfile; import jakarta.inject.Inject; import java.util.List; -import java.util.Map; import org.apache.polaris.core.persistence.MetaStoreManagerFactory; import org.apache.polaris.core.persistence.bootstrap.RootCredentialsSet; -import org.apache.polaris.service.admin.PolarisAuthzTestBase; +import org.apache.polaris.service.Profiles; @QuarkusTest -@TestProfile(IcebergCatalogHandlerNoSqlAuthzTest.Profile.class) +@TestProfile(Profiles.IcebergCatalogHandlerNoSqlAuthzProfile.class) public class IcebergCatalogHandlerNoSqlAuthzTest extends AbstractIcebergCatalogHandlerAuthzTest { - public static class Profile extends PolarisAuthzTestBase.Profile { - @Override - public Map getConfigOverrides() { - return ImmutableMap.builder() - .putAll(super.getConfigOverrides()) - .putAll(NOSQL_IN_MEM) - .build(); - } - } - @Inject MetaStoreManagerFactory metaStoreManagerFactory; @Override diff --git a/runtime/service/src/test/java/org/apache/polaris/service/catalog/iceberg/LocalIcebergCatalogNoSqlInMemTest.java b/runtime/service/src/test/java/org/apache/polaris/service/catalog/iceberg/LocalIcebergCatalogNoSqlInMemTest.java index 1c0c0278bb1..c0c8473fe6b 100644 --- a/runtime/service/src/test/java/org/apache/polaris/service/catalog/iceberg/LocalIcebergCatalogNoSqlInMemTest.java +++ b/runtime/service/src/test/java/org/apache/polaris/service/catalog/iceberg/LocalIcebergCatalogNoSqlInMemTest.java @@ -18,29 +18,15 @@ */ package org.apache.polaris.service.catalog.iceberg; -import static org.apache.polaris.service.catalog.Profiles.NOSQL_IN_MEM; - -import com.google.common.collect.ImmutableMap; import io.quarkus.test.junit.QuarkusTest; import io.quarkus.test.junit.TestProfile; import java.util.List; -import java.util.Map; import org.apache.polaris.core.persistence.bootstrap.RootCredentialsSet; +import org.apache.polaris.service.Profiles; @QuarkusTest -@TestProfile(LocalIcebergCatalogNoSqlInMemTest.Profile.class) +@TestProfile(Profiles.NoSqlIcebergCatalogProfile.class) public class LocalIcebergCatalogNoSqlInMemTest extends AbstractLocalIcebergCatalogTest { - - public static class Profile extends AbstractLocalIcebergCatalogTest.Profile { - @Override - public Map getConfigOverrides() { - return ImmutableMap.builder() - .putAll(super.getConfigOverrides()) - .putAll(NOSQL_IN_MEM) - .build(); - } - } - @Override protected void bootstrapRealm(String realmName) { metaStoreManagerFactory diff --git a/runtime/service/src/test/java/org/apache/polaris/service/catalog/iceberg/LocalIcebergCatalogRelationalTest.java b/runtime/service/src/test/java/org/apache/polaris/service/catalog/iceberg/LocalIcebergCatalogRelationalTest.java index 26c2d86ce77..063153b606f 100644 --- a/runtime/service/src/test/java/org/apache/polaris/service/catalog/iceberg/LocalIcebergCatalogRelationalTest.java +++ b/runtime/service/src/test/java/org/apache/polaris/service/catalog/iceberg/LocalIcebergCatalogRelationalTest.java @@ -20,7 +20,8 @@ import io.quarkus.test.junit.QuarkusTest; import io.quarkus.test.junit.TestProfile; +import org.apache.polaris.service.Profiles; @QuarkusTest -@TestProfile(AbstractLocalIcebergCatalogTest.Profile.class) +@TestProfile(Profiles.DefaultIcebergCatalogProfile.class) public class LocalIcebergCatalogRelationalTest extends AbstractLocalIcebergCatalogTest {} diff --git a/runtime/service/src/test/java/org/apache/polaris/service/catalog/iceberg/LocalIcebergViewCatalogNoSqlInMemTest.java b/runtime/service/src/test/java/org/apache/polaris/service/catalog/iceberg/LocalIcebergViewCatalogNoSqlInMemTest.java index 623f68db1d5..658050ca496 100644 --- a/runtime/service/src/test/java/org/apache/polaris/service/catalog/iceberg/LocalIcebergViewCatalogNoSqlInMemTest.java +++ b/runtime/service/src/test/java/org/apache/polaris/service/catalog/iceberg/LocalIcebergViewCatalogNoSqlInMemTest.java @@ -18,33 +18,20 @@ */ package org.apache.polaris.service.catalog.iceberg; -import static org.apache.polaris.service.catalog.Profiles.NOSQL_IN_MEM; - -import com.google.common.collect.ImmutableMap; import io.quarkus.test.junit.QuarkusTest; import io.quarkus.test.junit.TestProfile; import jakarta.inject.Inject; import java.util.List; -import java.util.Map; import org.apache.polaris.core.persistence.MetaStoreManagerFactory; import org.apache.polaris.core.persistence.bootstrap.RootCredentialsSet; +import org.apache.polaris.service.Profiles; @QuarkusTest -@TestProfile(LocalIcebergViewCatalogNoSqlInMemTest.Profile.class) +@TestProfile(Profiles.NoSqlIcebergCatalogProfile.class) public class LocalIcebergViewCatalogNoSqlInMemTest extends AbstractLocalIcebergCatalogViewTest { @Inject MetaStoreManagerFactory metaStoreManagerFactory; - public static class Profile extends AbstractLocalIcebergCatalogViewTest.Profile { - @Override - public Map getConfigOverrides() { - return ImmutableMap.builder() - .putAll(super.getConfigOverrides()) - .putAll(NOSQL_IN_MEM) - .build(); - } - } - @Override protected void bootstrapRealm(String realmName) { metaStoreManagerFactory diff --git a/runtime/service/src/test/java/org/apache/polaris/service/catalog/iceberg/LocalIcebergViewCatalogRelationalTest.java b/runtime/service/src/test/java/org/apache/polaris/service/catalog/iceberg/LocalIcebergViewCatalogRelationalTest.java index f8c34b103f1..2ce70c9db16 100644 --- a/runtime/service/src/test/java/org/apache/polaris/service/catalog/iceberg/LocalIcebergViewCatalogRelationalTest.java +++ b/runtime/service/src/test/java/org/apache/polaris/service/catalog/iceberg/LocalIcebergViewCatalogRelationalTest.java @@ -20,7 +20,8 @@ import io.quarkus.test.junit.QuarkusTest; import io.quarkus.test.junit.TestProfile; +import org.apache.polaris.service.Profiles; @QuarkusTest -@TestProfile(AbstractLocalIcebergCatalogViewTest.Profile.class) +@TestProfile(Profiles.DefaultIcebergCatalogProfile.class) public class LocalIcebergViewCatalogRelationalTest extends AbstractLocalIcebergCatalogViewTest {} diff --git a/runtime/service/src/test/java/org/apache/polaris/service/catalog/policy/PolicyCatalogHandlerAuthzTest.java b/runtime/service/src/test/java/org/apache/polaris/service/catalog/policy/PolicyCatalogHandlerAuthzTest.java index e8d186900b7..5fd566febda 100644 --- a/runtime/service/src/test/java/org/apache/polaris/service/catalog/policy/PolicyCatalogHandlerAuthzTest.java +++ b/runtime/service/src/test/java/org/apache/polaris/service/catalog/policy/PolicyCatalogHandlerAuthzTest.java @@ -28,6 +28,7 @@ import org.apache.polaris.core.catalog.PolarisCatalogHelpers; import org.apache.polaris.core.entity.PolarisPrivilege; import org.apache.polaris.core.policy.PredefinedPolicyTypes; +import org.apache.polaris.service.Profiles; import org.apache.polaris.service.admin.PolarisAuthzTestBase; import org.apache.polaris.service.types.AttachPolicyRequest; import org.apache.polaris.service.types.CreatePolicyRequest; @@ -39,7 +40,7 @@ import org.junit.jupiter.api.TestFactory; @QuarkusTest -@TestProfile(PolarisAuthzTestBase.Profile.class) +@TestProfile(Profiles.PolarisAuthzBaseProfile.class) public class PolicyCatalogHandlerAuthzTest extends PolarisAuthzTestBase { @Inject PolicyCatalogHandlerFactory policyCatalogHandlerFactory; diff --git a/runtime/service/src/test/java/org/apache/polaris/service/catalog/policy/PolicyCatalogNoSqlInMemTest.java b/runtime/service/src/test/java/org/apache/polaris/service/catalog/policy/PolicyCatalogNoSqlInMemTest.java index 521769e4bfa..e5c9d67b3e3 100644 --- a/runtime/service/src/test/java/org/apache/polaris/service/catalog/policy/PolicyCatalogNoSqlInMemTest.java +++ b/runtime/service/src/test/java/org/apache/polaris/service/catalog/policy/PolicyCatalogNoSqlInMemTest.java @@ -24,7 +24,7 @@ import java.util.List; import org.apache.polaris.core.persistence.MetaStoreManagerFactory; import org.apache.polaris.core.persistence.bootstrap.RootCredentialsSet; -import org.apache.polaris.service.catalog.Profiles; +import org.apache.polaris.service.Profiles; @QuarkusTest @TestProfile(Profiles.DefaultNoSqlProfile.class) diff --git a/runtime/service/src/test/java/org/apache/polaris/service/catalog/policy/PolicyCatalogRelationalTest.java b/runtime/service/src/test/java/org/apache/polaris/service/catalog/policy/PolicyCatalogRelationalTest.java index fd812262b9a..35b5361e2cc 100644 --- a/runtime/service/src/test/java/org/apache/polaris/service/catalog/policy/PolicyCatalogRelationalTest.java +++ b/runtime/service/src/test/java/org/apache/polaris/service/catalog/policy/PolicyCatalogRelationalTest.java @@ -20,7 +20,7 @@ import io.quarkus.test.junit.QuarkusTest; import io.quarkus.test.junit.TestProfile; -import org.apache.polaris.service.catalog.Profiles; +import org.apache.polaris.service.Profiles; @QuarkusTest @TestProfile(Profiles.DefaultProfile.class) diff --git a/runtime/service/src/test/java/org/apache/polaris/service/distcache/TestPersistenceDistCacheInvalidationsIntegration.java b/runtime/service/src/test/java/org/apache/polaris/service/distcache/TestPersistenceDistCacheInvalidationsIntegration.java index 4ba068027e8..c46052af054 100644 --- a/runtime/service/src/test/java/org/apache/polaris/service/distcache/TestPersistenceDistCacheInvalidationsIntegration.java +++ b/runtime/service/src/test/java/org/apache/polaris/service/distcache/TestPersistenceDistCacheInvalidationsIntegration.java @@ -49,7 +49,7 @@ import org.apache.polaris.persistence.nosql.api.cache.CacheInvalidations; import org.apache.polaris.persistence.nosql.api.cache.CacheInvalidations.CacheInvalidation; import org.apache.polaris.persistence.nosql.api.obj.SimpleTestObj; -import org.apache.polaris.service.catalog.iceberg.AbstractLocalIcebergCatalogTest; +import org.apache.polaris.service.Profiles; import org.assertj.core.api.SoftAssertions; import org.eclipse.microprofile.config.ConfigProvider; import org.junit.jupiter.api.AfterEach; @@ -69,7 +69,7 @@ @SuppressWarnings("CdiInjectionPointsInspection") public class TestPersistenceDistCacheInvalidationsIntegration { - public static class Profile extends AbstractLocalIcebergCatalogTest.Profile { + public static class Profile extends Profiles.NoSqlIcebergCatalogProfile { @Override public Map getConfigOverrides() { return ImmutableMap.builder() @@ -77,9 +77,6 @@ public Map getConfigOverrides() { .put("quarkus.management.port", "0") .put("quarkus.management.host", "127.0.0.1") .put("quarkus.management.enabled", "true") - .put("polaris.persistence.type", "nosql") - .put("polaris.persistence.auto-bootstrap-types", "nosql") - .put("polaris.persistence.nosql.backend", "InMemory") .put( "polaris.persistence.distributed-cache-invalidations.valid-tokens", "token1," + TOKEN) .put( diff --git a/runtime/service/src/test/java/org/apache/polaris/service/events/PolarisEventListenerOldConfigurationTest.java b/runtime/service/src/test/java/org/apache/polaris/service/events/PolarisEventListenerOldConfigurationTest.java index 5f98d2c2ff8..a95d39603ba 100644 --- a/runtime/service/src/test/java/org/apache/polaris/service/events/PolarisEventListenerOldConfigurationTest.java +++ b/runtime/service/src/test/java/org/apache/polaris/service/events/PolarisEventListenerOldConfigurationTest.java @@ -22,28 +22,17 @@ import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.assertTrue; -import com.google.common.collect.ImmutableMap; import io.quarkus.test.junit.QuarkusTest; -import io.quarkus.test.junit.QuarkusTestProfile; import io.quarkus.test.junit.TestProfile; import jakarta.inject.Inject; -import java.util.Map; import java.util.Set; +import org.apache.polaris.service.Profiles; import org.junit.jupiter.api.Test; @QuarkusTest -@TestProfile(PolarisEventListenerOldConfigurationTest.OldConfigEventListenerProfile.class) +@TestProfile(Profiles.DefaultProfile.class) public class PolarisEventListenerOldConfigurationTest { - public static class OldConfigEventListenerProfile implements QuarkusTestProfile { - @Override - public Map getConfigOverrides() { - return ImmutableMap.builder() - .put("polaris.event-listener.type", "test") - .build(); - } - } - @Inject PolarisEventListenerConfiguration polarisEventListenerConfiguration; @SuppressWarnings("removal") diff --git a/runtime/service/src/test/java/org/apache/polaris/service/events/PolarisEventListenersTest.java b/runtime/service/src/test/java/org/apache/polaris/service/events/PolarisEventListenersTest.java index 39553e9f8b6..f058ba2bf09 100644 --- a/runtime/service/src/test/java/org/apache/polaris/service/events/PolarisEventListenersTest.java +++ b/runtime/service/src/test/java/org/apache/polaris/service/events/PolarisEventListenersTest.java @@ -24,9 +24,7 @@ import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.assertTrue; -import com.google.common.collect.ImmutableMap; import io.quarkus.test.junit.QuarkusTest; -import io.quarkus.test.junit.QuarkusTestProfile; import io.quarkus.test.junit.TestProfile; import io.smallrye.common.annotation.Identifier; import io.vertx.core.Context; @@ -34,18 +32,18 @@ import jakarta.inject.Singleton; import java.util.HashSet; import java.util.List; -import java.util.Map; import java.util.Set; import java.util.concurrent.CopyOnWriteArrayList; import java.util.concurrent.atomic.AtomicInteger; import java.util.function.Predicate; import java.util.stream.Collectors; import java.util.stream.Stream; +import org.apache.polaris.service.Profiles; import org.apache.polaris.service.events.listeners.PolarisEventListener; import org.junit.jupiter.api.Test; @QuarkusTest -@TestProfile(PolarisEventListenersTest.PolarisEventListenersTestProfile.class) +@TestProfile(Profiles.PolarisEventListenersTestProfile.class) public class PolarisEventListenersTest { static final Set CATALOG_EVENT_TYPES = PolarisEventType.typesOfCategory(PolarisEventType.Category.CATALOG); @@ -127,32 +125,6 @@ public static class ConsumeCatalogAndNotificationEventsListener extends Filterin } } - public static class PolarisEventListenersTestProfile implements QuarkusTestProfile { - @Override - public Map getConfigOverrides() { - return ImmutableMap.builder() - .put( - "polaris.event-listener.types", - "after-send-listener,before-send-listener,consume-all-listener,consume-all-listener-2,consume-only-catalog-listener,consume-catalog-and-after-notification-listener") - .put( - "polaris.event-listener.after-send-listener.enabled-event-types", - "AFTER_SEND_NOTIFICATION") - .put( - "polaris.event-listener.before-send-listener.enabled-event-types", - "BEFORE_SEND_NOTIFICATION") - .put( - "polaris.event-listener.consume-only-catalog-listener.enabled-event-categories", - "CATALOG") - .put( - "polaris.event-listener.consume-catalog-and-after-notification-listener.enabled-event-categories", - "CATALOG") - .put( - "polaris.event-listener.consume-catalog-and-after-notification-listener.enabled-event-types", - "AFTER_SEND_NOTIFICATION") - .build(); - } - } - @Inject PolarisEventDispatcher eventDispatcher; @Inject diff --git a/runtime/service/src/test/java/org/apache/polaris/service/events/listeners/inmemory/InMemoryBufferEventListenerBufferSizeTest.java b/runtime/service/src/test/java/org/apache/polaris/service/events/listeners/inmemory/InMemoryBufferEventListenerBufferSizeTest.java index ca6bcbdd84a..ad45042be00 100644 --- a/runtime/service/src/test/java/org/apache/polaris/service/events/listeners/inmemory/InMemoryBufferEventListenerBufferSizeTest.java +++ b/runtime/service/src/test/java/org/apache/polaris/service/events/listeners/inmemory/InMemoryBufferEventListenerBufferSizeTest.java @@ -25,35 +25,21 @@ import static org.mockito.Mockito.doReturn; import static org.mockito.Mockito.doThrow; -import com.google.common.collect.ImmutableMap; import io.quarkus.test.junit.QuarkusTest; -import io.quarkus.test.junit.QuarkusTestProfile; import io.quarkus.test.junit.TestProfile; import io.smallrye.mutiny.subscription.BackPressureFailure; import java.time.Duration; -import java.util.Map; import org.apache.polaris.core.persistence.PolarisMetaStoreManager; +import org.apache.polaris.service.Profiles; import org.junit.jupiter.api.Test; import org.junit.jupiter.api.TestInstance; import org.mockito.Mockito; @QuarkusTest @TestInstance(TestInstance.Lifecycle.PER_CLASS) -@TestProfile(InMemoryBufferEventListenerBufferSizeTest.Profile.class) +@TestProfile(Profiles.InMemoryBufferEventListenerBufferSizeProfile.class) class InMemoryBufferEventListenerBufferSizeTest extends InMemoryBufferEventListenerTestBase { - public static class Profile implements QuarkusTestProfile { - - @Override - public Map getConfigOverrides() { - return ImmutableMap.builder() - .putAll(BASE_CONFIG) - .put("polaris.event-listener.persistence-in-memory-buffer.buffer-time", "60s") - .put("polaris.event-listener.persistence-in-memory-buffer.max-buffer-size", "10") - .build(); - } - } - @Test void testFlushOnSize() { sendAsync("test1", 10); diff --git a/runtime/service/src/test/java/org/apache/polaris/service/events/listeners/inmemory/InMemoryBufferEventListenerBufferTimeTest.java b/runtime/service/src/test/java/org/apache/polaris/service/events/listeners/inmemory/InMemoryBufferEventListenerBufferTimeTest.java index 3a6c7210e33..83681f50f25 100644 --- a/runtime/service/src/test/java/org/apache/polaris/service/events/listeners/inmemory/InMemoryBufferEventListenerBufferTimeTest.java +++ b/runtime/service/src/test/java/org/apache/polaris/service/events/listeners/inmemory/InMemoryBufferEventListenerBufferTimeTest.java @@ -19,31 +19,17 @@ package org.apache.polaris.service.events.listeners.inmemory; -import com.google.common.collect.ImmutableMap; import io.quarkus.test.junit.QuarkusTest; -import io.quarkus.test.junit.QuarkusTestProfile; import io.quarkus.test.junit.TestProfile; -import java.util.Map; +import org.apache.polaris.service.Profiles; import org.junit.jupiter.api.Test; import org.junit.jupiter.api.TestInstance; @QuarkusTest @TestInstance(TestInstance.Lifecycle.PER_CLASS) -@TestProfile(InMemoryBufferEventListenerBufferTimeTest.Profile.class) +@TestProfile(Profiles.InMemoryBufferEventListenerBufferTimeProfile.class) class InMemoryBufferEventListenerBufferTimeTest extends InMemoryBufferEventListenerTestBase { - public static class Profile implements QuarkusTestProfile { - - @Override - public Map getConfigOverrides() { - return ImmutableMap.builder() - .putAll(BASE_CONFIG) - .put("polaris.event-listener.persistence-in-memory-buffer.buffer-time", "100ms") - .put("polaris.event-listener.persistence-in-memory-buffer.max-buffer-size", "1000") - .build(); - } - } - @Test void testFlushOnTimeout() { sendAsync("test1", 5); diff --git a/runtime/service/src/test/java/org/apache/polaris/service/events/listeners/inmemory/InMemoryBufferEventListenerIntegrationTest.java b/runtime/service/src/test/java/org/apache/polaris/service/events/listeners/inmemory/InMemoryBufferEventListenerIntegrationTest.java index bc90a47c6f6..735ba69f1f3 100644 --- a/runtime/service/src/test/java/org/apache/polaris/service/events/listeners/inmemory/InMemoryBufferEventListenerIntegrationTest.java +++ b/runtime/service/src/test/java/org/apache/polaris/service/events/listeners/inmemory/InMemoryBufferEventListenerIntegrationTest.java @@ -27,7 +27,6 @@ import com.google.common.collect.ImmutableList; import com.google.common.collect.ImmutableMap; import io.quarkus.test.junit.QuarkusTest; -import io.quarkus.test.junit.QuarkusTestProfile; import io.quarkus.test.junit.TestProfile; import jakarta.enterprise.inject.Instance; import jakarta.inject.Inject; @@ -41,7 +40,6 @@ import java.sql.Statement; import java.time.Duration; import java.util.List; -import java.util.Map; import javax.sql.DataSource; import org.apache.iceberg.PartitionSpec; import org.apache.iceberg.Schema; @@ -58,6 +56,7 @@ import org.apache.polaris.core.admin.model.PolarisCatalog; import org.apache.polaris.core.admin.model.StorageConfigInfo; import org.apache.polaris.core.entity.PolarisEvent; +import org.apache.polaris.service.Profiles; import org.apache.polaris.service.it.env.ClientPrincipal; import org.apache.polaris.service.it.env.IntegrationTestsHelper; import org.apache.polaris.service.it.env.PolarisApiEndpoints; @@ -72,31 +71,10 @@ @QuarkusTest @TestInstance(TestInstance.Lifecycle.PER_CLASS) -@TestProfile(InMemoryBufferEventListenerIntegrationTest.Profile.class) +@TestProfile(Profiles.InMemoryBufferEventListenerIntegrationProfile.class) @ExtendWith(PolarisIntegrationTestExtension.class) class InMemoryBufferEventListenerIntegrationTest { - public static class Profile implements QuarkusTestProfile { - - @Override - public Map getConfigOverrides() { - return ImmutableMap.builder() - .put("polaris.persistence.type", "relational-jdbc") - .put("polaris.persistence.auto-bootstrap-types", "relational-jdbc") - .put("quarkus.datasource.db-kind", "h2") - .put("quarkus.otel.sdk.disabled", "false") - .put( - "quarkus.datasource.jdbc.url", - "jdbc:h2:mem:test;DB_CLOSE_DELAY=-1;MODE=PostgreSQL;DATABASE_TO_LOWER=TRUE") - .put("polaris.event-listener.type", "persistence-in-memory-buffer") - .put("polaris.event-listener.persistence-in-memory-buffer.buffer-time", "100ms") - .put("polaris.features.\"ALLOW_INSECURE_STORAGE_TYPES\"", "true") - .put("polaris.features.\"SUPPORTED_CATALOG_STORAGE_TYPES\"", "[\"FILE\",\"S3\"]") - .put("polaris.readiness.ignore-severe-issues", "true") - .build(); - } - } - private RestApi managementApi; private PolarisApiEndpoints endpoints; private PolarisClient client; diff --git a/runtime/service/src/test/java/org/apache/polaris/service/events/listeners/inmemory/InMemoryBufferEventListenerTestBase.java b/runtime/service/src/test/java/org/apache/polaris/service/events/listeners/inmemory/InMemoryBufferEventListenerTestBase.java index a1d92485196..7d943461b42 100644 --- a/runtime/service/src/test/java/org/apache/polaris/service/events/listeners/inmemory/InMemoryBufferEventListenerTestBase.java +++ b/runtime/service/src/test/java/org/apache/polaris/service/events/listeners/inmemory/InMemoryBufferEventListenerTestBase.java @@ -24,7 +24,6 @@ import static org.awaitility.Awaitility.await; import static org.mockito.Mockito.reset; -import com.google.common.collect.ImmutableMap; import io.netty.channel.EventLoopGroup; import io.quarkus.netty.MainEventLoopGroup; import io.quarkus.test.junit.mockito.InjectSpy; @@ -35,7 +34,6 @@ import java.sql.ResultSet; import java.sql.Statement; import java.time.Duration; -import java.util.Map; import java.util.UUID; import javax.sql.DataSource; import org.apache.polaris.core.entity.PolarisEvent; @@ -45,24 +43,6 @@ abstract class InMemoryBufferEventListenerTestBase { - static final Map BASE_CONFIG = - ImmutableMap.builder() - .put("polaris.realm-context.realms", "test1,test2") - .put("polaris.persistence.type", "relational-jdbc") - .put("polaris.persistence.auto-bootstrap-types", "relational-jdbc") - .put("quarkus.datasource.db-kind", "h2") - .put( - "quarkus.datasource.jdbc.url", - "jdbc:h2:mem:test;DB_CLOSE_DELAY=-1;MODE=PostgreSQL;DATABASE_TO_LOWER=TRUE") - .put("polaris.event-listener.type", "persistence-in-memory-buffer") - .put( - "quarkus.fault-tolerance.\"org.apache.polaris.service.events.listeners.inmemory.InMemoryBufferEventListener/flush\".retry.max-retries", - "1") - .put( - "quarkus.fault-tolerance.\"org.apache.polaris.service.events.listeners.inmemory.InMemoryBufferEventListener/flush\".retry.delay", - "10") - .build(); - @Inject @Identifier("persistence-in-memory-buffer") Instance producerInstance; diff --git a/runtime/service/src/test/java/org/apache/polaris/service/it/ApplicationIntegrationTest.java b/runtime/service/src/test/java/org/apache/polaris/service/it/ApplicationIntegrationTest.java index b0f27fba592..214276723bc 100644 --- a/runtime/service/src/test/java/org/apache/polaris/service/it/ApplicationIntegrationTest.java +++ b/runtime/service/src/test/java/org/apache/polaris/service/it/ApplicationIntegrationTest.java @@ -24,7 +24,6 @@ import com.auth0.jwt.JWT; import com.auth0.jwt.algorithms.Algorithm; import io.quarkus.test.junit.QuarkusTest; -import io.quarkus.test.junit.QuarkusTestProfile; import io.quarkus.test.junit.TestProfile; import java.io.IOException; import java.time.Instant; @@ -37,38 +36,15 @@ import org.apache.iceberg.rest.auth.AuthSession; import org.apache.iceberg.rest.auth.OAuth2Util; import org.apache.iceberg.rest.responses.OAuthTokenResponse; +import org.apache.polaris.service.Profiles; import org.apache.polaris.service.it.env.ClientCredentials; import org.apache.polaris.service.it.env.PolarisApiEndpoints; import org.apache.polaris.service.it.test.PolarisApplicationIntegrationTest; import org.junit.jupiter.api.Test; @QuarkusTest -@TestProfile(ApplicationIntegrationTest.Profile.class) +@TestProfile(Profiles.ApplicationIntegrationProfile.class) public class ApplicationIntegrationTest extends PolarisApplicationIntegrationTest { - - public static class Profile implements QuarkusTestProfile { - @Override - public Map getConfigOverrides() { - return Map.of( - "quarkus.http.limits.max-body-size", - "1000000", - "polaris.realm-context.realms", - "POLARIS,OTHER", - "polaris.features.\"ALLOW_OVERLAPPING_CATALOG_URLS\"", - "true", - "polaris.features.\"SKIP_CREDENTIAL_SUBSCOPING_INDIRECTION\"", - "true", - "polaris.features.\"ALLOW_SPECIFYING_FILE_IO_IMPL\"", - "true", - "polaris.features.\"ALLOW_INSECURE_STORAGE_TYPES\"", - "true", - "polaris.features.\"SUPPORTED_CATALOG_STORAGE_TYPES\"", - "[\"FILE\",\"S3\"]", - "polaris.readiness.ignore-severe-issues", - "true"); - } - } - @Test public void testIcebergRestApiRefreshExpiredToken( PolarisApiEndpoints endpoints, ClientCredentials clientCredentials) throws IOException { diff --git a/runtime/service/src/test/java/org/apache/polaris/service/it/ManagementServiceIntegrationTest.java b/runtime/service/src/test/java/org/apache/polaris/service/it/ManagementServiceIntegrationTest.java index 057e251d95b..8a66b5945e4 100644 --- a/runtime/service/src/test/java/org/apache/polaris/service/it/ManagementServiceIntegrationTest.java +++ b/runtime/service/src/test/java/org/apache/polaris/service/it/ManagementServiceIntegrationTest.java @@ -19,28 +19,10 @@ package org.apache.polaris.service.it; import io.quarkus.test.junit.QuarkusTest; -import io.quarkus.test.junit.QuarkusTestProfile; import io.quarkus.test.junit.TestProfile; -import java.util.Map; +import org.apache.polaris.service.Profiles; import org.apache.polaris.service.it.test.PolarisManagementServiceIntegrationTest; @QuarkusTest -@TestProfile(ManagementServiceIntegrationTest.Profile.class) -public class ManagementServiceIntegrationTest extends PolarisManagementServiceIntegrationTest { - - public static class Profile implements QuarkusTestProfile { - - @Override - public Map getConfigOverrides() { - return Map.of( - "polaris.features.\"ALLOW_OVERLAPPING_CATALOG_URLS\"", - "true", - "polaris.features.\"ENFORCE_PRINCIPAL_CREDENTIAL_ROTATION_REQUIRED_CHECKING\"", - "true", - "polaris.storage.gcp.token", - "token", - "polaris.storage.gcp.lifespan", - "PT1H"); - } - } -} +@TestProfile(Profiles.ManagementIntegrationProfile.class) +public class ManagementServiceIntegrationTest extends PolarisManagementServiceIntegrationTest {} diff --git a/runtime/service/src/test/java/org/apache/polaris/service/it/PolicyServiceIntegrationTest.java b/runtime/service/src/test/java/org/apache/polaris/service/it/PolicyServiceIntegrationTest.java index 280f381bbf1..46914160197 100644 --- a/runtime/service/src/test/java/org/apache/polaris/service/it/PolicyServiceIntegrationTest.java +++ b/runtime/service/src/test/java/org/apache/polaris/service/it/PolicyServiceIntegrationTest.java @@ -19,27 +19,10 @@ package org.apache.polaris.service.it; import io.quarkus.test.junit.QuarkusTest; -import io.quarkus.test.junit.QuarkusTestProfile; import io.quarkus.test.junit.TestProfile; -import java.util.Map; +import org.apache.polaris.service.Profiles; import org.apache.polaris.service.it.test.PolarisPolicyServiceIntegrationTest; @QuarkusTest -@TestProfile(PolicyServiceIntegrationTest.Profile.class) -public class PolicyServiceIntegrationTest extends PolarisPolicyServiceIntegrationTest { - - public static class Profile implements QuarkusTestProfile { - @Override - public Map getConfigOverrides() { - return Map.of( - "polaris.features.\"ALLOW_SPECIFYING_FILE_IO_IMPL\"", - "true", - "polaris.features.\"ALLOW_INSECURE_STORAGE_TYPES\"", - "true", - "polaris.features.\"SUPPORTED_CATALOG_STORAGE_TYPES\"", - "[\"FILE\",\"S3\"]", - "polaris.readiness.ignore-severe-issues", - "true"); - } - } -} +@TestProfile(Profiles.DefaultProfile.class) +public class PolicyServiceIntegrationTest extends PolarisPolicyServiceIntegrationTest {} diff --git a/runtime/service/src/test/java/org/apache/polaris/service/it/RestCatalogFileIntegrationTest.java b/runtime/service/src/test/java/org/apache/polaris/service/it/RestCatalogFileIntegrationTest.java index c08d81e2f66..f30740bf96b 100644 --- a/runtime/service/src/test/java/org/apache/polaris/service/it/RestCatalogFileIntegrationTest.java +++ b/runtime/service/src/test/java/org/apache/polaris/service/it/RestCatalogFileIntegrationTest.java @@ -19,39 +19,17 @@ package org.apache.polaris.service.it; import io.quarkus.test.junit.QuarkusTest; -import io.quarkus.test.junit.QuarkusTestProfile; import io.quarkus.test.junit.TestProfile; import io.smallrye.common.annotation.Identifier; import jakarta.inject.Inject; -import java.util.Map; +import org.apache.polaris.service.Profiles; import org.apache.polaris.service.it.test.PolarisRestCatalogFileIntegrationTest; import org.apache.polaris.service.task.TaskErrorHandler; import org.junit.jupiter.api.AfterEach; @QuarkusTest -@TestProfile(RestCatalogFileIntegrationTest.Profile.class) +@TestProfile(Profiles.RestCatalogFileIntegrationProfile.class) public class RestCatalogFileIntegrationTest extends PolarisRestCatalogFileIntegrationTest { - - public static class Profile implements QuarkusTestProfile { - - @Override - public Map getConfigOverrides() { - return Map.of( - "polaris.features.\"ALLOW_SPECIFYING_FILE_IO_IMPL\"", - "true", - "polaris.features.\"ALLOW_EXTERNAL_CATALOG_CREDENTIAL_VENDING\"", - "false", - "polaris.features.\"ALLOW_INSECURE_STORAGE_TYPES\"", - "true", - "polaris.features.\"SUPPORTED_CATALOG_STORAGE_TYPES\"", - "[\"FILE\",\"S3\"]", - "polaris.readiness.ignore-severe-issues", - "true", - "polaris.features.\"ALLOW_NAMESPACE_CUSTOM_LOCATION\"", - "true"); - } - } - @Inject @Identifier("task-error-handler") TaskErrorHandler taskErrorHandler; diff --git a/runtime/service/src/test/java/org/apache/polaris/service/it/RestCatalogViewFileIntegrationTest.java b/runtime/service/src/test/java/org/apache/polaris/service/it/RestCatalogViewFileIntegrationTest.java index 79114767a3f..204921a1beb 100644 --- a/runtime/service/src/test/java/org/apache/polaris/service/it/RestCatalogViewFileIntegrationTest.java +++ b/runtime/service/src/test/java/org/apache/polaris/service/it/RestCatalogViewFileIntegrationTest.java @@ -19,27 +19,11 @@ package org.apache.polaris.service.it; import io.quarkus.test.junit.QuarkusTest; -import io.quarkus.test.junit.QuarkusTestProfile; import io.quarkus.test.junit.TestProfile; -import java.util.Map; +import org.apache.polaris.service.Profiles; import org.apache.polaris.service.it.test.PolarisRestCatalogViewFileIntegrationTestBase; @QuarkusTest -@TestProfile(RestCatalogViewFileIntegrationTest.Profile.class) +@TestProfile(Profiles.RestCatalogViewFileIntegrationProfile.class) public class RestCatalogViewFileIntegrationTest - extends PolarisRestCatalogViewFileIntegrationTestBase { - - public static class Profile implements QuarkusTestProfile { - - @Override - public Map getConfigOverrides() { - return Map.of( - "polaris.features.\"ALLOW_INSECURE_STORAGE_TYPES\"", - "true", - "polaris.features.\"SUPPORTED_CATALOG_STORAGE_TYPES\"", - "[\"FILE\"]", - "polaris.readiness.ignore-severe-issues", - "true"); - } - } -} + extends PolarisRestCatalogViewFileIntegrationTestBase {} diff --git a/runtime/service/src/test/java/org/apache/polaris/service/metrics/RealmIdTagDisabledMetricsTest.java b/runtime/service/src/test/java/org/apache/polaris/service/metrics/RealmIdTagDisabledMetricsTest.java index 68fbc5180f1..e60c6b22b20 100644 --- a/runtime/service/src/test/java/org/apache/polaris/service/metrics/RealmIdTagDisabledMetricsTest.java +++ b/runtime/service/src/test/java/org/apache/polaris/service/metrics/RealmIdTagDisabledMetricsTest.java @@ -19,20 +19,9 @@ package org.apache.polaris.service.metrics; import io.quarkus.test.junit.QuarkusTest; -import io.quarkus.test.junit.QuarkusTestProfile; import io.quarkus.test.junit.TestProfile; -import java.util.Map; +import org.apache.polaris.service.Profiles; @QuarkusTest -@TestProfile(RealmIdTagDisabledMetricsTest.Profile.class) -public class RealmIdTagDisabledMetricsTest extends MetricsTestBase { - - public static class Profile implements QuarkusTestProfile { - - @Override - public Map getConfigOverrides() { - return Map.of( - "polaris.metrics.tags.environment", "prod", "polaris.realm-context.type", "test"); - } - } -} +@TestProfile(Profiles.TagsDisabledMetricsProfile.class) +public class RealmIdTagDisabledMetricsTest extends MetricsTestBase {} diff --git a/runtime/service/src/test/java/org/apache/polaris/service/metrics/RealmIdTagEnabledMetricsTest.java b/runtime/service/src/test/java/org/apache/polaris/service/metrics/RealmIdTagEnabledMetricsTest.java index 86ddf1fb99a..9b37b29f7a9 100644 --- a/runtime/service/src/test/java/org/apache/polaris/service/metrics/RealmIdTagEnabledMetricsTest.java +++ b/runtime/service/src/test/java/org/apache/polaris/service/metrics/RealmIdTagEnabledMetricsTest.java @@ -19,29 +19,9 @@ package org.apache.polaris.service.metrics; import io.quarkus.test.junit.QuarkusTest; -import io.quarkus.test.junit.QuarkusTestProfile; import io.quarkus.test.junit.TestProfile; -import java.util.Map; +import org.apache.polaris.service.Profiles; @QuarkusTest -@TestProfile(RealmIdTagEnabledMetricsTest.Profile.class) -public class RealmIdTagEnabledMetricsTest extends MetricsTestBase { - - public static class Profile implements QuarkusTestProfile { - - @Override - public Map getConfigOverrides() { - return Map.of( - "polaris.metrics.tags.environment", - "prod", - "polaris.realm-context.type", - "test", - "polaris.metrics.realm-id-tag.enable-in-api-metrics", - "true", - "polaris.metrics.realm-id-tag.enable-in-http-metrics", - "true", - "polaris.metrics.user-principal-tag.enable-in-api-metrics", - "false"); - } - } -} +@TestProfile(Profiles.RealmIdTagEnabledMetricsProfile.class) +public class RealmIdTagEnabledMetricsTest extends MetricsTestBase {} diff --git a/runtime/service/src/test/java/org/apache/polaris/service/metrics/UserPrincipalTagDisabledMetricsTest.java b/runtime/service/src/test/java/org/apache/polaris/service/metrics/UserPrincipalTagDisabledMetricsTest.java index 9f45bb51c48..119572cf539 100644 --- a/runtime/service/src/test/java/org/apache/polaris/service/metrics/UserPrincipalTagDisabledMetricsTest.java +++ b/runtime/service/src/test/java/org/apache/polaris/service/metrics/UserPrincipalTagDisabledMetricsTest.java @@ -19,20 +19,9 @@ package org.apache.polaris.service.metrics; import io.quarkus.test.junit.QuarkusTest; -import io.quarkus.test.junit.QuarkusTestProfile; import io.quarkus.test.junit.TestProfile; -import java.util.Map; +import org.apache.polaris.service.Profiles; @QuarkusTest -@TestProfile(UserPrincipalTagDisabledMetricsTest.Profile.class) -public class UserPrincipalTagDisabledMetricsTest extends MetricsTestBase { - - public static class Profile implements QuarkusTestProfile { - - @Override - public Map getConfigOverrides() { - return Map.of( - "polaris.metrics.tags.environment", "prod", "polaris.realm-context.type", "test"); - } - } -} +@TestProfile(Profiles.TagsDisabledMetricsProfile.class) +public class UserPrincipalTagDisabledMetricsTest extends MetricsTestBase {} diff --git a/runtime/service/src/test/java/org/apache/polaris/service/metrics/UserPrincipalTagEnabledMetricsTest.java b/runtime/service/src/test/java/org/apache/polaris/service/metrics/UserPrincipalTagEnabledMetricsTest.java index b7e706fea01..cd743a4b245 100644 --- a/runtime/service/src/test/java/org/apache/polaris/service/metrics/UserPrincipalTagEnabledMetricsTest.java +++ b/runtime/service/src/test/java/org/apache/polaris/service/metrics/UserPrincipalTagEnabledMetricsTest.java @@ -19,27 +19,9 @@ package org.apache.polaris.service.metrics; import io.quarkus.test.junit.QuarkusTest; -import io.quarkus.test.junit.QuarkusTestProfile; import io.quarkus.test.junit.TestProfile; -import java.util.Map; +import org.apache.polaris.service.Profiles; @QuarkusTest -@TestProfile(UserPrincipalTagEnabledMetricsTest.Profile.class) -public class UserPrincipalTagEnabledMetricsTest extends MetricsTestBase { - - public static class Profile implements QuarkusTestProfile { - - @Override - public Map getConfigOverrides() { - return Map.of( - "polaris.metrics.tags.environment", - "prod", - "polaris.metrics.user-principal-tag.enable-in-api-metrics", - "true", - "polaris.metrics.realm-id-tag.enable-in-api-metrics", - "false", - "polaris.metrics.realm-id-tag.enable-in-http-metrics", - "false"); - } - } -} +@TestProfile(Profiles.UserPrincipalTagEnabledMetricsProfile.class) +public class UserPrincipalTagEnabledMetricsTest extends MetricsTestBase {} diff --git a/runtime/spark-tests/build.gradle.kts b/runtime/spark-tests/build.gradle.kts index 089aedb1163..cefd1ade52d 100644 --- a/runtime/spark-tests/build.gradle.kts +++ b/runtime/spark-tests/build.gradle.kts @@ -17,13 +17,33 @@ * under the License. */ +import io.quarkus.gradle.tasks.QuarkusBuild +import org.gradle.api.attributes.java.TargetJvmVersion +import org.gradle.api.plugins.jvm.JvmTestSuite +import org.gradle.language.base.plugins.LifecycleBasePlugin + plugins { alias(libs.plugins.quarkus) id("org.kordamp.gradle.jandex") id("polaris-runtime") + id("polaris-server-test-runner") +} + +val intTestJvmVersion = 21 +val sparkStartupAction = sourceSets.create("sparkStartupAction") +val sparkStartupActionCompileOnly by configurations.getting +val sparkStartupActionImplementation by configurations.getting +val quarkusBuild = tasks.named("quarkusBuild") +val localPolarisServer = files(provider { quarkusBuild.get().fastJar.resolve("quarkus-run.jar") }) + +localPolarisServer.builtBy(quarkusBuild) + +configurations.named(sparkStartupAction.runtimeOnlyConfigurationName) { + extendsFrom(configurations.named(sparkStartupAction.implementationConfigurationName)) } dependencies { + polarisServer(localPolarisServer) // must be enforced to get a consistent and validated set of dependencies implementation(enforcedPlatform(libs.quarkus.bom)) { @@ -33,6 +53,7 @@ dependencies { } implementation(project(":polaris-runtime-service")) + runtimeOnly(project(":polaris-extensions-federation-hadoop")) runtimeOnly(project(":polaris-extensions-federation-hive")) { // Brings shaded parquet 1.10 which conflicts with Iceberg's parquet 1.16 in test code. exclude("org.apache.parquet", "parquet-hadoop-bundle") @@ -54,57 +75,185 @@ dependencies { // (the bundle fat-jar provided them); add it explicitly at the BOM-managed version. runtimeOnly("software.amazon.awssdk:s3-transfer-manager") - testImplementation(project(":polaris-tests")) - testImplementation(project(":polaris-rustfs-testcontainer")) - testImplementation(testFixtures(project(":polaris-runtime-service"))) - testImplementation(project(":polaris-runtime-test-common")) - - testImplementation(platform(libs.quarkus.bom)) - testImplementation("io.quarkus:quarkus-junit") - testImplementation("io.quarkus:quarkus-rest-client") - testImplementation("io.quarkus:quarkus-rest-client-jackson") - - testImplementation(platform(libs.awssdk.bom)) - testImplementation("software.amazon.awssdk:glue") - testImplementation("software.amazon.awssdk:kms") - testImplementation("software.amazon.awssdk:dynamodb") - - testImplementation(platform(libs.testcontainers.bom)) - testImplementation("org.testcontainers:testcontainers") - testImplementation(libs.s3mock.testcontainers) - - // Required for Spark integration tests - testImplementation(enforcedPlatform(libs.scala212.lang.library)) - testImplementation(enforcedPlatform(libs.scala212.lang.reflect)) - testImplementation(libs.javax.servlet.api) - testImplementation(libs.antlr4.runtime.spark35) + sparkStartupActionCompileOnly("org.apache.polaris.server-test-runner:polaris-server-test-runner") + sparkStartupActionImplementation(project(":polaris-rustfs-testcontainer")) } -tasks.named("intTest").configure { - if (System.getenv("AWS_REGION") == null) { - environment("AWS_REGION", "us-west-2") +@Suppress("UnstableApiUsage") +fun JvmTestSuite.configureSparkIntegrationDependencies() { + dependencies { + implementation(project(":polaris-tests")) + implementation(testFixtures(project(":polaris-runtime-service"))) + implementation(project(":polaris-runtime-test-common")) + + implementation(platform(libs.awssdk.bom)) + implementation("software.amazon.awssdk:glue") + implementation("software.amazon.awssdk:kms") + implementation("software.amazon.awssdk:dynamodb") + implementation("software.amazon.awssdk:url-connection-client") + + // Required for Spark integration tests + implementation(enforcedPlatform(libs.scala212.lang.library)) + implementation(enforcedPlatform(libs.scala212.lang.reflect)) + implementation(libs.javax.servlet.api) + implementation(libs.antlr4.runtime.spark35) } - // Note: the test secrets are referenced in - // org.apache.polaris.service.it.ServerManager - environment("POLARIS_BOOTSTRAP_CREDENTIALS", "POLARIS,test-admin,test-secret") +} + +fun Test.configureSparkIntegrationTestTask( + suiteName: String, + skipCredentialSubscoping: Boolean, + storageAccessKey: String? = null, + storageSecretKey: String? = null, + withHiveRustfsStartupAction: Boolean = false, +) { + environment("AWS_REGION", providers.environmentVariable("AWS_REGION").getOrElse("us-west-2")) jvmArgs("--add-exports", "java.base/sun.nio.ch=ALL-UNNAMED") // Need to allow a java security manager after Java 21, for Subject.getSubject to work // "getSubject is supported only if a security manager is allowed". systemProperty("java.security.manager", "allow") - // Same issue as above: allow a java security manager after Java 21 - // (this setting is for the application under test, while the setting above is for test code). - systemProperty("quarkus.test.arg-line", "-Djava.security.manager=allow") - val logsDir = project.layout.buildDirectory.get().asFile.resolve("logs") - // delete files from previous runs + + val logsDir = project.layout.buildDirectory.dir("logs/$suiteName") + val hiveRustfsProperties = project.layout.buildDirectory.file("$suiteName/hive-rustfs.properties") + doFirst { - // delete log files written by Polaris - logsDir.deleteRecursively() - // delete quarkus.log file (captured Polaris stdout/stderr) - project.layout.buildDirectory.get().asFile.resolve("quarkus.log").delete() + val logsDirFile = logsDir.get().asFile + logsDirFile.deleteRecursively() + logsDirFile.mkdirs() + } + + if (withHiveRustfsStartupAction) { + systemProperty( + "polaris.spark-tests.hive-rustfs.properties", + hiveRustfsProperties.get().asFile.absolutePath, + ) + } + + withPolarisServer(configurations.polarisServer) { + if (withHiveRustfsStartupAction) { + startupActionClasspath.from(sparkStartupAction.runtimeClasspath) + startupActionClass.set("org.apache.polaris.service.spark.it.SparkTestsStartupAction") + startupActionParameters.put( + "hiveRustfsProperties", + hiveRustfsProperties.get().asFile.absolutePath, + ) + } + + environment.put("AWS_REGION", providers.environmentVariable("AWS_REGION").orElse("us-west-2")) + environment.put( + "AWS_ACCESS_KEY_ID", + providers.environmentVariable("AWS_ACCESS_KEY_ID").orElse("ap1"), + ) + environment.put( + "AWS_SECRET_ACCESS_KEY", + providers.environmentVariable("AWS_SECRET_ACCESS_KEY").orElse("s3cr3t"), + ) + environment.put("AWS_EC2_METADATA_DISABLED", "true") + environment.put("POLARIS_BOOTSTRAP_CREDENTIALS", "POLARIS,test-admin,test-secret") + + systemProperties.putAll( + mapOf( + "quarkus.profile" to "it", + "java.security.manager" to "allow", + "quarkus.log.file.enabled" to "true", + "quarkus.log.file.path" to logsDir.get().asFile.resolve("polaris.log").absolutePath, + "polaris.features.\"SUPPORTED_CATALOG_STORAGE_TYPES\"" to + "[\"FILE\",\"S3\",\"GCS\",\"AZURE\"]", + "polaris.features.\"ALLOW_INSECURE_STORAGE_TYPES\"" to "true", + "polaris.features.\"DROP_WITH_PURGE_ENABLED\"" to "true", + "polaris.features.\"ALLOW_OVERLAPPING_CATALOG_URLS\"" to "true", + "polaris.features.\"SKIP_CREDENTIAL_SUBSCOPING_INDIRECTION\"" to + skipCredentialSubscoping.toString(), + "polaris.features.\"ENABLE_CATALOG_FEDERATION\"" to "true", + "polaris.features.\"SUPPORTED_CATALOG_CONNECTION_TYPES\"" to "[\"ICEBERG_REST\",\"HIVE\"]", + "polaris.features.\"SUPPORTED_EXTERNAL_CATALOG_AUTHENTICATION_TYPES\"" to + "[\"IMPLICIT\",\"OAUTH\"]", + "polaris.features.\"ENABLE_SUB_CATALOG_RBAC_FOR_FEDERATED_CATALOGS\"" to "true", + "polaris.features.\"ALLOW_DROPPING_NON_EMPTY_PASSTHROUGH_FACADE_CATALOG\"" to "true", + "polaris.features.\"ALLOW_EXTERNAL_CATALOG_CREDENTIAL_VENDING\"" to "true", + "polaris.features.\"ALLOW_FEDERATED_CATALOGS_CREDENTIAL_VENDING\"" to "true", + ) + ) + storageAccessKey?.let { systemProperties.put("polaris.storage.aws.access-key", it) } + storageSecretKey?.let { systemProperties.put("polaris.storage.aws.secret-key", it) } } - // This property is not honored in a per-profile application.properties file, - // so we need to set it here. - systemProperty("quarkus.log.file.path", logsDir.resolve("polaris.log").absolutePath) // For Spark integration tests addSparkJvmOptions() } + +@Suppress("UnstableApiUsage") +testing { + suites { + register("sparkIntTest") { + useJUnitJupiter() + configureSparkIntegrationDependencies() + + targets.configureEach { + testTask.configure { + configureSparkIntegrationTestTask( + suiteName = "sparkIntTest", + skipCredentialSubscoping = true, + ) + } + } + } + + register("catalogFederationIntTest") { + useJUnitJupiter() + configureSparkIntegrationDependencies() + + targets.configureEach { + testTask.configure { + configureSparkIntegrationTestTask( + suiteName = "catalogFederationIntTest", + skipCredentialSubscoping = false, + storageAccessKey = "test-ak-123-catalog-federation", + storageSecretKey = "test-sk-123-catalog-federation", + ) + } + } + } + + register("hiveFederationIntTest") { + useJUnitJupiter() + configureSparkIntegrationDependencies() + dependencies { implementation(project(":polaris-rustfs-testcontainer")) } + + targets.configureEach { + testTask.configure { + configureSparkIntegrationTestTask( + suiteName = "hiveFederationIntTest", + skipCredentialSubscoping = false, + storageAccessKey = "test-ak-123-hive-federation", + storageSecretKey = "test-sk-123-hive-federation", + withHiveRustfsStartupAction = true, + ) + } + } + } + } +} + +listOf( + "sparkIntTestCompileClasspath", + "sparkIntTestRuntimeClasspath", + "catalogFederationIntTestCompileClasspath", + "catalogFederationIntTestRuntimeClasspath", + "hiveFederationIntTestCompileClasspath", + "hiveFederationIntTestRuntimeClasspath", + "sparkStartupActionCompileClasspath", + "sparkStartupActionRuntimeClasspath", + ) + .forEach { + configurations.named(it).configure { + attributes.attribute(TargetJvmVersion.TARGET_JVM_VERSION_ATTRIBUTE, intTestJvmVersion) + } + } + +tasks.named("intTest") { + description = "Runs all Spark integration tests." + group = LifecycleBasePlugin.VERIFICATION_GROUP + dependsOn("sparkIntTest", "catalogFederationIntTest", "hiveFederationIntTest") + testClassesDirs = files() + classpath = files() +} diff --git a/runtime/spark-tests/src/catalogFederationIntTest/java/org/apache/polaris/service/spark/it/CatalogFederationIT.java b/runtime/spark-tests/src/catalogFederationIntTest/java/org/apache/polaris/service/spark/it/CatalogFederationIT.java new file mode 100644 index 00000000000..f7002ce08c9 --- /dev/null +++ b/runtime/spark-tests/src/catalogFederationIntTest/java/org/apache/polaris/service/spark/it/CatalogFederationIT.java @@ -0,0 +1,23 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.polaris.service.spark.it; + +import org.apache.polaris.service.it.test.CatalogFederationIntegrationTest; + +public class CatalogFederationIT extends CatalogFederationIntegrationTest {} diff --git a/runtime/spark-tests/src/catalogFederationIntTest/resources/META-INF/services/org.apache.polaris.service.it.ext.PolarisServerManager b/runtime/spark-tests/src/catalogFederationIntTest/resources/META-INF/services/org.apache.polaris.service.it.ext.PolarisServerManager new file mode 100644 index 00000000000..6aa2dac9e0c --- /dev/null +++ b/runtime/spark-tests/src/catalogFederationIntTest/resources/META-INF/services/org.apache.polaris.service.it.ext.PolarisServerManager @@ -0,0 +1,20 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# + +org.apache.polaris.service.it.ext.ExternalPolarisServerManager diff --git a/runtime/spark-tests/src/hiveFederationIntTest/java/org/apache/polaris/service/spark/it/HiveCatalogFederationIT.java b/runtime/spark-tests/src/hiveFederationIntTest/java/org/apache/polaris/service/spark/it/HiveCatalogFederationIT.java new file mode 100644 index 00000000000..9fc1f1e86a6 --- /dev/null +++ b/runtime/spark-tests/src/hiveFederationIntTest/java/org/apache/polaris/service/spark/it/HiveCatalogFederationIT.java @@ -0,0 +1,32 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.polaris.service.spark.it; + +import java.nio.file.Path; +import org.apache.polaris.service.it.test.HiveCatalogFederationIntegrationTest; +import org.apache.polaris.test.rustfs.RustfsAccess; + +public class HiveCatalogFederationIT extends HiveCatalogFederationIntegrationTest { + + @Override + protected RustfsAccess rustfsAccess() { + return PropertiesRustfsAccess.from( + Path.of(System.getProperty("polaris.spark-tests.hive-rustfs.properties"))); + } +} diff --git a/runtime/spark-tests/src/hiveFederationIntTest/java/org/apache/polaris/service/spark/it/PropertiesRustfsAccess.java b/runtime/spark-tests/src/hiveFederationIntTest/java/org/apache/polaris/service/spark/it/PropertiesRustfsAccess.java new file mode 100644 index 00000000000..d5bbada494b --- /dev/null +++ b/runtime/spark-tests/src/hiveFederationIntTest/java/org/apache/polaris/service/spark/it/PropertiesRustfsAccess.java @@ -0,0 +1,128 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.polaris.service.spark.it; + +import java.io.IOException; +import java.io.InputStream; +import java.net.URI; +import java.nio.file.Path; +import java.util.HashMap; +import java.util.Map; +import java.util.Optional; +import java.util.Properties; +import org.apache.polaris.test.rustfs.RustfsAccess; +import software.amazon.awssdk.auth.credentials.AwsBasicCredentials; +import software.amazon.awssdk.auth.credentials.StaticCredentialsProvider; +import software.amazon.awssdk.http.urlconnection.UrlConnectionHttpClient; +import software.amazon.awssdk.regions.Region; +import software.amazon.awssdk.services.s3.S3Client; + +/** Test-side view of the RustFS container owned by {@link SparkTestsStartupAction}. */ +final class PropertiesRustfsAccess implements RustfsAccess { + private final Properties properties; + + private PropertiesRustfsAccess(Properties properties) { + this.properties = properties; + } + + static PropertiesRustfsAccess from(Path path) { + Properties properties = new Properties(); + try (InputStream input = java.nio.file.Files.newInputStream(path)) { + properties.load(input); + } catch (IOException e) { + throw new IllegalStateException("Failed to load RustFS properties from " + path, e); + } + return new PropertiesRustfsAccess(properties); + } + + @Override + public String hostPort() { + return value("hostPort"); + } + + @Override + public String accessKey() { + return value("accessKey"); + } + + @Override + public String secretKey() { + return value("secretKey"); + } + + @Override + public String bucket() { + return value("bucket"); + } + + @Override + public Optional region() { + return Optional.ofNullable(properties.getProperty("region")); + } + + @Override + public String s3endpoint() { + return value("s3endpoint"); + } + + @Override + public S3Client s3Client() { + return S3Client.builder() + .httpClientBuilder(UrlConnectionHttpClient.builder()) + .endpointOverride(URI.create(s3endpoint())) + .applyMutation(builder -> region().ifPresent(region -> builder.region(Region.of(region)))) + .credentialsProvider( + StaticCredentialsProvider.create(AwsBasicCredentials.create(accessKey(), secretKey()))) + .build(); + } + + @Override + public Map icebergProperties() { + Map props = new HashMap<>(); + props.put("s3.access-key-id", accessKey()); + props.put("s3.secret-access-key", secretKey()); + props.put("s3.endpoint", s3endpoint()); + props.put("http-client.type", "urlconnection"); + region().ifPresent(region -> props.put("client.region", region)); + return props; + } + + @Override + public Map hadoopConfig() { + Map props = new HashMap<>(); + props.put("fs.s3.impl", "org.apache.hadoop.fs.s3a.S3AFileSystem"); + props.put("fs.s3a.access.key", accessKey()); + props.put("fs.s3a.secret.key", secretKey()); + props.put("fs.s3a.endpoint", s3endpoint()); + return props; + } + + @Override + public URI s3BucketUri(String path) { + return URI.create(String.format("s3://%s/", bucket())).resolve(path); + } + + private String value(String name) { + String value = properties.getProperty(name); + if (value == null || value.isBlank()) { + throw new IllegalStateException("Missing RustFS property: " + name); + } + return value; + } +} diff --git a/runtime/spark-tests/src/hiveFederationIntTest/resources/META-INF/services/org.apache.polaris.service.it.ext.PolarisServerManager b/runtime/spark-tests/src/hiveFederationIntTest/resources/META-INF/services/org.apache.polaris.service.it.ext.PolarisServerManager new file mode 100644 index 00000000000..6aa2dac9e0c --- /dev/null +++ b/runtime/spark-tests/src/hiveFederationIntTest/resources/META-INF/services/org.apache.polaris.service.it.ext.PolarisServerManager @@ -0,0 +1,20 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# + +org.apache.polaris.service.it.ext.ExternalPolarisServerManager diff --git a/runtime/spark-tests/src/intTest/java/org/apache/polaris/service/spark/it/CatalogFederationIT.java b/runtime/spark-tests/src/intTest/java/org/apache/polaris/service/spark/it/CatalogFederationIT.java deleted file mode 100644 index a05211b3016..00000000000 --- a/runtime/spark-tests/src/intTest/java/org/apache/polaris/service/spark/it/CatalogFederationIT.java +++ /dev/null @@ -1,49 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ -package org.apache.polaris.service.spark.it; - -import com.google.common.collect.ImmutableMap; -import io.quarkus.test.junit.QuarkusIntegrationTest; -import io.quarkus.test.junit.QuarkusTestProfile; -import io.quarkus.test.junit.TestProfile; -import java.util.Map; -import org.apache.polaris.service.it.test.CatalogFederationIntegrationTest; - -@TestProfile(CatalogFederationIT.CatalogFederationProfile.class) -@QuarkusIntegrationTest -public class CatalogFederationIT extends CatalogFederationIntegrationTest { - - public static class CatalogFederationProfile implements QuarkusTestProfile { - @Override - public Map getConfigOverrides() { - return ImmutableMap.builder() - .put("polaris.features.\"ENABLE_CATALOG_FEDERATION\"", "true") - .put("polaris.features.\"SUPPORTED_CATALOG_CONNECTION_TYPES\"", "[\"ICEBERG_REST\"]") - .put("polaris.features.\"ALLOW_OVERLAPPING_CATALOG_URLS\"", "true") - .put("polaris.features.\"ENABLE_SUB_CATALOG_RBAC_FOR_FEDERATED_CATALOGS\"", "true") - .put("polaris.features.\"ALLOW_DROPPING_NON_EMPTY_PASSTHROUGH_FACADE_CATALOG\"", "true") - .put("polaris.features.\"SKIP_CREDENTIAL_SUBSCOPING_INDIRECTION\"", "false") - .put("polaris.features.\"ALLOW_EXTERNAL_CATALOG_CREDENTIAL_VENDING\"", "true") - .put("polaris.features.\"ALLOW_FEDERATED_CATALOGS_CREDENTIAL_VENDING\"", "true") - .put("polaris.storage.aws.access-key", CatalogFederationIntegrationTest.RUSTFS_ACCESS_KEY) - .put("polaris.storage.aws.secret-key", CatalogFederationIntegrationTest.RUSTFS_SECRET_KEY) - .build(); - } - } -} diff --git a/runtime/spark-tests/src/intTest/java/org/apache/polaris/service/spark/it/HiveCatalogFederationIT.java b/runtime/spark-tests/src/intTest/java/org/apache/polaris/service/spark/it/HiveCatalogFederationIT.java deleted file mode 100644 index ccfdc4249e6..00000000000 --- a/runtime/spark-tests/src/intTest/java/org/apache/polaris/service/spark/it/HiveCatalogFederationIT.java +++ /dev/null @@ -1,108 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ -package org.apache.polaris.service.spark.it; - -import com.google.common.collect.ImmutableMap; -import io.quarkus.test.common.QuarkusTestResource; -import io.quarkus.test.common.QuarkusTestResourceLifecycleManager; -import io.quarkus.test.junit.QuarkusIntegrationTest; -import io.quarkus.test.junit.QuarkusTestProfile; -import io.quarkus.test.junit.TestProfile; -import java.util.Map; -import org.apache.polaris.service.it.test.HiveCatalogFederationIntegrationTest; -import org.apache.polaris.test.rustfs.RustfsAccess; -import org.apache.polaris.test.rustfs.RustfsContainer; - -@TestProfile(HiveCatalogFederationIT.HiveFederationProfile.class) -@QuarkusTestResource(HiveCatalogFederationIT.RustfsResource.class) -@QuarkusIntegrationTest -public class HiveCatalogFederationIT extends HiveCatalogFederationIntegrationTest { - - /** Populated by {@link RustfsResource#inject} before {@code @BeforeAll} runs. */ - private RustfsAccess rustfsAccess; - - @Override - protected RustfsAccess rustfsAccess() { - return rustfsAccess; - } - - public static class HiveFederationProfile implements QuarkusTestProfile { - @Override - public Map getConfigOverrides() { - return ImmutableMap.builder() - .put("polaris.features.\"ENABLE_CATALOG_FEDERATION\"", "true") - .put("polaris.features.\"SUPPORTED_CATALOG_CONNECTION_TYPES\"", "[\"HIVE\"]") - .put( - "polaris.features.\"SUPPORTED_EXTERNAL_CATALOG_AUTHENTICATION_TYPES\"", - "[\"IMPLICIT\"]") - .put("polaris.features.\"ALLOW_OVERLAPPING_CATALOG_URLS\"", "true") - .put("polaris.features.\"ENABLE_SUB_CATALOG_RBAC_FOR_FEDERATED_CATALOGS\"", "true") - .put("polaris.features.\"ALLOW_DROPPING_NON_EMPTY_PASSTHROUGH_FACADE_CATALOG\"", "true") - .put("polaris.features.\"SKIP_CREDENTIAL_SUBSCOPING_INDIRECTION\"", "false") - .put("polaris.features.\"ALLOW_EXTERNAL_CATALOG_CREDENTIAL_VENDING\"", "true") - .put("polaris.features.\"ALLOW_FEDERATED_CATALOGS_CREDENTIAL_VENDING\"", "true") - .put( - "polaris.storage.aws.access-key", - HiveCatalogFederationIntegrationTest.RUSTFS_ACCESS_KEY) - .put( - "polaris.storage.aws.secret-key", - HiveCatalogFederationIntegrationTest.RUSTFS_SECRET_KEY) - .build(); - } - } - - /** - * Starts a RustFS container , exposes its endpoint and credentials, and injects the running - * container into {@link HiveCatalogFederationIT#rustfsAccess} on the test instance. - */ - public static class RustfsResource implements QuarkusTestResourceLifecycleManager { - - private RustfsContainer container; - - @Override - public Map start() { - container = - new RustfsContainer( - null, - HiveCatalogFederationIntegrationTest.RUSTFS_ACCESS_KEY, - HiveCatalogFederationIntegrationTest.RUSTFS_SECRET_KEY, - null, - null) - .withStartupAttempts(5); - container.start(); - return ImmutableMap.of( - "polaris.s3.endpoint", container.s3endpoint(), - "polaris.s3.access-key", container.accessKey(), - "polaris.s3.secret-key", container.secretKey()); - } - - @Override - public void inject(TestInjector testInjector) { - testInjector.injectIntoFields(container, new TestInjector.MatchesType(RustfsAccess.class)); - } - - @Override - public void stop() { - if (container != null) { - container.close(); - container = null; - } - } - } -} diff --git a/runtime/spark-tests/src/intTest/java/org/apache/polaris/service/spark/it/SparkIT.java b/runtime/spark-tests/src/sparkIntTest/java/org/apache/polaris/service/spark/it/SparkIT.java similarity index 92% rename from runtime/spark-tests/src/intTest/java/org/apache/polaris/service/spark/it/SparkIT.java rename to runtime/spark-tests/src/sparkIntTest/java/org/apache/polaris/service/spark/it/SparkIT.java index f82140e9c1c..4b9a21a663f 100644 --- a/runtime/spark-tests/src/intTest/java/org/apache/polaris/service/spark/it/SparkIT.java +++ b/runtime/spark-tests/src/sparkIntTest/java/org/apache/polaris/service/spark/it/SparkIT.java @@ -18,8 +18,6 @@ */ package org.apache.polaris.service.spark.it; -import io.quarkus.test.junit.QuarkusIntegrationTest; import org.apache.polaris.service.it.test.PolarisSparkIntegrationTest; -@QuarkusIntegrationTest public class SparkIT extends PolarisSparkIntegrationTest {} diff --git a/runtime/spark-tests/src/sparkIntTest/resources/META-INF/services/org.apache.polaris.service.it.ext.PolarisServerManager b/runtime/spark-tests/src/sparkIntTest/resources/META-INF/services/org.apache.polaris.service.it.ext.PolarisServerManager new file mode 100644 index 00000000000..6aa2dac9e0c --- /dev/null +++ b/runtime/spark-tests/src/sparkIntTest/resources/META-INF/services/org.apache.polaris.service.it.ext.PolarisServerManager @@ -0,0 +1,20 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# + +org.apache.polaris.service.it.ext.ExternalPolarisServerManager diff --git a/runtime/spark-tests/src/sparkStartupAction/java/org/apache/polaris/service/spark/it/SparkTestsStartupAction.java b/runtime/spark-tests/src/sparkStartupAction/java/org/apache/polaris/service/spark/it/SparkTestsStartupAction.java new file mode 100644 index 00000000000..df8b75fd4e6 --- /dev/null +++ b/runtime/spark-tests/src/sparkStartupAction/java/org/apache/polaris/service/spark/it/SparkTestsStartupAction.java @@ -0,0 +1,75 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.polaris.service.spark.it; + +import java.io.IOException; +import java.io.OutputStream; +import java.nio.file.Files; +import java.nio.file.Path; +import java.util.Properties; +import org.apache.polaris.server.test.runner.spi.PolarisServerStartupAction; +import org.apache.polaris.server.test.runner.spi.PolarisServerStartupContext; +import org.apache.polaris.test.rustfs.RustfsContainer; + +/** Starts services required before the custom Spark-test Polaris server process starts. */ +public class SparkTestsStartupAction implements PolarisServerStartupAction { + private static final String RUSTFS_ACCESS_KEY = "test-ak-123-hive-federation"; + private static final String RUSTFS_SECRET_KEY = "test-sk-123-hive-federation"; + + private RustfsContainer hiveRustfs; + + @Override + public void start(PolarisServerStartupContext context) throws IOException { + hiveRustfs = + new RustfsContainer(null, RUSTFS_ACCESS_KEY, RUSTFS_SECRET_KEY, null, "us-west-2") + .withStartupAttempts(5); + hiveRustfs.start(); + + context.getSystemProperties().put("polaris.s3.endpoint", hiveRustfs.s3endpoint()); + context.getSystemProperties().put("polaris.s3.access-key", hiveRustfs.accessKey()); + context.getSystemProperties().put("polaris.s3.secret-key", hiveRustfs.secretKey()); + + String propertiesFile = context.getParameters().get("hiveRustfsProperties"); + if (propertiesFile != null) { + writeHiveRustfsProperties(Path.of(propertiesFile)); + } + } + + private void writeHiveRustfsProperties(Path path) throws IOException { + Files.createDirectories(path.getParent()); + Properties properties = new Properties(); + properties.setProperty("hostPort", hiveRustfs.hostPort()); + properties.setProperty("accessKey", hiveRustfs.accessKey()); + properties.setProperty("secretKey", hiveRustfs.secretKey()); + properties.setProperty("bucket", hiveRustfs.bucket()); + properties.setProperty("s3endpoint", hiveRustfs.s3endpoint()); + hiveRustfs.region().ifPresent(region -> properties.setProperty("region", region)); + try (OutputStream output = Files.newOutputStream(path)) { + properties.store(output, "RustFS instance started by SparkTestsStartupAction"); + } + } + + @Override + public void close() { + if (hiveRustfs != null) { + hiveRustfs.close(); + hiveRustfs = null; + } + } +} diff --git a/runtime/test-common/build.gradle.kts b/runtime/test-common/build.gradle.kts index ff22995ecaf..73725c032cc 100644 --- a/runtime/test-common/build.gradle.kts +++ b/runtime/test-common/build.gradle.kts @@ -23,7 +23,7 @@ plugins { id("polaris-runtime") } -configurations.all { +configurations.configureEach { if (name != "checkstyle") { exclude(group = "org.antlr", module = "antlr4-runtime") exclude(group = "org.scala-lang", module = "scala-library") diff --git a/settings.gradle.kts b/settings.gradle.kts index 8e4db0cb62e..44070ca6aaf 100644 --- a/settings.gradle.kts +++ b/settings.gradle.kts @@ -35,6 +35,8 @@ if ( includeBuild("build-logic") { name = "polaris-build-logic" } +includeBuild("gradle/server-test-runner") { name = "polaris-server-test-runner" } + if (!JavaVersion.current().isCompatibleWith(JavaVersion.VERSION_21)) { throw GradleException( """ @@ -50,15 +52,12 @@ if (!JavaVersion.current().isCompatibleWith(JavaVersion.VERSION_21)) { rootProject.name = "polaris" -val baseVersion = layout.rootDirectory.file("version.txt").asFile.readText().trim() +val baseVersion = + providers.fileContents(layout.rootDirectory.file("version.txt")).asText.map { it.trim() } gradle.beforeProject { - version = baseVersion + version = baseVersion.get() group = "org.apache.polaris" - - if (noSourceChecksProjects.contains(this.path)) { - project.extra["duplicated-project-sources"] = true - } } fun loadProperties(file: File): Properties { @@ -87,9 +86,6 @@ val polarisSparkDir = "plugins/spark" val sparkScalaVersions = loadProperties(file("${polarisSparkDir}/spark-scala.properties")) val sparkVersions = sparkScalaVersions["sparkVersions"].toString().split(",").map { it.trim() } -// records the spark projects that maps to the same project dir -val noSourceChecksProjects = mutableSetOf() - for (sparkVersion in sparkVersions) { val scalaVersionsKey = "scalaVersions.${sparkVersion}" val scalaVersionsStr = sparkScalaVersions[scalaVersionsKey].toString() @@ -97,20 +93,12 @@ for (sparkVersion in sparkVersions) { var first = true for (scalaVersion in scalaVersions) { val sparkArtifactId = "polaris-spark-${sparkVersion}_${scalaVersion}" - val sparkIntArtifactId = "polaris-spark-integration-${sparkVersion}_${scalaVersion}" polarisProject( "polaris-spark-${sparkVersion}_${scalaVersion}", file("${polarisSparkDir}/v${sparkVersion}/spark"), ) - polarisProject( - "polaris-spark-integration-${sparkVersion}_${scalaVersion}", - file("${polarisSparkDir}/v${sparkVersion}/integration"), - ) if (first) { first = false - } else { - noSourceChecksProjects.add(":$sparkArtifactId") - noSourceChecksProjects.add(":$sparkIntArtifactId") } // Skip all duplicated spark client projects while using Intelij IDE. // This is to avoid problems during dependency analysis and sync when @@ -159,12 +147,13 @@ dependencyResolutionManagement { } } -val isCI = System.getenv("CI") != null +val isCI = providers.environmentVariable("CI").isPresent val isBuildScanRequested = gradle.startParameter.isBuildScan develocity { - val isApachePolarisGitHub = "apache/polaris" == System.getenv("GITHUB_REPOSITORY") - val gitHubRef: String? = System.getenv("GITHUB_REF") + val isApachePolarisGitHub = + "apache/polaris" == providers.environmentVariable("GITHUB_REPOSITORY").orNull + val gitHubRef: String? = providers.environmentVariable("GITHUB_REF").orNull val isGitHubBranchOrTag = gitHubRef != null && (gitHubRef.startsWith("refs/heads/") || gitHubRef.startsWith("refs/tags/")) if (isApachePolarisGitHub && isGitHubBranchOrTag) { @@ -180,25 +169,29 @@ develocity { } } else { // In all other cases, especially PR CI runs, use Gradle's public Develocity instance. - var cfgPrjId: String? = System.getenv("DEVELOCITY_PROJECT_ID") - projectId = if (cfgPrjId.isNullOrEmpty()) "polaris" else cfgPrjId + projectId = + providers + .environmentVariable("DEVELOCITY_PROJECT_ID") + .filter { it.isNotBlank() } + .getOrElse("polaris") buildScan { - val isGradleTosAccepted = "true" == System.getenv("GRADLE_TOS_ACCEPTED") + val isGradleTosAccepted = + "true" == providers.environmentVariable("GRADLE_TOS_ACCEPTED").orNull val isGitHubPullRequest = gitHubRef?.startsWith("refs/pull/") ?: false if (isGradleTosAccepted || (isCI && isGitHubPullRequest && isApachePolarisGitHub)) { // Leave TOS agreement to the user, if not running in CI. termsOfUseUrl = "https://gradle.com/terms-of-service" termsOfUseAgree = "yes" } - System.getenv("DEVELOCITY_SERVER")?.run { - if (isNotEmpty()) { - server = this - } - } + providers + .environmentVariable("DEVELOCITY_SERVER") + .filter { it.isNotBlank() } + .orNull + ?.run { server = this } if (isGitHubPullRequest) { - System.getenv("GITHUB_SERVER_URL")?.run { + providers.environmentVariable("GITHUB_SERVER_URL").orNull?.run { val ghUrl = this - val ghRepo = System.getenv("GITHUB_REPOSITORY") + val ghRepo = providers.environmentVariable("GITHUB_REPOSITORY").get() val prNumber = gitHubRef.substringAfter("refs/pull/").substringBefore("/merge") link("GitHub pull request", "$ghUrl/$ghRepo/pull/$prNumber") } diff --git a/tools/minio-testcontainer/build.gradle.kts b/tools/minio-testcontainer/build.gradle.kts index 5aa1eb9cd00..8a558aa891e 100644 --- a/tools/minio-testcontainer/build.gradle.kts +++ b/tools/minio-testcontainer/build.gradle.kts @@ -36,4 +36,7 @@ dependencies { compileOnly(platform(libs.junit.bom)) compileOnly("org.junit.jupiter:junit-jupiter-api") + + compileOnly(platform(libs.quarkus.bom)) + compileOnly("io.quarkus:quarkus-test-common") } diff --git a/tools/minio-testcontainer/src/main/java/org/apache/polaris/test/minio/MinioConditionExtension.java b/tools/minio-testcontainer/src/main/java/org/apache/polaris/test/minio/MinioConditionExtension.java new file mode 100644 index 00000000000..fbb96539dd3 --- /dev/null +++ b/tools/minio-testcontainer/src/main/java/org/apache/polaris/test/minio/MinioConditionExtension.java @@ -0,0 +1,46 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.polaris.test.minio; + +import static java.lang.String.format; +import static org.junit.jupiter.api.extension.ConditionEvaluationResult.disabled; +import static org.junit.jupiter.api.extension.ConditionEvaluationResult.enabled; + +import org.junit.jupiter.api.condition.OS; +import org.junit.jupiter.api.extension.ConditionEvaluationResult; +import org.junit.jupiter.api.extension.ExecutionCondition; +import org.junit.jupiter.api.extension.ExtensionContext; + +/** JUnit extension that gates whether the Minio container can be run on the current platform. */ +public class MinioConditionExtension implements ExecutionCondition { + @Override + public ConditionEvaluationResult evaluateExecutionCondition(ExtensionContext context) { + if (OS.current() == OS.LINUX) { + return enabled("Running on Linux"); + } + if (OS.current() == OS.MAC + && System.getenv("CI_MAC") == null + && MinioContainer.canRunOnMacOs()) { + // Disable tests on GitHub Actions + return enabled("Running on macOS locally"); + } + return disabled(format("Disabled on %s", OS.current().name())); + } +} diff --git a/tools/minio-testcontainer/src/main/java/org/apache/polaris/test/minio/MinioExtension.java b/tools/minio-testcontainer/src/main/java/org/apache/polaris/test/minio/MinioExtension.java index 4063e13a6fd..e30e60685f5 100644 --- a/tools/minio-testcontainer/src/main/java/org/apache/polaris/test/minio/MinioExtension.java +++ b/tools/minio-testcontainer/src/main/java/org/apache/polaris/test/minio/MinioExtension.java @@ -19,18 +19,12 @@ package org.apache.polaris.test.minio; -import static java.lang.String.format; -import static org.junit.jupiter.api.extension.ConditionEvaluationResult.disabled; -import static org.junit.jupiter.api.extension.ConditionEvaluationResult.enabled; import static org.junit.platform.commons.util.AnnotationUtils.findAnnotatedFields; import static org.junit.platform.commons.util.ReflectionUtils.makeAccessible; import java.lang.reflect.Field; -import org.junit.jupiter.api.condition.OS; import org.junit.jupiter.api.extension.BeforeAllCallback; import org.junit.jupiter.api.extension.BeforeEachCallback; -import org.junit.jupiter.api.extension.ConditionEvaluationResult; -import org.junit.jupiter.api.extension.ExecutionCondition; import org.junit.jupiter.api.extension.ExtensionContext; import org.junit.jupiter.api.extension.ParameterContext; import org.junit.jupiter.api.extension.ParameterResolutionException; @@ -46,25 +40,11 @@ * annotated with {@link Minio}. */ // CODE_COPIED_TO_POLARIS from Project Nessie 0.104.2 -public class MinioExtension - implements BeforeAllCallback, BeforeEachCallback, ParameterResolver, ExecutionCondition { +public class MinioExtension extends MinioConditionExtension + implements BeforeAllCallback, BeforeEachCallback, ParameterResolver { private static final ExtensionContext.Namespace NAMESPACE = ExtensionContext.Namespace.create(MinioExtension.class); - @Override - public ConditionEvaluationResult evaluateExecutionCondition(ExtensionContext context) { - if (OS.current() == OS.LINUX) { - return enabled("Running on Linux"); - } - if (OS.current() == OS.MAC - && System.getenv("CI_MAC") == null - && MinioContainer.canRunOnMacOs()) { - // Disable tests on GitHub Actions - return enabled("Running on macOS locally"); - } - return disabled(format("Disabled on %s", OS.current().name())); - } - @Override public void beforeAll(ExtensionContext context) { Class testClass = context.getRequiredTestClass(); diff --git a/tools/minio-testcontainer/src/main/java/org/apache/polaris/test/minio/MinioTestResource.java b/tools/minio-testcontainer/src/main/java/org/apache/polaris/test/minio/MinioTestResource.java new file mode 100644 index 00000000000..df2d26d6830 --- /dev/null +++ b/tools/minio-testcontainer/src/main/java/org/apache/polaris/test/minio/MinioTestResource.java @@ -0,0 +1,65 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.polaris.test.minio; + +import static java.util.Objects.requireNonNull; + +import io.quarkus.test.common.QuarkusTestResourceLifecycleManager; +import java.util.Map; + +public class MinioTestResource implements QuarkusTestResourceLifecycleManager { + + private Map initArgs = Map.of(); + private MinioContainer container; + + @Override + public void init(Map initArgs) { + this.initArgs = Map.copyOf(initArgs); + } + + @Override + public Map start() { + var accessKey = initArgs.get("accessKey"); + var secretKey = initArgs.get("secretKey"); + var bucket = initArgs.get("bucket"); + var region = initArgs.get("region"); + this.container = + new MinioContainer(null, accessKey, secretKey, bucket, region).withStartupAttempts(5); + this.container.start(); + return Map.of(); + } + + @Override + public void stop() { + if (container != null) { + try { + container.close(); + } catch (Exception e) { + throw new RuntimeException(e); + } + } + } + + @Override + public void inject(TestInjector testInjector) { + MinioAccess minioAccess = requireNonNull(container, "Minio not started"); + testInjector.injectIntoFields(minioAccess, field -> field.isAnnotationPresent(Minio.class)); + } +} diff --git a/tools/rustfs-testcontainer/build.gradle.kts b/tools/rustfs-testcontainer/build.gradle.kts index 5aa1eb9cd00..8a558aa891e 100644 --- a/tools/rustfs-testcontainer/build.gradle.kts +++ b/tools/rustfs-testcontainer/build.gradle.kts @@ -36,4 +36,7 @@ dependencies { compileOnly(platform(libs.junit.bom)) compileOnly("org.junit.jupiter:junit-jupiter-api") + + compileOnly(platform(libs.quarkus.bom)) + compileOnly("io.quarkus:quarkus-test-common") } diff --git a/tools/rustfs-testcontainer/src/main/java/org/apache/polaris/test/rustfs/RustfsConditionExtension.java b/tools/rustfs-testcontainer/src/main/java/org/apache/polaris/test/rustfs/RustfsConditionExtension.java new file mode 100644 index 00000000000..9dcaa6e693b --- /dev/null +++ b/tools/rustfs-testcontainer/src/main/java/org/apache/polaris/test/rustfs/RustfsConditionExtension.java @@ -0,0 +1,46 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.polaris.test.rustfs; + +import static java.lang.String.format; +import static org.junit.jupiter.api.extension.ConditionEvaluationResult.disabled; +import static org.junit.jupiter.api.extension.ConditionEvaluationResult.enabled; + +import org.junit.jupiter.api.condition.OS; +import org.junit.jupiter.api.extension.ConditionEvaluationResult; +import org.junit.jupiter.api.extension.ExecutionCondition; +import org.junit.jupiter.api.extension.ExtensionContext; + +/** JUnit extension that gates whether the Rustfs container can be run on the current platform. */ +public class RustfsConditionExtension implements ExecutionCondition { + @Override + public ConditionEvaluationResult evaluateExecutionCondition(ExtensionContext context) { + if (OS.current() == OS.LINUX) { + return enabled("Running on Linux"); + } + if (OS.current() == OS.MAC + && System.getenv("CI_MAC") == null + && RustfsContainer.canRunOnMacOs()) { + // Disable tests on GitHub Actions + return enabled("Running on macOS locally"); + } + return disabled(format("Disabled on %s", OS.current().name())); + } +} diff --git a/tools/rustfs-testcontainer/src/main/java/org/apache/polaris/test/rustfs/RustfsExtension.java b/tools/rustfs-testcontainer/src/main/java/org/apache/polaris/test/rustfs/RustfsExtension.java index 3205cd9e616..deeae4cd9b2 100644 --- a/tools/rustfs-testcontainer/src/main/java/org/apache/polaris/test/rustfs/RustfsExtension.java +++ b/tools/rustfs-testcontainer/src/main/java/org/apache/polaris/test/rustfs/RustfsExtension.java @@ -19,18 +19,12 @@ package org.apache.polaris.test.rustfs; -import static java.lang.String.format; -import static org.junit.jupiter.api.extension.ConditionEvaluationResult.disabled; -import static org.junit.jupiter.api.extension.ConditionEvaluationResult.enabled; import static org.junit.platform.commons.util.AnnotationUtils.findAnnotatedFields; import static org.junit.platform.commons.util.ReflectionUtils.makeAccessible; import java.lang.reflect.Field; -import org.junit.jupiter.api.condition.OS; import org.junit.jupiter.api.extension.BeforeAllCallback; import org.junit.jupiter.api.extension.BeforeEachCallback; -import org.junit.jupiter.api.extension.ConditionEvaluationResult; -import org.junit.jupiter.api.extension.ExecutionCondition; import org.junit.jupiter.api.extension.ExtensionContext; import org.junit.jupiter.api.extension.ParameterContext; import org.junit.jupiter.api.extension.ParameterResolutionException; @@ -46,25 +40,11 @@ * annotated with {@link Rustfs}. */ // CODE_COPIED_TO_POLARIS from Project Nessie 0.104.2 -public class RustfsExtension - implements BeforeAllCallback, BeforeEachCallback, ParameterResolver, ExecutionCondition { +public class RustfsExtension extends RustfsConditionExtension + implements BeforeAllCallback, BeforeEachCallback, ParameterResolver { private static final ExtensionContext.Namespace NAMESPACE = ExtensionContext.Namespace.create(RustfsExtension.class); - @Override - public ConditionEvaluationResult evaluateExecutionCondition(ExtensionContext context) { - if (OS.current() == OS.LINUX) { - return enabled("Running on Linux"); - } - if (OS.current() == OS.MAC - && System.getenv("CI_MAC") == null - && RustfsContainer.canRunOnMacOs()) { - // Disable tests on GitHub Actions - return enabled("Running on macOS locally"); - } - return disabled(format("Disabled on %s", OS.current().name())); - } - @Override public void beforeAll(ExtensionContext context) { Class testClass = context.getRequiredTestClass(); diff --git a/tools/rustfs-testcontainer/src/main/java/org/apache/polaris/test/rustfs/RustfsTestResource.java b/tools/rustfs-testcontainer/src/main/java/org/apache/polaris/test/rustfs/RustfsTestResource.java new file mode 100644 index 00000000000..7063084f611 --- /dev/null +++ b/tools/rustfs-testcontainer/src/main/java/org/apache/polaris/test/rustfs/RustfsTestResource.java @@ -0,0 +1,65 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.polaris.test.rustfs; + +import static java.util.Objects.requireNonNull; + +import io.quarkus.test.common.QuarkusTestResourceLifecycleManager; +import java.util.Map; + +public class RustfsTestResource implements QuarkusTestResourceLifecycleManager { + + private Map initArgs = Map.of(); + private RustfsContainer container; + + @Override + public void init(Map initArgs) { + this.initArgs = Map.copyOf(initArgs); + } + + @Override + public Map start() { + var accessKey = initArgs.get("accessKey"); + var secretKey = initArgs.get("secretKey"); + var bucket = initArgs.get("bucket"); + var region = initArgs.get("region"); + this.container = + new RustfsContainer(null, accessKey, secretKey, bucket, region).withStartupAttempts(5); + this.container.start(); + return Map.of(); + } + + @Override + public void stop() { + if (container != null) { + try { + container.close(); + } catch (Exception e) { + throw new RuntimeException(e); + } + } + } + + @Override + public void inject(TestInjector testInjector) { + RustfsAccess rustfsAccess = requireNonNull(container, "RustFS not started"); + testInjector.injectIntoFields(rustfsAccess, field -> field.isAnnotationPresent(Rustfs.class)); + } +} diff --git a/tools/version/build.gradle.kts b/tools/version/build.gradle.kts index dd443e74f8b..2b23916d6b7 100644 --- a/tools/version/build.gradle.kts +++ b/tools/version/build.gradle.kts @@ -32,20 +32,19 @@ description = val syncNoticeAndLicense by tasks.registering(Sync::class) { // Have to manually declare the inputs of this task here on top of the from/include below + val rootDir = layout.settingsDirectory inputs - .files( - rootProject.fileTree(rootProject.rootDir) { include("NOTICE*", "LICENSE*", "version.txt") } - ) + .files(fileTree(rootDir) { include("NOTICE*", "LICENSE*", "version.txt") }) .withPathSensitivity(PathSensitivity.RELATIVE) inputs.property("version", project.version) destinationDir = project.layout.buildDirectory.dir("notice-licenses").get().asFile - from(rootProject.rootDir) { + from(rootDir) { include("NOTICE*", "LICENSE*") // Put NOTICE/LICENSE* files under META-INF/resources/ so those files can be directly // accessed as static web resources in Quarkus. eachFile { path = "META-INF/resources/apache-polaris/${file.name}.txt" } } - from(rootProject.rootDir) { + from(rootDir) { include("version.txt") // Put NOTICE/LICENSE* files under META-INF/resources/ so those files can be directly // accessed as static web resources in Quarkus. @@ -86,7 +85,7 @@ testing { register("jarTest") { dependencies { runtimeOnly(files(jarTestJar.map { it.archiveFile })) } - targets.all { + targets.configureEach { testTask.configure { dependsOn("jar", jarTestJar) systemProperty("rootProjectDir", rootProject.rootDir.relativeTo(project.projectDir))