OpenLineage: add no-op POST /api/v1/lineage ingest endpoint

api/openlineage-service/build.gradle.kts

@@ -0,0 +1,62 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+// This module deliberately does NOT use the OpenAPI generator. The OpenLineage
+// event schema is a oneOf over RunEvent / JobEvent / DatasetEvent, which the
+// jaxrs-resteasy generator collapses into a single class with every variant's
+// required fields marked @NotNull -- making every real OpenLineage event fail
+// bean validation. Marquez (the OpenLineage reference server) takes the same
+// approach: hand-written JAX-RS resource on top of the official
+// `io.openlineage:openlineage-java` types, using Jackson polymorphism keyed on
+// the `schemaURL` field to dispatch between event variants.
+plugins {
+  id("polaris-client")
+  id("org.kordamp.gradle.jandex")
+}
+
+dependencies {
+  implementation(project(":polaris-core"))
+
+  // The official OpenLineage Java library ships POJOs for the receive side
+  // (`io.openlineage.server.OpenLineage.{BaseEvent,RunEvent,JobEvent,DatasetEvent}`).
+  // We expose them through the API surface; callers reading event content read
+  // these types directly.
+  api(libs.openlineage.java)
+
+  implementation(libs.jakarta.annotation.api)
+  implementation(libs.jakarta.inject.api)
+  implementation(libs.jakarta.validation.api)
+
+  implementation(libs.jakarta.servlet.api)
+  implementation(libs.jakarta.ws.rs.api)
+
+  implementation(platform(libs.micrometer.bom))
+  implementation("io.micrometer:micrometer-core")
+
+  implementation(platform(libs.jackson.bom))
+  implementation("com.fasterxml.jackson.core:jackson-annotations")
+  implementation("com.fasterxml.jackson.core:jackson-core")
+  implementation("com.fasterxml.jackson.core:jackson-databind")
+
+  implementation(libs.slf4j.api)
+
+  compileOnly(libs.microprofile.fault.tolerance.api)
+}
+
+tasks.named("javadoc") { dependsOn("jandex") }

api/openlineage-service/src/main/java/org/apache/polaris/service/lineage/api/LineageEventTypeResolver.java

@@ -0,0 +1,90 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.polaris.service.lineage.api;
+
+import com.fasterxml.jackson.annotation.JsonTypeInfo;
+import com.fasterxml.jackson.databind.DatabindContext;
+import com.fasterxml.jackson.databind.JavaType;
+import com.fasterxml.jackson.databind.jsontype.impl.TypeIdResolverBase;
+
+/**
+ * Resolves an OpenLineage event JSON body to one of {@link PolarisLineageEvent.OfRunEvent}, {@link
+ * PolarisLineageEvent.OfJobEvent}, or {@link PolarisLineageEvent.OfDatasetEvent} by inspecting the
+ * trailing path segment of the {@code schemaURL} field.
+ *
+ * <p>The OpenLineage spec requires every event to include a {@code schemaURL} pointing at the
+ * variant's JSON schema fragment, e.g. {@code
+ * https://openlineage.io/spec/2-0-2/OpenLineage.json#/$defs/RunEvent}. The fragment name is the
+ * stable discriminator across spec versions.
+ *
+ * <p>If {@code schemaURL} is missing or unrecognized, the body is parsed as a {@code RunEvent} —
+ * the most common variant and the same fallback Marquez (the OpenLineage reference server) uses.
+ */
+public class LineageEventTypeResolver extends TypeIdResolverBase {
+
+  private JavaType superType;
+
+  @Override
+  public void init(JavaType baseType) {
+    this.superType = baseType;
+  }
+
+  @Override
+  public String idFromValue(Object value) {
+    return idFromValueAndType(value, value == null ? null : value.getClass());
+  }
+
+  @Override
+  public String idFromValueAndType(Object value, Class<?> suggestedType) {
+    // Polaris does not currently emit lineage events; serialization is not on the
+    // ingest path. Returning a stable label keeps Jackson happy if a downstream
+    // module ever serializes a wrapper.
+    if (suggestedType == PolarisLineageEvent.OfJobEvent.class) {
+      return "JobEvent";
+    }
+    if (suggestedType == PolarisLineageEvent.OfDatasetEvent.class) {
+      return "DatasetEvent";
+    }
+    return "RunEvent";
+  }
+
+  @Override
+  public JavaType typeFromId(DatabindContext context, String id) {
+    Class<?> subType = subTypeFor(id);
+    return context.constructSpecializedType(superType, subType);
+  }
+
+  @Override
+  public JsonTypeInfo.Id getMechanism() {
+    return JsonTypeInfo.Id.CUSTOM;
+  }
+
+  private static Class<?> subTypeFor(String schemaUrl) {
+    if (schemaUrl == null) {
+      return PolarisLineageEvent.OfRunEvent.class;
+    }
+    String[] segments = schemaUrl.split("/");
+    String tail = segments[segments.length - 1];
+    return switch (tail) {
+      case "JobEvent" -> PolarisLineageEvent.OfJobEvent.class;
+      case "DatasetEvent" -> PolarisLineageEvent.OfDatasetEvent.class;
+      default -> PolarisLineageEvent.OfRunEvent.class;
+    };
+  }
+}

api/openlineage-service/src/main/java/org/apache/polaris/service/lineage/api/PolarisLineageEvent.java

@@ -0,0 +1,106 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.polaris.service.lineage.api;
+
+import com.fasterxml.jackson.annotation.JsonCreator;
+import com.fasterxml.jackson.annotation.JsonTypeInfo;
+import com.fasterxml.jackson.annotation.JsonValue;
+import com.fasterxml.jackson.databind.annotation.JsonTypeIdResolver;
+import io.openlineage.server.OpenLineage;
+
+/**
+ * Polaris-side wrapper for an OpenLineage event accepted by the lineage ingest endpoint.
+ *
+ * <p>The OpenLineage spec models events as a {@code oneOf} over {@code RunEvent}, {@code
+ * JobEvent}, and {@code DatasetEvent}, discriminated by the value of the {@code schemaURL} field.
+ * The official {@code openlineage-java} library ships POJOs for these three event types in {@link
+ * io.openlineage.server.OpenLineage} but does not annotate them for polymorphic JSON
+ * deserialization. This class provides that polymorphism on top.
+ *
+ * <p>Wrappers compose the OpenLineage event by reference rather than extending it because the
+ * {@code OpenLineage.*Event} classes are {@code final}. {@link #event()} returns the underlying
+ * event for follow-up code that needs typed access to fields (inputs, outputs, columnLineage
+ * facet, etc.).
+ *
+ * <p>This pattern follows Marquez (the OpenLineage reference server), which uses the same
+ * {@code @JsonTypeInfo} + {@code @JsonTypeIdResolver} discrimination on {@code schemaURL}.
+ */
+@JsonTypeInfo(
+    use = JsonTypeInfo.Id.CUSTOM,
+    include = JsonTypeInfo.As.EXISTING_PROPERTY,
+    property = "schemaURL",
+    defaultImpl = PolarisLineageEvent.OfRunEvent.class,
+    visible = true)
+@JsonTypeIdResolver(LineageEventTypeResolver.class)
+public abstract sealed class PolarisLineageEvent
+    permits PolarisLineageEvent.OfRunEvent,
+        PolarisLineageEvent.OfJobEvent,
+        PolarisLineageEvent.OfDatasetEvent {
+
+  /** Returns the underlying OpenLineage event. */
+  public abstract OpenLineage.BaseEvent event();
+
+  /** Wraps an {@link OpenLineage.RunEvent}. */
+  public static final class OfRunEvent extends PolarisLineageEvent {
+    private final OpenLineage.RunEvent event;
+
+    @JsonCreator(mode = JsonCreator.Mode.DELEGATING)
+    public OfRunEvent(OpenLineage.RunEvent event) {
+      this.event = event;
+    }
+
+    @JsonValue
+    @Override
+    public OpenLineage.RunEvent event() {
+      return event;
+    }
+  }
+
+  /** Wraps an {@link OpenLineage.JobEvent}. */
+  public static final class OfJobEvent extends PolarisLineageEvent {
+    private final OpenLineage.JobEvent event;
+
+    @JsonCreator(mode = JsonCreator.Mode.DELEGATING)
+    public OfJobEvent(OpenLineage.JobEvent event) {
+      this.event = event;
+    }
+
+    @JsonValue
+    @Override
+    public OpenLineage.JobEvent event() {
+      return event;
+    }
+  }
+
+  /** Wraps an {@link OpenLineage.DatasetEvent}. */
+  public static final class OfDatasetEvent extends PolarisLineageEvent {
+    private final OpenLineage.DatasetEvent event;
+
+    @JsonCreator(mode = JsonCreator.Mode.DELEGATING)
+    public OfDatasetEvent(OpenLineage.DatasetEvent event) {
+      this.event = event;
+    }
+
+    @JsonValue
+    @Override
+    public OpenLineage.DatasetEvent event() {
+      return event;
+    }
+  }
+}

api/openlineage-service/src/main/java/org/apache/polaris/service/lineage/api/PolarisOpenLineageApi.java

@@ -0,0 +1,74 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.polaris.service.lineage.api;
+
+import io.micrometer.core.annotation.Timed;
+import io.micrometer.core.aop.MeterTag;
+import jakarta.annotation.security.RolesAllowed;
+import jakarta.inject.Inject;
+import jakarta.validation.Valid;
+import jakarta.validation.constraints.NotNull;
+import jakarta.ws.rs.Consumes;
+import jakarta.ws.rs.POST;
+import jakarta.ws.rs.Path;
+import jakarta.ws.rs.core.Context;
+import jakarta.ws.rs.core.Response;
+import jakarta.ws.rs.core.SecurityContext;
+import org.apache.polaris.core.context.RealmContext;
+import org.eclipse.microprofile.faulttolerance.Timeout;
+
+/**
+ * JAX-RS resource for the OpenLineage ingest endpoint.
+ *
+ * <p>Mounted at the standard OpenLineage path ({@code POST /api/v1/lineage}) so that any engine
+ * already using the OpenLineage HTTP transport (Spark, Flink, Airflow, Trino, dbt) can target
+ * Polaris by URL change alone — no client-side rewriting.
+ *
+ * <p>The body is parsed into a {@link PolarisLineageEvent} which Jackson resolves to one of {@code
+ * OfRunEvent} / {@code OfJobEvent} / {@code OfDatasetEvent} based on the {@code schemaURL} field.
+ * Unrecognized {@code schemaURL} values fall back to {@code RunEvent}, matching Marquez behavior.
+ *
+ * <p>This resource is hand-written rather than generated from the OpenLineage JSON Schema because
+ * the OpenAPI generator's Java template does not faithfully translate the spec's {@code oneOf}
+ * over event variants.
+ */
+@Path("/api/v1/lineage")
+public class PolarisOpenLineageApi {
+
+  private final PolarisOpenLineageApiService service;
+
+  @Inject
+  public PolarisOpenLineageApi(PolarisOpenLineageApiService service) {
+    this.service = service;
+  }
+
+  @POST
+  @Consumes("application/json")
+  @RolesAllowed("**")
+  @Timed("polaris.OpenLineageApi.sendLineageEvent")
+  @Timeout
+  public Response sendLineageEvent(
+      @NotNull @Valid PolarisLineageEvent event,
+      @Context @MeterTag(key = "realm_id", expression = "realmIdentifier")
+          RealmContext realmContext,
+      @Context @MeterTag(key = "principal", expression = "userPrincipal")
+          SecurityContext securityContext) {
+    return service.sendLineageEvent(event, realmContext, securityContext);
+  }
+}

api/openlineage-service/src/main/java/org/apache/polaris/service/lineage/api/PolarisOpenLineageApiService.java

@@ -0,0 +1,43 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.polaris.service.lineage.api;
+
+import jakarta.ws.rs.core.Response;
+import jakarta.ws.rs.core.SecurityContext;
+import org.apache.polaris.core.context.RealmContext;
+
+/**
+ * Service interface implemented by the runtime to handle OpenLineage ingest. Mirrors the pattern
+ * used by other Polaris API modules where the JAX-RS resource sits in the API module and
+ * delegates to a CDI-scoped service implementation in {@code polaris-runtime-service}.
+ */
+public interface PolarisOpenLineageApiService {
+
+  /**
+   * Handle an OpenLineage event accepted at the ingest endpoint.
+   *
+   * @param event the parsed OpenLineage event, dispatched to the correct {@code RunEvent}, {@code
+   *     JobEvent}, or {@code DatasetEvent} variant by Jackson based on the {@code schemaURL}
+   *     field.
+   * @return the JAX-RS response. OpenLineage clients expect {@code 201 Created} with no body on
+   *     success.
+   */
+  Response sendLineageEvent(
+      PolarisLineageEvent event, RealmContext realmContext, SecurityContext securityContext);
+}

gradle/libs.versions.toml

@@ -94,6 +94,7 @@ microprofile-fault-tolerance-api = { module = "org.eclipse.microprofile.fault-to
 mockito-core = { module = "org.mockito:mockito-core", version = "5.23.0" }
 mockito-junit-jupiter = { module = "org.mockito:mockito-junit-jupiter", version = "5.23.0" }
 mongodb-driver-sync = { module = "org.mongodb:mongodb-driver-sync", version = "5.8.0" }
+openlineage-java = { module = "io.openlineage:openlineage-java", version = "1.48.0" }
 opentelemetry-instrumentation-bom-alpha = { module = "io.opentelemetry.instrumentation:opentelemetry-instrumentation-bom-alpha", version= "2.20.1-alpha" }
 picocli = { module = "info.picocli:picocli-codegen", version.ref = "picocli" }
 picocli-codegen = { module = "info.picocli:picocli-codegen", version.ref = "picocli" }

gradle/projects.main.properties

@@ -24,6 +24,7 @@ polaris-api-iceberg-service=api/iceberg-service
 polaris-api-management-model=api/management-model
 polaris-api-management-service=api/management-service
 polaris-api-catalog-service=api/polaris-catalog-service
+polaris-api-openlineage-service=api/openlineage-service
 polaris-runtime-defaults=runtime/defaults
 polaris-runtime-service=runtime/service
 polaris-server=runtime/server