diff --git a/CHANGELOG.md b/CHANGELOG.md index 2999a4f99..e10f47a2b 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -24,6 +24,7 @@ All notable changes to this project will be documented in this file. - druid: Add `37.0.0` ([#1535]). - hbase: Add `2.6.6` ([#1547]). - spark: Add `4.1.2` ([#1550]). +- spark: Bake the OpenLineage Spark listener (`openlineage-spark_2.13` `1.51.0`) into the `4.1.x` images under the dedicated directory `/stackable/spark/openlineage/` (off the `extra-jars` classpath), enabling the OpenLineage MVP in the operator ([#XXXX]). - opensearch: Add `3.6.0` ([#1549]). - opensearch-dashboards: Add `3.6.0` ([#1551]). @@ -95,6 +96,7 @@ All notable changes to this project will be documented in this file. [#1548]: https://github.com/stackabletech/docker-images/pull/1548 [#1549]: https://github.com/stackabletech/docker-images/pull/1549 [#1550]: https://github.com/stackabletech/docker-images/pull/1550 +[#XXXX]: https://github.com/stackabletech/docker-images/pull/XXXX [#1551]: https://github.com/stackabletech/docker-images/pull/1551 [#1559]: https://github.com/stackabletech/docker-images/pull/1559 [#1561]: https://github.com/stackabletech/docker-images/pull/1561 diff --git a/spark-k8s/Dockerfile.4 b/spark-k8s/Dockerfile.4 index fff71258e..c8602ac8f 100644 --- a/spark-k8s/Dockerfile.4 +++ b/spark-k8s/Dockerfile.4 @@ -39,6 +39,7 @@ ENV HADOOP_VERSION=${HADOOP_HADOOP_VERSION} ARG JACKSON_DATAFORMAT_XML_VERSION ARG STAX2_API_VERSION ARG WOODSTOX_CORE_VERSION +ARG OPENLINEAGE_SPARK_VERSION ARG JMX_EXPORTER_VERSION ARG TARGETARCH ARG TINI_VERSION @@ -122,6 +123,16 @@ mvn dependency:get -Dartifact=mvn dependency:get -Dartifact=com.fasterxml.woodst cp /root/.m2/repository/com/fasterxml/woodstox/woodstox-core/${WOODSTOX_CORE_VERSION}/woodstox-core-${WOODSTOX_CORE_VERSION}.jar \ /stackable/spark-${PRODUCT_VERSION}-stackable${RELEASE_VERSION}/dist/extra-jars/woodstox-core-${WOODSTOX_CORE_VERSION}.jar +# Download the OpenLineage Spark listener jar into a DEDICATED directory (dist/openlineage), +# NOT dist/extra-jars. The operator references it via `--jars local:///stackable/spark/openlineage/...` +# so it loads on Spark's child/user classloader, sharing it with connectors delivered via `--packages` +# (e.g. Iceberg). If it landed on extra-jars it would be added to the system classloader via +# extraClassPath, OpenLineage's connector probe would fail, and no datasets would be emitted. +mkdir -p /stackable/spark-${PRODUCT_VERSION}-stackable${RELEASE_VERSION}/dist/openlineage +mvn dependency:get -Dartifact=io.openlineage:openlineage-spark_2.13:${OPENLINEAGE_SPARK_VERSION} +cp /root/.m2/repository/io/openlineage/openlineage-spark_2.13/${OPENLINEAGE_SPARK_VERSION}/openlineage-spark_2.13-${OPENLINEAGE_SPARK_VERSION}.jar \ + /stackable/spark-${PRODUCT_VERSION}-stackable${RELEASE_VERSION}/dist/openlineage/openlineage-spark_2.13-${OPENLINEAGE_SPARK_VERSION}.jar + # Get the correct `tini` binary for our architecture. curl --fail "https://repo.stackable.tech/repository/packages/tini/tini-${TINI_VERSION}-${TARGETARCH}" \ -o /usr/bin/tini diff --git a/spark-k8s/boil-config.toml b/spark-k8s/boil-config.toml index 724c436d9..b8559480c 100644 --- a/spark-k8s/boil-config.toml +++ b/spark-k8s/boil-config.toml @@ -37,6 +37,7 @@ stax2-api-version = "4.2.2" # Needs to match https://mvnrepository woodstox-core-version = "7.1.1" # Needs to match https://mvnrepository.com/artifact/com.fasterxml.jackson.dataformat/jackson-dataformat-xml/2.20.0 jmx-exporter-version = "1.3.0" tini-version = "0.19.0" +openlineage-spark-version = "1.51.0" # Scala 2.13 build for Spark 4.1.x; see https://mvnrepository.com/artifact/io.openlineage/openlineage-spark_2.13 # LTS since 26.7.0 [versions."4.1.2"] @@ -54,3 +55,4 @@ stax2-api-version = "4.2.2" # Needs to match https://mvnrepository woodstox-core-version = "7.1.1" # Needs to match https://mvnrepository.com/artifact/com.fasterxml.jackson.dataformat/jackson-dataformat-xml/2.21.2 jmx-exporter-version = "1.3.0" tini-version = "0.19.0" +openlineage-spark-version = "1.51.0" # Scala 2.13 build for Spark 4.1.x; see https://mvnrepository.com/artifact/io.openlineage/openlineage-spark_2.13