[SPARK-56356][BUILD] Fix an issue in release build caused by error on fetching artifacts

sarutak · HyukjinKwon · commit a20bfc7cafca · 2026-04-05T12:53:39.000+09:00
### What changes were proposed in this pull request? This PR fixes a release build failure which recently happens at document generation phase due to error on fetching artifacts. https://github.com/apache/spark/actions/runs/23083193766/job/67055792458 ``` Error: lmcoursier.internal.shaded.coursier.error.FetchError$DownloadingArtifacts: Error fetching artifacts: Error: file:/home/spark-rm/.m2/repository/io/netty/netty-codec-protobuf/4.2.10.Final/netty-codec-protobuf-4.2.10.Final.jar: not found: /home/spark-rm/.m2/repository/io/netty/netty-codec-protobuf/4.2.10.Final/netty-codec-protobuf-4.2.10.Final.jar Error: file:/home/spark-rm/.m2/repository/io/netty/netty-codec-marshalling/4.2.10.Final/netty-codec-marshalling-4.2.10.Final.jar: not found: /home/spark-rm/.m2/repository/io/netty/netty-codec-marshalling/4.2.10.Final/netty-codec-marshalling-4.2.10.Final.jar Error: Error: at lmcoursier.internal.shaded.coursier.Artifacts$.$anonfun$fetchArtifacts$9(Artifacts.scala:365) Error: at lmcoursier.internal.shaded.coursier.util.Task$.$anonfun$flatMap$extension$1(Task.scala:14) Error: at lmcoursier.internal.shaded.coursier.util.Task$.$anonfun$flatMap$extension$1$adapted(Task.scala:14) Error: at lmcoursier.internal.shaded.coursier.util.Task$.wrap(Task.scala:82) Error: at lmcoursier.internal.shaded.coursier.util.Task$.$anonfun$flatMap$2(Task.scala:14) Error: at scala.concurrent.Future.$anonfun$flatMap$1(Future.scala:307) Error: at scala.concurrent.impl.Promise.$anonfun$transformWith$1(Promise.scala:51) Error: at scala.concurrent.impl.CallbackRunnable.run(Promise.scala:74) Error: at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) Error: at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) Error: at java.base/java.lang.Thread.run(Thread.java:840) Error: Caused by: lmcoursier.internal.shaded.coursier.cache.ArtifactError$NotFound: not found: /home/spark-rm/.m2/repository/io/netty/netty-codec-protobuf/4.2.10.Final/netty-codec-protobuf-4.2.10.Final.jar Error: at lmcoursier.internal.shaded.coursier.cache.internal.Downloader.$anonfun$checkFileExists$1(Downloader.scala:603) Error: at scala.concurrent.Future$.$anonfun$apply$1(Future.scala:659) Error: at scala.util.Success.$anonfun$map$1(Try.scala:255) Error: at scala.util.Success.map(Try.scala:213) Error: at scala.concurrent.Future.$anonfun$map$1(Future.scala:292) Error: at scala.concurrent.impl.Promise.$anonfun$transform$1(Promise.scala:42) Error: at scala.concurrent.impl.CallbackRunnable.run(Promise.scala:74) Error: at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) Error: at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) Error: at java.base/java.lang.Thread.run(Thread.java:840) Error: (streaming-kinesis-asl / update) lmcoursier.internal.shaded.coursier.error.FetchError$DownloadingArtifacts: Error fetching artifacts: Error: file:/home/spark-rm/.m2/repository/io/netty/netty-codec-protobuf/4.2.10.Final/netty-codec-protobuf-4.2.10.Final.jar: not found: /home/spark-rm/.m2/repository/io/netty/netty-codec-protobuf/4.2.10.Final/netty-codec-protobuf-4.2.10.Final.jar Error: file:/home/spark-rm/.m2/repository/io/netty/netty-codec-marshalling/4.2.10.Final/netty-codec-marshalling-4.2.10.Final.jar: not found: /home/spark-rm/.m2/repository/io/netty/netty-codec-marshalling/4.2.10.Final/netty-codec-marshalling-4.2.10.Final.jar Error: Total time: 368 s (0:06:08.0), completed Mar 14, 2026, 8:40:17 AM ------------------------------------------------ Jekyll 4.4.1 Please append `--trace` to the `build` command for any additional information or backtrace. ------------------------------------------------ ``` This issue is similar to SPARK-34762 and SPARK-37302 in that there are pom files but are not corresponding jar files under `.m2` for some dependencies. In this case, the following command is executed through [make-distribution.sh](https://github.com/apache/spark/blob/d9c8eda57e22f65d0443cab7078c632462c11272/dev/make-distribution.sh#L183) and downloads pom files for `xz:1.10`, `netty-codec-protobuf` and `netty-codec-marshalling` ``` build/mvn clean package -DskipTests -Dmaven.javadoc.skip=true -Dmaven.scaladoc.skip=true -Dmaven.source.skip -Dcyclonedx.skip=true -B -Pyarn -Pkubernetes -Phadoop-3 -Phive -Phive-thriftserver ``` And when building documents, the following command is executed through [build_api_docs.rb](https://github.com/apache/spark/blob/d9c8eda57e22f65d0443cab7078c632462c11272/docs/_plugins/build_api_docs.rb#L48) and tries to download the dependencies. ``` NO_PROVIDED_SPARK_JARS=0 build/sbt -Phive -Pkinesis-asl clean package ``` Regarding xz, `1.12` is declared in `pom.xml` so this PR fixes `SparkBuild.scala` to pin the version. Regarding `netty-codec-protobuf` and `netty-codec-marshalling`, they are declared in pom.xml to be excluded. So, this PR fixes `SparkBuild.scala` to exclude them. ### Why are the changes needed? To recover the release build. ### Does this PR introduce _any_ user-facing change? No. ### How was this patch tested? The following command successfully finishes on my laptop. ``` $ build/mvn clean package -DskipTests -Dmaven.javadoc.skip=true -Dmaven.scaladoc.skip=true -Dmaven.source.skip -Dcyclonedx.skip=true -B -Pyarn -Pkubernetes -Phadoop-3 -Phive -Phive-thriftserver $ SKIP_SCALADOC=1 SKIP_RDOC=1 SKIP_SQLDOC=1 bundle exec jekyll build ``` Note that to build documents, please follow the instructions in `docs/README.md` ### Was this patch authored or co-authored using generative AI tooling? No. Closes #55198 from sarutak/fix-dependency-resolution-issue. Authored-by: Kousuke Saruta <sarutak@amazon.co.jp> Signed-off-by: Hyukjin Kwon <gurwls223@apache.org>
diff --git a/pom.xml b/pom.xml
@@ -239,6 +239,7 @@
     <leveldbjni.group>org.fusesource.leveldbjni</leveldbjni.group>
     <kubernetes-client.version>7.6.1</kubernetes-client.version>
     <vertx.version>4.5.26</vertx.version>
+    <xz.version>1.12</xz.version>
 
     <test.java.home>${java.home}</test.java.home>
 
@@ -1602,7 +1603,7 @@
       <dependency>
         <groupId>org.tukaani</groupId>
         <artifactId>xz</artifactId>
-        <version>1.12</version>
+        <version>${xz.version}</version>
       </dependency>
       <dependency>
         <groupId>org.apache.zookeeper</groupId>
diff --git a/project/SparkBuild.scala b/project/SparkBuild.scala
@@ -1200,11 +1200,14 @@ object DependencyOverrides {
         SbtPomKeys.effectivePom.value.getProperties.get("avro.version").asInstanceOf[String]
       val slf4jVersion =
         SbtPomKeys.effectivePom.value.getProperties.get("slf4j.version").asInstanceOf[String]
+      val xzVersion =
+        SbtPomKeys.effectivePom.value.getProperties.get("xz.version").asInstanceOf[String]
       Seq(
         "com.google.guava" % "guava" % guavaVersion,
         "jline" % "jline" % jlineVersion,
         "org.apache.avro" % "avro" % avroVersion,
         "org.slf4j" % "slf4j-api" % slf4jVersion,
+        "org.tukaani" % "xz" % xzVersion,
         "org.scala-lang" % "scalap" % scalaVersion.value
       ) ++ jacksonDeps.key.value
     }
@@ -1222,7 +1225,9 @@ object ExcludedDependencies {
       ExclusionRule(organization = "ch.qos.logback"),
       ExclusionRule("org.lz4", "lz4-java"),
       ExclusionRule("org.slf4j", "slf4j-simple"),
-      ExclusionRule("javax.servlet", "javax.servlet-api"))
+      ExclusionRule("javax.servlet", "javax.servlet-api"),
+      ExclusionRule("io.netty", "netty-codec-protobuf"),
+      ExclusionRule("io.netty", "netty-codec-marshalling"))
   )
 }
 
