[SPARK-51318][BUILD] Remove test jars in source releases

### What changes were proposed in this pull request?

This PR proposes to remove test jars in source releases during release process.

### Why are the changes needed?

Apache source releases must not contained jar files.
The issue is discussed on https://lists.apache.org/thread/0ro5yn6lbbpmvmqp2px3s2pf7cwljlc4

### Does this PR introduce _any_ user-facing change?

To end users, no.

### How was this patch tested?

Manually tested, and I will work together with release manager to make sure this wroks.

### Was this patch authored or co-authored using generative AI tooling?

No.

Closes apache#50378 from HyukjinKwon/SPARK-51318.

Lead-authored-by: Hyukjin Kwon <[email protected]>
Co-authored-by: Vlad Rozov <[email protected]>
Signed-off-by: Hyukjin Kwon <[email protected]>

Author: HyukjinKwon

connect-examples/server-library-example/README.md

@@ -85,7 +85,7 @@ reading, writing and processing data in the custom format. The plugins (`CustomC
 4. **Copy relevant JARs to the root of the unpacked Spark distribution**:
    ```bash
     cp \
-    <SPARK_HOME>/connect-examples/server-library-example/resources/spark-daria_2.13-1.2.3.jar \
+    <SPARK_HOME>/connect-examples/server-library-example/common/target/spark-daria_2.13-1.2.3.jar \
     <SPARK_HOME>/connect-examples/server-library-example/common/target/spark-server-library-example-common-1.0.0.jar \
     <SPARK_HOME>/connect-examples/server-library-example/server/target/spark-server-library-example-server-extension-1.0.0.jar \
     .

connect-examples/server-library-example/resources/spark-daria_2.13-1.2.3.jar

@@ -62,9 +62,8 @@
     <!-- Include Spark Daria for utility Dataframe write methods -->
     <dependency>
       <groupId>com.github.mrpowers</groupId>
-      <artifactId>spark-daria_2.12</artifactId>
+      <artifactId>spark-daria_${scala.binary}</artifactId>
       <version>1.2.3</version>
-      <scope>provided</scope>
     </dependency>
   </dependencies>
 

connect-examples/server-library-example/server/pom.xml

@@ -244,10 +244,11 @@ class SparkContextSuite extends SparkFunSuite with LocalSparkContext with Eventu
   }
 
   test("add and list jar files") {
-    val jarPath = Thread.currentThread().getContextClassLoader.getResource("TestUDTF.jar")
+    val testJar = Thread.currentThread().getContextClassLoader.getResource("TestUDTF.jar")
+    assume(testJar != null)
     try {
       sc = new SparkContext(new SparkConf().setAppName("test").setMaster("local"))
-      sc.addJar(jarPath.toString)
+      sc.addJar(testJar.toString)
       assert(sc.listJars().count(_.contains("TestUDTF.jar")) == 1)
     } finally {
       sc.stop()
@@ -396,13 +397,15 @@ class SparkContextSuite extends SparkFunSuite with LocalSparkContext with Eventu
     schedulingMode <- Seq("local-mode", "non-local-mode");
     method <- Seq("addJar", "addFile")
   ) {
-    val jarPath = Thread.currentThread().getContextClassLoader.getResource("TestUDTF.jar").toString
     val master = schedulingMode match {
       case "local-mode" => "local"
       case "non-local-mode" => "local-cluster[1,1,1024]"
     }
     test(s"$method can be called twice with same file in $schedulingMode (SPARK-16787)") {
+      val testJar = Thread.currentThread().getContextClassLoader.getResource("TestUDTF.jar")
+      assume(testJar != null)
       sc = new SparkContext(master, "test")
+      val jarPath = testJar.toString
       method match {
         case "addJar" =>
           sc.addJar(jarPath)

core/src/test/scala/org/apache/spark/SparkContextSuite.scala

@@ -506,6 +506,8 @@ class SparkSubmitSuite
 
   test("SPARK-47495: Not to add primary resource to jars again" +
     " in k8s client mode & driver runs inside a POD") {
+    val testJar = "src/test/resources/TestUDTF.jar"
+    assume(new File(testJar).exists)
     val clArgs = Seq(
       "--deploy-mode", "client",
       "--proxy-user", "test.user",
@@ -514,7 +516,7 @@ class SparkSubmitSuite
       "--class", "org.SomeClass",
       "--driver-memory", "1g",
       "--conf", "spark.kubernetes.submitInDriver=true",
-      "--jars", "src/test/resources/TestUDTF.jar",
+      "--jars", testJar,
       "/home/jarToIgnore.jar",
       "arg1")
     val appArgs = new SparkSubmitArguments(clArgs)
@@ -524,6 +526,8 @@ class SparkSubmitSuite
   }
 
   test("SPARK-33782: handles k8s files download to current directory") {
+    val testJar = "src/test/resources/TestUDTF.jar"
+    assume(new File(testJar).exists)
     val clArgs = Seq(
       "--deploy-mode", "client",
       "--proxy-user", "test.user",
@@ -537,7 +541,7 @@ class SparkSubmitSuite
       "--files", "src/test/resources/test_metrics_config.properties",
       "--py-files", "src/test/resources/test_metrics_system.properties",
       "--archives", "src/test/resources/log4j2.properties",
-      "--jars", "src/test/resources/TestUDTF.jar",
+      "--jars", testJar,
       "/home/thejar.jar",
       "arg1")
     val appArgs = new SparkSubmitArguments(clArgs)
@@ -561,6 +565,8 @@ class SparkSubmitSuite
   test("SPARK-47475: Avoid jars download if scheme matches " +
     "spark.kubernetes.jars.avoidDownloadSchemes " +
     "in k8s client mode & driver runs inside a POD") {
+    val testJar = "src/test/resources/TestUDTF.jar"
+    assume(new File(testJar).exists)
     val hadoopConf = new Configuration()
     updateConfWithFakeS3Fs(hadoopConf)
     withTempDir { tmpDir =>
@@ -579,7 +585,7 @@ class SparkSubmitSuite
         "--files", "src/test/resources/test_metrics_config.properties",
         "--py-files", "src/test/resources/test_metrics_system.properties",
         "--archives", "src/test/resources/log4j2.properties",
-        "--jars", s"src/test/resources/TestUDTF.jar,$remoteJarFile",
+        "--jars", s"$testJar,$remoteJarFile",
         "/home/jarToIgnore.jar",
         "arg1")
       val appArgs = new SparkSubmitArguments(clArgs)

core/src/test/scala/org/apache/spark/deploy/SparkSubmitSuite.scala

@@ -29,21 +29,28 @@ class ClassLoaderIsolationSuite extends SparkFunSuite with LocalSparkContext  {
     .take(2)
     .mkString(".")
 
-  val jar1 = Thread.currentThread().getContextClassLoader.getResource("TestUDTF.jar").toString
+  private val jarURL1 = Thread.currentThread().getContextClassLoader.getResource("TestUDTF.jar")
+  private lazy val jar1 = jarURL1.toString
 
   // package com.example
   // object Hello { def test(): Int = 2 }
   // case class Hello(x: Int, y: Int)
-  val jar2 = Thread.currentThread().getContextClassLoader
-    .getResource(s"TestHelloV2_$scalaVersion.jar").toString
+  private val jarURL2 = Thread.currentThread().getContextClassLoader
+    .getResource(s"TestHelloV2_$scalaVersion.jar")
+  private lazy val jar2 = jarURL2.toString
 
   // package com.example
   // object Hello { def test(): Int = 3 }
   // case class Hello(x: String)
-  val jar3 = Thread.currentThread().getContextClassLoader
-    .getResource(s"TestHelloV3_$scalaVersion.jar").toString
+  private val jarURL3 = Thread.currentThread().getContextClassLoader
+    .getResource(s"TestHelloV3_$scalaVersion.jar")
+  private lazy val jar3 = jarURL3.toString
 
   test("Executor classloader isolation with JobArtifactSet") {
+    assume(jarURL1 != null)
+    assume(jarURL2 != null)
+    assume(jarURL3 != null)
+
     sc = new SparkContext(new SparkConf().setAppName("test").setMaster("local"))
     sc.addJar(jar1)
     sc.addJar(jar2)

core/src/test/scala/org/apache/spark/executor/ClassLoaderIsolationSuite.scala

@@ -1573,6 +1573,8 @@ class TaskSetManagerSuite
   }
 
   test("SPARK-21563 context's added jars shouldn't change mid-TaskSet") {
+    val jarPath = Thread.currentThread().getContextClassLoader.getResource("TestUDTF.jar")
+    assume(jarPath != null)
     sc = new SparkContext("local", "test")
     val addedJarsPreTaskSet = Map[String, Long](sc.allAddedJars.toSeq: _*)
     assert(addedJarsPreTaskSet.size === 0)
@@ -1588,7 +1590,6 @@ class TaskSetManagerSuite
     assert(taskOption2.get.artifacts.jars === addedJarsPreTaskSet)
 
     // even with a jar added mid-TaskSet
-    val jarPath = Thread.currentThread().getContextClassLoader.getResource("TestUDTF.jar")
     sc.addJar(jarPath.toString)
     val addedJarsMidTaskSet = Map[String, Long](sc.allAddedJars.toSeq: _*)
     assert(addedJarsPreTaskSet !== addedJarsMidTaskSet)

core/src/test/scala/org/apache/spark/scheduler/TaskSetManagerSuite.scala

@@ -106,6 +106,8 @@ sed -i".tmp7" 's/SPARK_VERSION:.*$/SPARK_VERSION: '"$NEXT_VERSION"'/g' docs/_con
 sed -i".tmp8" 's/SPARK_VERSION_SHORT:.*$/SPARK_VERSION_SHORT: '"$R_NEXT_VERSION"'/g' docs/_config.yml
 # Update the version index of DocSearch as the short version
 sed -i".tmp9" "s/'facetFilters':.*$/'facetFilters': [\"version:$R_NEXT_VERSION\"]/g" docs/_config.yml
+# Remove test jars that do not beling to source releases.
+rm $(<dev/test-jars.txt)
 
 git commit -a -m "Preparing development version $NEXT_VERSION"
 

dev/create-release/release-tag.sh

@@ -0,0 +1,17 @@
+core/src/test/resources/TestHelloV2_2.13.jar
+core/src/test/resources/TestHelloV3_2.13.jar
+core/src/test/resources/TestUDTF.jar
+data/artifact-tests/junitLargeJar.jar
+data/artifact-tests/smallJar.jar
+sql/connect/client/jvm/src/test/resources/TestHelloV2_2.13.jar
+sql/connect/client/jvm/src/test/resources/udf2.13.jar
+sql/connect/common/src/test/resources/artifact-tests/junitLargeJar.jar
+sql/connect/common/src/test/resources/artifact-tests/smallJar.jar
+sql/core/src/test/resources/SPARK-33084.jar
+sql/core/src/test/resources/artifact-tests/udf_noA.jar
+sql/hive-thriftserver/src/test/resources/TestUDTF.jar
+sql/hive/src/test/noclasspath/hive-test-udfs.jar
+sql/hive/src/test/resources/SPARK-21101-1.0.jar
+sql/hive/src/test/resources/TestUDTF.jar
+sql/hive/src/test/resources/data/files/TestSerDe.jar
+sql/hive/src/test/resources/regression-test-SPARK-8489/test-2.13.jar

dev/test-jars.txt

@@ -224,6 +224,8 @@ def conf(cls):
     def test_basic_requests(self):
         file_name = "smallJar"
         small_jar_path = os.path.join(self.artifact_file_path, f"{file_name}.jar")
+        if not os.path.isfile(small_jar_path):
+            raise unittest.SkipTest(f"Skipped as {small_jar_path} does not exist.")
         response = self.artifact_manager._retrieve_responses(
             self.artifact_manager._create_requests(
                 small_jar_path, pyfile=False, archive=False, file=False
@@ -235,6 +237,8 @@ def test_single_chunk_artifact(self):
         file_name = "smallJar"
         small_jar_path = os.path.join(self.artifact_file_path, f"{file_name}.jar")
         small_jar_crc_path = os.path.join(self.artifact_crc_path, f"{file_name}.txt")
+        if not os.path.isfile(small_jar_path):
+            raise unittest.SkipTest(f"Skipped as {small_jar_path} does not exist.")
 
         requests = list(
             self.artifact_manager._create_requests(
@@ -261,6 +265,8 @@ def test_chunked_artifacts(self):
         file_name = "junitLargeJar"
         large_jar_path = os.path.join(self.artifact_file_path, f"{file_name}.jar")
         large_jar_crc_path = os.path.join(self.artifact_crc_path, f"{file_name}.txt")
+        if not os.path.isfile(large_jar_path):
+            raise unittest.SkipTest(f"Skipped as {large_jar_path} does not exist.")
 
         requests = list(
             self.artifact_manager._create_requests(
@@ -296,6 +302,8 @@ def test_batched_artifacts(self):
         file_name = "smallJar"
         small_jar_path = os.path.join(self.artifact_file_path, f"{file_name}.jar")
         small_jar_crc_path = os.path.join(self.artifact_crc_path, f"{file_name}.txt")
+        if not os.path.isfile(small_jar_path):
+            raise unittest.SkipTest(f"Skipped as {small_jar_path} does not exist.")
 
         requests = list(
             self.artifact_manager._create_requests(
@@ -333,6 +341,10 @@ def test_single_chunked_and_chunked_artifact(self):
         large_jar_path = os.path.join(self.artifact_file_path, f"{file_name2}.jar")
         large_jar_crc_path = os.path.join(self.artifact_crc_path, f"{file_name2}.txt")
         large_jar_size = os.path.getsize(large_jar_path)
+        if not os.path.isfile(small_jar_path):
+            raise unittest.SkipTest(f"Skipped as {small_jar_path} does not exist.")
+        if not os.path.isfile(large_jar_path):
+            raise unittest.SkipTest(f"Skipped as {large_jar_path} does not exist.")
 
         requests = list(
             self.artifact_manager._create_requests(