[SPARK-51623][BUILD] Remove class files in source releases

### What changes were proposed in this pull request?

This PR proposes to remove test class files in source releases during release process.

### Why are the changes needed?

Apache source releases must not contained class files.
The issue is discussed on https://lists.apache.org/thread/0ro5yn6lbbpmvmqp2px3s2pf7cwljlc4

### Does this PR introduce _any_ user-facing change?

To end users, no.

### How was this patch tested?

Will be tested in CI.

### Was this patch authored or co-authored using generative AI tooling?

No.

Closes apache#50422 from HyukjinKwon/SPARK-51623.

Lead-authored-by: Hyukjin Kwon <[email protected]>
Co-authored-by: Hyukjin Kwon <[email protected]>
Signed-off-by: Hyukjin Kwon <[email protected]>

Author: HyukjinKwon

dev/create-release/release-tag.sh

@@ -75,7 +75,8 @@ git config user.email "$GIT_EMAIL"
 
 # Remove test jars that do not belong to source releases.
 rm $(<dev/test-jars.txt)
-git commit -a -m "Removing test jars"
+rm $(<dev/test-classes.txt)
+git commit -a -m "Removing test jars and class files"
 JAR_RM_REF=$(git rev-parse HEAD)
 
 # Create release version

dev/is-changed.py

@@ -80,6 +80,20 @@ def main():
                     "please add the file name into dev/test-jars.txt."
                 )
 
+    if any(f.endswith(".class") for f in changed_files):
+        with open(
+            os.path.join(os.path.dirname(os.path.realpath(__file__)), "test-classes.txt")
+        ) as clslist:
+            itrsect = set((line.strip() for line in clslist.readlines())).intersection(
+                set(changed_files)
+            )
+            if len(itrsect) > 0:
+                raise SystemExit(
+                    f"Cannot include class files in source codes ({', '.join(itrsect)}). "
+                    "If they have to be added temporarily, "
+                    "please add the file name into dev/test-classes.txt."
+                )
+
     changed_modules = determine_modules_to_test(
         determine_modules_for_files(changed_files), deduplicated=False
     )

dev/test-classes.txt

@@ -0,0 +1,8 @@
+repl/src/test/resources/IntSumUdf.class
+sql/core/src/test/resources/artifact-tests/Hello.class
+sql/core/src/test/resources/artifact-tests/IntSumUdf.class
+sql/core/src/test/resources/artifact-tests/smallClassFile.class
+sql/connect/common/src/test/resources/artifact-tests/Hello.class
+sql/core/src/test/resources/artifact-tests/HelloWithPackage.class
+sql/connect/common/src/test/resources/artifact-tests/smallClassFile.class
+sql/connect/common/src/test/resources/artifact-tests/smallClassFileDup.class

repl/src/test/scala/org/apache/spark/repl/ReplSuite.scala

@@ -400,6 +400,7 @@ class ReplSuite extends SparkFunSuite {
   test("register UDF via SparkSession.addArtifact") {
     val artifactPath = new File("src/test/resources").toPath
     val intSumUdfPath = artifactPath.resolve("IntSumUdf.class")
+    assume(intSumUdfPath.toFile.exists)
     val output = runInterpreterInPasteMode("local",
       s"""
          |import org.apache.spark.sql.api.java.UDF2
@@ -438,6 +439,7 @@ class ReplSuite extends SparkFunSuite {
   test("register a class via SparkSession.addArtifact") {
     val artifactPath = new File("src/test/resources").toPath
     val intSumUdfPath = artifactPath.resolve("IntSumUdf.class")
+    assume(intSumUdfPath.toFile.exists)
     val output = runInterpreterInPasteMode("local",
       s"""
          |import org.apache.spark.sql.functions.udf

sql/connect/client/jvm/src/test/scala/org/apache/spark/sql/connect/client/ArtifactSuite.scala

@@ -199,10 +199,12 @@ class ArtifactSuite extends ConnectFunSuite with BeforeAndAfterEach {
   }
 
   test("Batched SingleChunkArtifacts") {
-    val file1 = artifactFilePath.resolve("smallClassFile.class").toUri
-    val path = artifactFilePath.resolve("smallJar.jar")
-    val file2 = path.toUri
-    assume(path.toFile.exists)
+    val path1 = artifactFilePath.resolve("smallClassFile.class")
+    assume(path1.toFile.exists)
+    val file1 = path1.toUri
+    val path2 = artifactFilePath.resolve("smallJar.jar")
+    assume(path2.toFile.exists)
+    val file2 = path2.toUri
     artifactManager.addArtifacts(Seq(file1, file2))
     val receivedRequests = service.getAndClearLatestAddArtifactRequests()
     // Single request containing 2 artifacts.
@@ -223,11 +225,15 @@ class ArtifactSuite extends ConnectFunSuite with BeforeAndAfterEach {
   }
 
   test("Mix of SingleChunkArtifact and chunked artifact") {
-    val file1 = artifactFilePath.resolve("smallClassFile.class").toUri
+    val path1 = artifactFilePath.resolve("smallClassFile.class")
+    assume(path1.toFile.exists)
+    val file1 = path1.toUri
     val path2 = artifactFilePath.resolve("junitLargeJar.jar")
     assume(path2.toFile.exists)
     val file2 = path2.toUri
-    val file3 = artifactFilePath.resolve("smallClassFileDup.class").toUri
+    val path3 = artifactFilePath.resolve("smallClassFileDup.class")
+    assume(path3.toFile.exists)
+    val file3 = path3.toUri
     val path4 = artifactFilePath.resolve("smallJar.jar")
     assume(path4.toFile.exists)
     val file4 = path4.toUri
@@ -298,6 +304,7 @@ class ArtifactSuite extends ConnectFunSuite with BeforeAndAfterEach {
 
   test("artifact with custom target") {
     val artifactPath = artifactFilePath.resolve("smallClassFile.class")
+    assume(artifactPath.toFile.exists)
     val target = "sub/package/smallClassFile.class"
     artifactManager.addArtifact(artifactPath.toString, target)
     val receivedRequests = service.getAndClearLatestAddArtifactRequests()
@@ -318,6 +325,7 @@ class ArtifactSuite extends ConnectFunSuite with BeforeAndAfterEach {
 
   test("in-memory artifact with custom target") {
     val artifactPath = artifactFilePath.resolve("smallClassFile.class")
+    assume(artifactPath.toFile.exists)
     val artifactBytes = Files.readAllBytes(artifactPath)
     val target = "sub/package/smallClassFile.class"
     artifactManager.addArtifact(artifactBytes, target)
@@ -341,6 +349,7 @@ class ArtifactSuite extends ConnectFunSuite with BeforeAndAfterEach {
     "When both source and target paths are given, extension conditions are checked " +
       "on target path") {
     val artifactPath = artifactFilePath.resolve("smallClassFile.class")
+    assume(artifactPath.toFile.exists)
     assertThrows[UnsupportedOperationException] {
       artifactManager.addArtifact(artifactPath.toString, "dummy.extension")
     }

sql/connect/client/jvm/src/test/scala/org/apache/spark/sql/connect/client/ClassFinderSuite.scala

@@ -37,6 +37,7 @@ class ClassFinderSuite extends ConnectFunSuite {
         "Hello.class",
         "smallClassFile.class",
         "smallClassFileDup.class") ++ additionalClasses).map(name => Paths.get(name))
+      expectedClassFiles.foreach(p => assume(p.toFile.exists))
 
       val foundArtifacts = monitor.findClasses().toSeq
       assert(expectedClassFiles.forall { classPath =>

sql/connect/client/jvm/src/test/scala/org/apache/spark/sql/connect/client/SparkConnectClientSuite.scala

@@ -487,7 +487,9 @@ class SparkConnectClientSuite extends ConnectFunSuite with BeforeAndAfterEach {
 
     val session = SparkSession.builder().client(client).create()
     val artifactFilePath = commonResourcePath.resolve("artifact-tests")
-    session.addArtifact(artifactFilePath.resolve("smallClassFile.class").toString)
+    val path = artifactFilePath.resolve("smallClassFile.class")
+    assume(path.toFile.exists)
+    session.addArtifact(path.toString)
   }
 
   private def buildPlan(query: String): proto.Plan = {

sql/connect/server/src/test/scala/org/apache/spark/sql/connect/service/AddArtifactsHandlerSuite.scala

@@ -193,6 +193,7 @@ class AddArtifactsHandlerSuite extends SharedSparkSession with ResourceHelper {
     try {
       val name = "classes/smallClassFile.class"
       val artifactPath = inputFilePath.resolve("smallClassFile.class")
+      assume(artifactPath.toFile.exists)
       addSingleChunkArtifact(handler, name, artifactPath)
       handler.onCompleted()
       val response = ThreadUtils.awaitResult(promise.future, 5.seconds)
@@ -246,15 +247,12 @@ class AddArtifactsHandlerSuite extends SharedSparkSession with ResourceHelper {
         "classes/smallClassFileDup.class",
         "jars/smallJar.jar")
 
-      val path1 = inputFilePath.resolve("junitLargeJar.jar")
-      val path2 = inputFilePath.resolve("smallJar.jar")
       val artifactPaths = Seq(
         inputFilePath.resolve("smallClassFile.class"),
-        path1,
+        inputFilePath.resolve("junitLargeJar.jar"),
         inputFilePath.resolve("smallClassFileDup.class"),
-        path2)
-      assume(path1.toFile.exists)
-      assume(path2.toFile.exists)
+        inputFilePath.resolve("smallJar.jar"))
+      artifactPaths.foreach(p => assume(p.toFile.exists))
 
       addSingleChunkArtifact(handler, names.head, artifactPaths.head)
       addChunkedArtifact(handler, names(1), artifactPaths(1))
@@ -286,6 +284,7 @@ class AddArtifactsHandlerSuite extends SharedSparkSession with ResourceHelper {
     try {
       val name = "classes/smallClassFile.class"
       val artifactPath = inputFilePath.resolve("smallClassFile.class")
+      assume(artifactPath.toFile.exists)
       val dataChunks = getDataChunks(artifactPath)
       assert(dataChunks.size == 1)
       val bytes = dataChunks.head

sql/core/src/test/scala/org/apache/spark/sql/artifact/ArtifactManagerSuite.scala

@@ -53,11 +53,13 @@ class ArtifactManagerSuite extends SharedSparkSession {
   }
 
   test("Class artifacts are added to the correct directory.") {
+    assume(artifactPath.resolve("smallClassFile.class").toFile.exists)
+
     val copyDir = Utils.createTempDir().toPath
     FileUtils.copyDirectory(artifactPath.toFile, copyDir.toFile)
     val stagingPath = copyDir.resolve("smallClassFile.class")
-    val remotePath = Paths.get("classes/smallClassFile.class")
     assert(stagingPath.toFile.exists())
+    val remotePath = Paths.get("classes/smallClassFile.class")
     artifactManager.addArtifact(remotePath, stagingPath, None)
 
     val movedClassFile = ArtifactManager.artifactRootDirectory
@@ -67,11 +69,13 @@ class ArtifactManagerSuite extends SharedSparkSession {
   }
 
   test("Class file artifacts are added to SC classloader") {
+    assume(artifactPath.resolve("Hello.class").toFile.exists)
+
     val copyDir = Utils.createTempDir().toPath
     FileUtils.copyDirectory(artifactPath.toFile, copyDir.toFile)
     val stagingPath = copyDir.resolve("Hello.class")
-    val remotePath = Paths.get("classes/Hello.class")
     assert(stagingPath.toFile.exists())
+    val remotePath = Paths.get("classes/Hello.class")
     artifactManager.addArtifact(remotePath, stagingPath, None)
 
     val movedClassFile = ArtifactManager.artifactRootDirectory
@@ -91,11 +95,13 @@ class ArtifactManagerSuite extends SharedSparkSession {
   }
 
   test("UDF can reference added class file") {
+    assume(artifactPath.resolve("Hello.class").toFile.exists)
+
     val copyDir = Utils.createTempDir().toPath
     FileUtils.copyDirectory(artifactPath.toFile, copyDir.toFile)
     val stagingPath = copyDir.resolve("Hello.class")
-    val remotePath = Paths.get("classes/Hello.class")
     assert(stagingPath.toFile.exists())
+    val remotePath = Paths.get("classes/Hello.class")
 
     artifactManager.addArtifact(remotePath, stagingPath, None)
 
@@ -170,6 +176,8 @@ class ArtifactManagerSuite extends SharedSparkSession {
   }
 
   test("SPARK-43790: Forward artifact file to cloud storage path") {
+    assume(artifactPath.resolve("smallClassFile.class").toFile.exists)
+
     val copyDir = Utils.createTempDir().toPath
     val destFSDir = Utils.createTempDir().toPath
     FileUtils.copyDirectory(artifactPath.toFile, copyDir.toFile)
@@ -184,6 +192,8 @@ class ArtifactManagerSuite extends SharedSparkSession {
   }
 
   test("Removal of resources") {
+    assume(artifactPath.resolve("smallClassFile.class").toFile.exists)
+
     withTempPath { path =>
       // Setup cache
       val stagingPath = path.toPath
@@ -228,6 +238,8 @@ class ArtifactManagerSuite extends SharedSparkSession {
   }
 
   test("Classloaders for spark sessions are isolated") {
+    assume(artifactPath.resolve("Hello.class").toFile.exists)
+
     val session1 = spark.newSession()
     val session2 = spark.newSession()
     val session3 = spark.newSession()
@@ -284,6 +296,8 @@ class ArtifactManagerSuite extends SharedSparkSession {
   }
 
   test("SPARK-44300: Cleaning up resources only deletes session-specific resources") {
+    assume(artifactPath.resolve("Hello.class").toFile.exists)
+
     val copyDir = Utils.createTempDir().toPath
     FileUtils.copyDirectory(artifactPath.toFile, copyDir.toFile)
     val stagingPath = copyDir.resolve("Hello.class")
@@ -335,7 +349,9 @@ class ArtifactManagerSuite extends SharedSparkSession {
   }
 
   test("Added artifact can be loaded by the current SparkSession") {
-    val buffer = Files.readAllBytes(artifactPath.resolve("IntSumUdf.class"))
+    val path = artifactPath.resolve("IntSumUdf.class")
+    assume(path.toFile.exists)
+    val buffer = Files.readAllBytes(path)
     spark.addArtifact(buffer, "IntSumUdf.class")
 
     spark.udf.registerJava("intSum", "IntSumUdf", DataTypes.LongType)
@@ -350,6 +366,8 @@ class ArtifactManagerSuite extends SharedSparkSession {
   private def testAddArtifactToLocalSession(
       classFileToUse: String, binaryName: String)(addFunc: Path => String): Unit = {
     val copyDir = Utils.createTempDir().toPath
+    assume(artifactPath.resolve(classFileToUse).toFile.exists)
+
     FileUtils.copyDirectory(artifactPath.toFile, copyDir.toFile)
     val classPath = copyDir.resolve(classFileToUse)
     assert(classPath.toFile.exists())
@@ -395,8 +413,10 @@ class ArtifactManagerSuite extends SharedSparkSession {
       Files.write(randomFilePath, testBytes)
 
       // Register multiple kinds of artifacts
+      val clsPath = path.resolve("Hello.class")
+      assume(clsPath.toFile.exists)
       artifactManager.addArtifact( // Class
-        Paths.get("classes/Hello.class"), path.resolve("Hello.class"), None)
+        Paths.get("classes/Hello.class"), clsPath, None)
       artifactManager.addArtifact( // Python
         Paths.get("pyfiles/abc.zip"), randomFilePath, None, deleteStagedFile = false)
       val jarPath = Paths.get("jars/udf_noA.jar")