[CELEBORN-1413][FOLLOWUP] Bump spark 4.0 version to 4.0.0

.github/workflows/maven.yml

@@ -146,6 +146,43 @@ jobs:
           **/target/test-reports/**
           **/target/unit-tests.log
 
+  spark4:
+    runs-on: ubuntu-22.04
+    strategy:
+      fail-fast: false
+      matrix:
+        java:
+          - 17
+        spark:
+          - '4.0'
+        shuffle-plugin-class:
+          - 'org.apache.spark.shuffle.celeborn.CelebornShuffleDataIO'
+    steps:
+      - uses: actions/checkout@v4
+      - name: Setup JDK ${{ matrix.java }}
+        uses: actions/setup-java@v4
+        with:
+          distribution: zulu
+          java-version: ${{ matrix.java }}
+          cache: maven
+          check-latest: false
+      - name: Test with Maven
+        run: |
+          SPARK_BINARY_VERSION=${{ matrix.spark }}
+          SPARK_MAJOR_VERSION=${SPARK_BINARY_VERSION%%.*}
+          PROFILES="-Pgoogle-mirror,spark-${{ matrix.spark }}"
+          TEST_MODULES="client-spark/common,client-spark/spark-3,client-spark/spark-3-columnar-common,client-spark/spark-${SPARK_MAJOR_VERSION}-shaded,tests/spark-it"
+          build/mvn $PROFILES -pl $TEST_MODULES -am clean install -DskipTests
+          build/mvn $PROFILES -pl $TEST_MODULES -Dspark.shuffle.sort.io.plugin.class=${{ matrix.shuffle-plugin-class }} test
+      - name: Upload test log
+        if: failure()
+        uses: actions/upload-artifact@v4
+        with:
+          name: spark-${{ matrix.spark }}-java-${{ matrix.java }}-unit-test-log
+          path: |
+            **/target/test-reports/**
+            **/target/unit-tests.log
+
   flink1:
     runs-on: ubuntu-22.04
     strategy:

.github/workflows/sbt.yml

@@ -196,6 +196,42 @@ jobs:
               **/target/test-reports/**
               **/target/unit-tests.log
 
+  spark4:
+    runs-on: ubuntu-22.04
+    strategy:
+      fail-fast: false
+      matrix:
+        spark:
+          - '4.0'
+        java:
+          - 17
+        scala-binary:
+          - '2.13'
+        shuffle-plugin-class:
+          - 'org.apache.spark.shuffle.celeborn.CelebornShuffleDataIO'
+        include:
+          - spark: '4.0'
+            scala: '2.13.16'
+    steps:
+      - uses: actions/checkout@v4
+      - name: Setup JDK ${{ matrix.java }}
+        uses: actions/setup-java@v4
+        with:
+          distribution: zulu
+          java-version: ${{ matrix.java }}
+          check-latest: false
+      - name: Test with SBT
+        run: |
+          build/sbt -Dspark.shuffle.plugin.class=${{ matrix.shuffle-plugin-class }} -Pspark-${{ matrix.spark }} ++${{ matrix.scala }} "clean; celeborn-spark-group/test"
+      - name: Upload test log
+        if: failure()
+        uses: actions/upload-artifact@v4
+        with:
+          name: spark-${{ matrix.spark }}-java-${{ matrix.java }}-scala-${{ matrix.scala }}-unit-test-log
+          path: |
+            **/target/test-reports/**
+            **/target/unit-tests.log
+
   flink1:
     runs-on: ubuntu-22.04
     strategy:

LICENSE-binary

@@ -362,3 +362,7 @@ EDL 1.0
 --------------------------------
 See licenses-binary/LICENSE-mimepull.txt for detail.
 org.jvnet.mimepull:mimepull
+
+CDDL License 1.1
+----------------
+javax.servlet-api

build/make-distribution.sh

@@ -346,6 +346,9 @@ if [ "$SBT_ENABLED" == "true" ]; then
     sbt_build_client -Pspark-2.4
     sbt_build_client -Pspark-3.4
     sbt_build_client -Pspark-3.5
+    export JAVA_HOME=$JAVA17_HOME
+    sbt_build_client -Pspark-4.0
+    export JAVA_HOME=$JAVA8_HOME
     sbt_build_client -Pflink-1.16
     sbt_build_client -Pflink-1.17
     sbt_build_client -Pflink-1.18
@@ -384,6 +387,9 @@ else
     build_spark_client -Pspark-2.4
     build_spark_client -Pspark-3.4
     build_spark_client -Pspark-3.5
+    export JAVA_HOME=$JAVA17_HOME
+    build_spark_client -Pspark-4.0
+    export JAVA_HOME=$JAVA8_HOME
     build_flink_client -Pflink-1.16
     build_flink_client -Pflink-1.17
     build_flink_client -Pflink-1.18

build/release/release.sh

@@ -28,6 +28,7 @@ ASF_PASSWORD=${ASF_PASSWORD:?"ASF_PASSWORD is required"}
 RELEASE_RC_NO=${RELEASE_RC_NO:?"RELEASE_RC_NO is required, e.g. 0"}
 JAVA8_HOME=${JAVA8_HOME:?"JAVA8_HOME is required"}
 JAVA11_HOME=${JAVA11_HOME:?"JAVA11_HOME is required"}
+JAVA17_HOME=${JAVA17_HOME:?"JAVA17_HOME is required"}
 
 RELEASE_VERSION=$(awk -F'"' '/ThisBuild \/ version/ {print $2}' version.sbt)
 
@@ -110,6 +111,11 @@ upload_nexus_staging() {
   echo "Deploying celeborn-client-spark-3-shaded_2.13"
   ${PROJECT_DIR}/build/sbt -Pspark-3.4 ++2.13.8 "clean;celeborn-client-spark-3-shaded/publishSigned"
 
+  export JAVA_HOME=$JAVA17_HOME
+  echo "Deploying celeborn-client-spark-4-shaded_2.13"
+  ${PROJECT_DIR}/build/sbt -Pspark-4.0 "clean;celeborn-client-spark-4-shaded/publishSigned"
+  export JAVA_HOME=$JAVA8_HOME
+
   echo "Deploying celeborn-client-flink-1.16-shaded_2.12"
   ${PROJECT_DIR}/build/sbt -Pflink-1.16 "clean;celeborn-client-flink-1_16-shaded/publishSigned"
 

client-mr/mr/pom.xml

@@ -55,6 +55,10 @@
         </exclusion>
       </exclusions>
     </dependency>
+    <dependency>
+      <groupId>javax.servlet</groupId>
+      <artifactId>javax.servlet-api</artifactId>
+    </dependency>
   </dependencies>
 
   <profiles>

client-spark/spark-3-columnar-shuffle/pom.xml

@@ -54,6 +54,16 @@
       <type>test-jar</type>
       <scope>test</scope>
     </dependency>
+    <dependency>
+      <groupId>javax.servlet</groupId>
+      <artifactId>javax.servlet-api</artifactId>
+      <scope>test</scope>
+    </dependency>
+    <dependency>
+      <groupId>jakarta.servlet</groupId>
+      <artifactId>jakarta.servlet-api</artifactId>
+      <scope>test</scope>
+    </dependency>
     <dependency>
       <groupId>org.mockito</groupId>
       <artifactId>mockito-core</artifactId>

client-spark/spark-3/pom.xml

@@ -86,6 +86,16 @@
         </exclusion>
       </exclusions>
     </dependency>
+    <dependency>
+      <groupId>javax.servlet</groupId>
+      <artifactId>javax.servlet-api</artifactId>
+      <scope>test</scope>
+    </dependency>
+    <dependency>
+      <groupId>jakarta.servlet</groupId>
+      <artifactId>jakarta.servlet-api</artifactId>
+      <scope>test</scope>
+    </dependency>
     <dependency>
       <groupId>org.mockito</groupId>
       <artifactId>mockito-core</artifactId>

client-spark/spark-4-columnar-shuffle/src/main/scala/org/apache/spark/sql/execution/columnar/CelebornColumnType.scala

@@ -440,7 +440,7 @@ private[columnar] trait DirectCopyCelebornColumnType[JvmType] extends CelebornCo
 
 private[columnar] object CELEBORN_STRING
   extends NativeCelebornColumnType(
-    PhysicalStringType(SqlApiConf.get.defaultStringType.collationId),
+    PhysicalStringType(StringType.collationId),
     8)
   with DirectCopyCelebornColumnType[UTF8String] {
 

client-tez/tez/pom.xml

@@ -188,6 +188,10 @@
         </exclusion>
       </exclusions>
     </dependency>
+    <dependency>
+      <groupId>javax.servlet</groupId>
+      <artifactId>javax.servlet-api</artifactId>
+    </dependency>
     <dependency>
       <groupId>org.mockito</groupId>
       <artifactId>mockito-core</artifactId>

client/src/main/scala/org/apache/celeborn/client/ApplicationHeartbeater.scala

@@ -170,7 +170,7 @@ class ApplicationHeartbeater(
   }
 
   def stop(): Unit = {
-    stopped.synchronized {
+    this.synchronized {
       if (!stopped) {
         // Stop appHeartbeat first
         logInfo(s"Stop Application heartbeat $appId")

dev/deps/dependencies-client-mr

@@ -83,7 +83,7 @@ javax-websocket-client-impl/9.4.51.v20230217//javax-websocket-client-impl-9.4.51
 javax-websocket-server-impl/9.4.51.v20230217//javax-websocket-server-impl-9.4.51.v20230217.jar
 javax.annotation-api/1.3.2//javax.annotation-api-1.3.2.jar
 javax.inject/1//javax.inject-1.jar
-javax.servlet-api/3.1.0//javax.servlet-api-3.1.0.jar
+javax.servlet-api/4.0.1//javax.servlet-api-4.0.1.jar
 javax.websocket-api/1.0//javax.websocket-api-1.0.jar
 javax.websocket-client-api/1.0//javax.websocket-client-api-1.0.jar
 jaxb-api/2.2.11//jaxb-api-2.2.11.jar

dev/deps/dependencies-client-tez

@@ -64,7 +64,7 @@ jackson-module-scala_2.12/2.15.3//jackson-module-scala_2.12-2.15.3.jar
 jackson-xc/1.9.2//jackson-xc-1.9.2.jar
 jakarta.activation-api/1.2.1//jakarta.activation-api-1.2.1.jar
 jakarta.activation/1.2.2//jakarta.activation-1.2.2.jar
-javax.servlet-api/3.1.0//javax.servlet-api-3.1.0.jar
+javax.servlet-api/4.0.1//javax.servlet-api-4.0.1.jar
 jaxb-api/2.2.2//jaxb-api-2.2.2.jar
 jaxb-impl/2.2.3-1//jaxb-impl-2.2.3-1.jar
 jcip-annotations/1.0-1//jcip-annotations-1.0-1.jar

dev/deps/dependencies-server

@@ -48,11 +48,12 @@ jackson-module-jaxb-annotations/2.15.3//jackson-module-jaxb-annotations-2.15.3.j
 jackson-module-scala_2.12/2.15.3//jackson-module-scala_2.12-2.15.3.jar
 jakarta.annotation-api/1.3.5//jakarta.annotation-api-1.3.5.jar
 jakarta.inject/2.6.1//jakarta.inject-2.6.1.jar
-jakarta.servlet-api/4.0.4//jakarta.servlet-api-4.0.4.jar
+jakarta.servlet-api/5.0.0//jakarta.servlet-api-5.0.0.jar
 jakarta.validation-api/2.0.2//jakarta.validation-api-2.0.2.jar
 jakarta.ws.rs-api/2.1.6//jakarta.ws.rs-api-2.1.6.jar
 jakarta.xml.bind-api/2.3.3//jakarta.xml.bind-api-2.3.3.jar
 javassist/3.29.0-GA//javassist-3.29.0-GA.jar
+javax.servlet-api/4.0.1//javax.servlet-api-4.0.1.jar
 jcl-over-slf4j/1.7.36//jcl-over-slf4j-1.7.36.jar
 jersey-client/2.39.1//jersey-client-2.39.1.jar
 jersey-common/2.39.1//jersey-common-2.39.1.jar

pom.xml

@@ -121,8 +121,11 @@
     <swagger-ui.version>4.9.1</swagger-ui.version>
     <jersey.version>2.39.1</jersey.version>
     <jetty.version>9.4.56.v20240826</jetty.version>
-    <jakarta.servlet-api.version>4.0.4</jakarta.servlet-api.version>
+    <javax.servlet-api.version>4.0.1</javax.servlet-api.version>
+    <!-- 6.0.0 requires JDK 11 -->
+    <jakarta.servlet-api.version>5.0.0</jakarta.servlet-api.version>
     <jakarta.annotation-api.version>1.3.5</jakarta.annotation-api.version>
+    <jakarta.ws.rs-api.version>2.1.6</jakarta.ws.rs-api.version>
     <!-- openapi-generator dependencies -->
     <openapitools.jackson-databind-nullable.version>0.2.6</openapitools.jackson-databind-nullable.version>
     <httpclient5.version>5.3.1</httpclient5.version>
@@ -149,7 +152,7 @@
     <maven.plugin.shade.version>3.5.2</maven.plugin.shade.version>
     <maven.plugin.spotless.version>2.24.1</maven.plugin.spotless.version>
     <maven.plugin.surefire.version>3.0.0-M7</maven.plugin.surefire.version>
-    <maven.plugin.silencer.version>1.7.13</maven.plugin.silencer.version>
+    <maven.plugin.silencer.version>1.7.19</maven.plugin.silencer.version>
     <maven.plugin.resources.version>3.3.1</maven.plugin.resources.version>
     <openapi.generator.version>7.8.0</openapi.generator.version>
 
@@ -498,6 +501,12 @@
       </dependency>
 
       <!-- RESTful service dependencies -->
+      <dependency>
+        <groupId>javax.servlet</groupId>
+        <artifactId>javax.servlet-api</artifactId>
+        <version>${javax.servlet-api.version}</version>
+      </dependency>
+
       <dependency>
         <groupId>jakarta.servlet</groupId>
         <artifactId>jakarta.servlet-api</artifactId>
@@ -510,6 +519,12 @@
         <version>${jakarta.annotation-api.version}</version>
       </dependency>
 
+      <dependency>
+        <groupId>jakarta.ws.rs</groupId>
+        <artifactId>jakarta.ws.rs-api</artifactId>
+        <version>${jakarta.ws.rs-api.version}</version>
+      </dependency>
+
       <dependency>
         <groupId>org.openapitools</groupId>
         <artifactId>jackson-databind-nullable</artifactId>
@@ -585,6 +600,10 @@
             <groupId>jakarta.activation</groupId>
             <artifactId>jakarta.activation-api</artifactId>
           </exclusion>
+          <exclusion>
+            <groupId>jakarta.servlet</groupId>
+            <artifactId>jakarta.servlet-api</artifactId>
+          </exclusion>
         </exclusions>
       </dependency>
 
@@ -909,6 +928,7 @@
               <java.io.tmpdir>${project.build.directory}/tmp</java.io.tmpdir>
               <spark.driver.memory>8g</spark.driver.memory>
               <spark.shuffle.sort.io.plugin.class>${spark.shuffle.plugin.class}</spark.shuffle.sort.io.plugin.class>
+              <spark.ui.enabled>false</spark.ui.enabled>
             </systemProperties>
             <environmentVariables>
               <CELEBORN_LOCAL_HOSTNAME>localhost</CELEBORN_LOCAL_HOSTNAME>
@@ -948,6 +968,7 @@
               <java.io.tmpdir>${project.build.directory}/tmp</java.io.tmpdir>
               <spark.driver.memory>8g</spark.driver.memory>
               <spark.shuffle.sort.io.plugin.class>${spark.shuffle.plugin.class}</spark.shuffle.sort.io.plugin.class>
+              <spark.ui.enabled>false</spark.ui.enabled>
             </systemProperties>
             <environmentVariables>
               <CELEBORN_LOCAL_HOSTNAME>localhost</CELEBORN_LOCAL_HOSTNAME>
@@ -1527,10 +1548,10 @@
       </modules>
       <properties>
         <lz4-java.version>1.8.0</lz4-java.version>
-        <scala.version>2.13.11</scala.version>
+        <scala.version>2.13.16</scala.version>
         <scala.binary.version>2.13</scala.binary.version>
-        <spark.version>4.0.0-preview2</spark.version>
-        <zstd-jni.version>1.5.6-5</zstd-jni.version>
+        <spark.version>4.0.0</spark.version>
+        <zstd-jni.version>1.5.6-9</zstd-jni.version>
       </properties>
     </profile>