diff --git a/dev/Dockerfile b/dev/Dockerfile index b55be39e9d..dab8dc85f8 100644 --- a/dev/Dockerfile +++ b/dev/Dockerfile @@ -37,7 +37,7 @@ RUN mkdir -p ${HADOOP_HOME} && mkdir -p ${SPARK_HOME} && mkdir -p /home/iceberg/ WORKDIR ${SPARK_HOME} # Remember to also update `tests/conftest`'s spark setting -ENV SPARK_VERSION=3.5.3 +ENV SPARK_VERSION=3.5.4 ENV ICEBERG_SPARK_RUNTIME_VERSION=3.5_2.12 ENV ICEBERG_VERSION=1.6.0 ENV PYICEBERG_VERSION=0.8.1 @@ -47,11 +47,11 @@ RUN curl --retry 5 -s -C - https://archive.apache.org/dist/spark/spark-${SPARK_V && rm -rf spark-${SPARK_VERSION}-bin-hadoop3.tgz # Download iceberg spark runtime -RUN curl --retry 5 -s https://repo1.maven.org/maven2/org/apache/iceberg/iceberg-spark-runtime-${ICEBERG_SPARK_RUNTIME_VERSION}/${ICEBERG_VERSION}/iceberg-spark-runtime-${ICEBERG_SPARK_RUNTIME_VERSION}-${ICEBERG_VERSION}.jar -Lo iceberg-spark-runtime-${ICEBERG_SPARK_RUNTIME_VERSION}-${ICEBERG_VERSION}.jar \ +RUN curl --retry 5 -s https://repository.apache.org/content/repositories/orgapacheiceberg-1180/org/apache/iceberg/iceberg-spark-runtime-3.5_2.12/1.7.2/iceberg-spark-runtime-3.5_2.12-1.7.2.jar -Lo iceberg-spark-runtime-${ICEBERG_SPARK_RUNTIME_VERSION}-${ICEBERG_VERSION}.jar \ && mv iceberg-spark-runtime-${ICEBERG_SPARK_RUNTIME_VERSION}-${ICEBERG_VERSION}.jar /opt/spark/jars # Download AWS bundle -RUN curl --retry 5 -s https://repo1.maven.org/maven2/org/apache/iceberg/iceberg-aws-bundle/${ICEBERG_VERSION}/iceberg-aws-bundle-${ICEBERG_VERSION}.jar -Lo /opt/spark/jars/iceberg-aws-bundle-${ICEBERG_VERSION}.jar +RUN curl --retry 5 -s https://repository.apache.org/content/repositories/orgapacheiceberg-1180/org/apache/iceberg/iceberg-aws-bundle/1.7.2/iceberg-aws-bundle-1.7.2.jar -Lo /opt/spark/jars/iceberg-aws-bundle-${ICEBERG_VERSION}.jar COPY spark-defaults.conf /opt/spark/conf ENV PATH="/opt/spark/sbin:/opt/spark/bin:${PATH}" diff --git a/tests/conftest.py b/tests/conftest.py index cfd9796312..7f784d5be2 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -2433,6 +2433,8 @@ def spark() -> "SparkSession": spark = ( SparkSession.builder.appName("PyIceberg integration test") .config("spark.sql.session.timeZone", "UTC") + .config("spark.sql.shuffle.partitions", "1") + .config("spark.default.parallelism", "1") .config("spark.sql.extensions", "org.apache.iceberg.spark.extensions.IcebergSparkSessionExtensions") .config("spark.sql.catalog.integration", "org.apache.iceberg.spark.SparkCatalog") .config("spark.sql.catalog.integration.catalog-impl", "org.apache.iceberg.rest.RESTCatalog")