move s3 implementation config to script

Signed-off-by: Manabu McCloskey <[email protected]>
awslabs · Jan 9, 2025 · 3000ecc · 3000ecc
1 parent 139681f
commit 3000ecc
Show file tree

Hide file tree

Showing 2 changed files with 1 addition and 1 deletion.
diff --git a/analytics/terraform/spark-k8s-operator/examples/s3-tables/s3table-iceberg-pyspark.py b/analytics/terraform/spark-k8s-operator/examples/s3-tables/s3table-iceberg-pyspark.py
@@ -39,6 +39,7 @@ def main(args):
              .config("spark.sql.catalog.s3tablesbucket", "org.apache.iceberg.spark.SparkCatalog")
              .config("spark.sql.catalog.s3tablesbucket.catalog-impl", "software.amazon.s3tables.iceberg.S3TablesCatalog")
              .config("spark.sql.catalog.s3tablesbucket.warehouse", s3table_arn)
+             .config('spark.hadoop.fs.s3.impl', "org.apache.hadoop.fs.s3a.S3AFileSystem")
              .config("spark.sql.defaultCatalog", "s3tablesbucket")
              .getOrCreate())
 

diff --git a/analytics/terraform/spark-k8s-operator/examples/s3-tables/s3table-spark-operator.yaml b/analytics/terraform/spark-k8s-operator/examples/s3-tables/s3table-spark-operator.yaml
@@ -35,7 +35,6 @@ spec:
     "spark.hadoop.fs.s3a.input.fadvise": "random"
     "spark.hadoop.fs.s3a.aws.credentials.provider.mapping": "com.amazonaws.auth.WebIdentityTokenCredentialsProvider=software.amazon.awssdk.auth.credentials.WebIdentityTokenFileCredentialsProvider"
     "spark.hadoop.fs.s3a.aws.credentials.provider": "software.amazon.awssdk.auth.credentials.WebIdentityTokenFileCredentialsProvider"  # AWS SDK V2 https://hadoop.apache.org/docs/stable/hadoop-aws/tools/hadoop-aws/aws_sdk_upgrade.html
-    "spark.hadoop.fs.s3.impl": "org.apache.hadoop.fs.s3a.S3AFileSystem"
 
   restartPolicy:
     type: OnFailure