bugfix: fix XAmzContentSHA256Mismatch (#249)

This appears to be a bug in the AWS s3 SDK. The root of the issue seems to be calling upload on lib-storage rather than an issue with the implementation. Switching from promise.all to a serial implentation fixes the issue, while also increasing the queue size off the default 3.

For future improvements, I would also thing about exposing more of the uploadOptions inside the next deployment construct. Perhaps we should also give consumers the options of uploading serial vs parallel.

Next steps I will also raise this with AWS team.

Author: radding

API.md

@@ -70,6 +70,10 @@ export interface NextjsBucketDeploymentProps {
    * @default false
    */
   readonly zip?: boolean | undefined;
+  /**
+   * The number of files to upload in parallel.
+   */
+  readonly queueSize?: number | undefined;
 }
 
 /**
@@ -80,6 +84,7 @@ export interface CustomResourceProperties {
   destinationKeyPrefix?: string;
   prune?: boolean | undefined;
   putConfig?: NextjsBucketDeploymentProps['putConfig'];
+  queueSize?: number | undefined;
   substitutionConfig?: NextjsBucketDeploymentProps['substitutionConfig'];
   sourceBucketName: string;
   sourceKeyPrefix?: string | undefined;
@@ -149,6 +154,7 @@ export class NextjsBucketDeployment extends Construct {
       prune: this.props.prune ?? false,
       substitutionConfig: this.props.substitutionConfig,
       zip: this.props.zip,
+      queueSize: this.props.queueSize,
     };
     return new CustomResource(this, 'CustomResource', {
       properties,

src/NextjsBucketDeployment.ts

@@ -25,7 +25,7 @@ import {
   type PutObjectCommandInput,
   S3Client,
 } from '@aws-sdk/client-s3';
-import { Upload } from '@aws-sdk/lib-storage';
+import { Options, Upload } from '@aws-sdk/lib-storage';
 import type { CloudFormationCustomResourceHandler } from 'aws-lambda';
 import type * as JSZipType from 'jszip';
 // @ts-ignore jsii doesn't support esModuleInterop
@@ -73,6 +73,7 @@ export const handler: CloudFormationCustomResourceHandler = async (event, contex
           filePaths,
           baseLocalDir: sourceDirPath,
           putConfig: props.putConfig,
+          queueSize: props.queueSize,
         });
         if (props.prune) {
           debug('Emptying/pruning bucket: ' + props.destinationBucketName);
@@ -272,12 +273,14 @@ async function uploadObjects({
   filePaths,
   baseLocalDir,
   putConfig = {},
+  queueSize,
 }: {
   bucket: CustomResourceProperties['destinationBucketName'];
   keyPrefix?: CustomResourceProperties['destinationKeyPrefix'];
   filePaths: string[];
   baseLocalDir: string;
   putConfig: CustomResourceProperties['putConfig'];
+  queueSize: CustomResourceProperties['queueSize'];
 }) {
   for await (const filePathChunk of chunkArray(filePaths, 100)) {
     const putObjectInputs: PutObjectCommandInput[] = filePathChunk.map((path) => {
@@ -293,7 +296,21 @@ async function uploadObjects({
       };
     });
 
-    await Promise.all(putObjectInputs.map((params) => new Upload({ client: s3, params }).done()));
+    // Call put objects serially, prevents XAmzContentSHA256Mismatch errors
+    // This seems to be a bug within the lib storage package, I have opened an issue here: https://github.com/aws/aws-sdk-js-v3/issues/6940
+    await putObjectInputs.reduce(async (acc, params) => {
+      await acc;
+      const opts: Options = {
+        client: s3,
+        params,
+      };
+      if (queueSize) {
+        opts.queueSize = queueSize;
+      }
+      const upload = new Upload(opts);
+      console.log('uploading', params);
+      return upload.done();
+    }, Promise.resolve<any>(null));
   }
 }