Skip to content

CompleteMultipartUpload is not idempotent #2586

@yankeenjg

Description

@yankeenjg

The following is a test against the real S3 endpoint that illustrates that the CompleteMultipartUpload API is idempotent when called multiple times with the same parameters for at least one hour. The same test when run against the mock returns NoSuchUpload instead of the prior response --

import software.amazon.awssdk.auth.credentials.AwsCredentials;
import software.amazon.awssdk.auth.credentials.AwsSessionCredentials;
import software.amazon.awssdk.auth.credentials.StaticCredentialsProvider;
import software.amazon.awssdk.regions.Region;
import software.amazon.awssdk.services.s3.S3Client;
import software.amazon.awssdk.services.s3.model.*;
import software.amazon.awssdk.core.sync.RequestBody;
import com.fasterxml.jackson.databind.JsonNode;
import com.fasterxml.jackson.databind.ObjectMapper;
import java.io.BufferedReader;
import java.io.InputStreamReader;
import java.nio.file.Files;
import java.nio.file.Path;
import java.util.List;

public class S3IdempotencyTest {
    
    public static void main(String[] args) throws Exception {
        String bucketName = "s3-idempotency-test-" + System.currentTimeMillis() + "-" + ProcessHandle.current().pid();
        

        
        // Create S3 client
        S3Client s3 = S3Client.builder()
                .region(Region.US_EAST_1)
                .build();
        
        try {
            // Create test file
            System.out.println("Creating test file...");
            Path testFile = Files.createTempFile("test-file", ".txt");
            byte[] data = new byte[10 * 1024 * 1024]; // 10MB
            Files.write(testFile, data);
            
            // Create bucket
            System.out.println("Creating bucket " + bucketName + "...");
            try {
                s3.createBucket(CreateBucketRequest.builder().bucket(bucketName).build());
                System.out.println("Bucket created: " + bucketName);
            } catch (Exception e) {
                System.out.println("Bucket creation failed: " + e.getMessage());
            }
            
            // Start multipart upload
            System.out.println("Starting multipart upload...");
            CreateMultipartUploadResponse createResponse = s3.createMultipartUpload(
                CreateMultipartUploadRequest.builder()
                    .bucket(bucketName)
                    .key("test-object")
                    .build()
            );
            String uploadId = createResponse.uploadId();
            System.out.println("Upload ID: " + uploadId);
            
            // Upload part
            System.out.println("Uploading part...");
            UploadPartResponse partResponse = s3.uploadPart(
                UploadPartRequest.builder()
                    .bucket(bucketName)
                    .key("test-object")
                    .uploadId(uploadId)
                    .partNumber(1)
                    .build(),
                RequestBody.fromFile(testFile)
            );
            String etag = partResponse.eTag();
            System.out.println("Part ETag: " + etag);
            
            // Complete multipart upload (first time)
            System.out.println("Completing multipart upload (first attempt)...");
            CompletedPart part = CompletedPart.builder()
                .partNumber(1)
                .eTag(etag)
                .build();
            
            CompleteMultipartUploadResponse firstResponse = s3.completeMultipartUpload(
                CompleteMultipartUploadRequest.builder()
                    .bucket(bucketName)
                    .key("test-object")
                    .uploadId(uploadId)
                    .multipartUpload(CompletedMultipartUpload.builder().parts(part).build())
                    .build()
            );
            
            String firstETag = firstResponse.eTag();
            System.out.println("First ETag: " + firstETag);
            
            // Test idempotency every minute for 60 minutes
            System.out.println("Testing idempotency every minute for 60 minutes...");
            for (int i = 1; i <= 60; i++) {
                System.out.println("Minute " + i + ": Waiting 60 seconds...");
                Thread.sleep(60000);
                
                System.out.println("Minute " + i + ": Testing idempotency...");
                try {
                    CompleteMultipartUploadResponse response = s3.completeMultipartUpload(
                        CompleteMultipartUploadRequest.builder()
                            .bucket(bucketName)
                            .key("test-object")
                            .uploadId(uploadId)
                            .multipartUpload(CompletedMultipartUpload.builder().parts(part).build())
                            .build()
                    );
                    
                    String currentETag = response.eTag();
                    if (firstETag.equals(currentETag)) {
                        System.out.println("Minute " + i + ": ✓ Still idempotent (ETag: " + currentETag + ")");
                    } else {
                        System.out.println("Minute " + i + ": ✗ ETag changed (was: " + firstETag + ", now: " + currentETag + ")");
                        break;
                    }
                } catch (Exception e) {
                    System.out.println("Minute " + i + ": ✗ Call failed: " + e.getMessage());
                    break;
                }
            }
            
            // Cleanup
            System.out.println("Cleaning up current test...");
            try {
                s3.deleteObject(DeleteObjectRequest.builder().bucket(bucketName).key("test-object").build());
                s3.deleteBucket(DeleteBucketRequest.builder().bucket(bucketName).build());
            } catch (Exception e) {
                System.out.println("Cleanup failed: " + e.getMessage());
            }
            
            // Clean up old test buckets
            System.out.println("Cleaning up old test buckets...");
            try {
                ListBucketsResponse bucketsResponse = s3.listBuckets();
                for (Bucket bucket : bucketsResponse.buckets()) {
                    if (bucket.name().startsWith("s3-idempotency-test-")) {
                        System.out.println("Removing old bucket: " + bucket.name());
                        try {
                            s3.deleteObject(DeleteObjectRequest.builder().bucket(bucket.name()).key("test-object").build());
                            s3.deleteBucket(DeleteBucketRequest.builder().bucket(bucket.name()).build());
                        } catch (Exception e) {
                            // Ignore cleanup errors
                        }
                    }
                }
            } catch (Exception e) {
                System.out.println("Old bucket cleanup failed: " + e.getMessage());
            }
            
            Files.delete(testFile);
            System.out.println("Test completed");
            
        } finally {
            s3.close();
        }
    }
}

Metadata

Metadata

Assignees

Labels

Type

Projects

No projects

Milestone

No milestone

Relationships

None yet

Development

No branches or pull requests

Issue actions