Skip to content

Commit 3b1bfa1

Browse files
bentskubentsku
committed
add AWS validated multipart checksum test
1 parent adb1ff5 commit 3b1bfa1

File tree

2 files changed

+360
-0
lines changed

2 files changed

+360
-0
lines changed

‎tests/integration/s3/test_s3.py‎

Lines changed: 166 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -3100,6 +3100,172 @@ def test_s3_delete_objects_trailing_slash(self, s3_client, aws_http_client_facto
31003100
assert "DeleteResult" in resp_dict
31013101
assert resp_dict["DeleteResult"]["Deleted"]["Key"] == object_key
31023102

3103+
@pytest.mark.aws_validated
3104+
@pytest.mark.xfail(reason="Behaviour not implemented yet")
3105+
def test_complete_multipart_parts_checksum(self, s3_client, s3_bucket, snapshot):
3106+
snapshot.add_transformer(
3107+
[
3108+
snapshot.transform.key_value("Bucket", reference_replacement=False),
3109+
snapshot.transform.key_value("Location"),
3110+
snapshot.transform.key_value("UploadId"),
3111+
snapshot.transform.key_value("DisplayName", reference_replacement=False),
3112+
snapshot.transform.key_value("ID", reference_replacement=False),
3113+
]
3114+
)
3115+
3116+
key_name = "test-multipart-checksum"
3117+
response = s3_client.create_multipart_upload(
3118+
Bucket=s3_bucket, Key=key_name, ChecksumAlgorithm="SHA256"
3119+
)
3120+
snapshot.match("create-mpu-checksum", response)
3121+
upload_id = response["UploadId"]
3122+
3123+
# data must be at least 5MiB
3124+
part_data = "a" * (5_242_880 + 1)
3125+
part_data = to_bytes(part_data)
3126+
3127+
parts = 3
3128+
multipart_upload_parts = []
3129+
for part in range(parts):
3130+
# Write contents to memory rather than a file.
3131+
part_number = part + 1
3132+
upload_file_object = BytesIO(part_data)
3133+
response = s3_client.upload_part(
3134+
Bucket=s3_bucket,
3135+
Key=key_name,
3136+
Body=upload_file_object,
3137+
PartNumber=part_number,
3138+
UploadId=upload_id,
3139+
ChecksumAlgorithm="SHA256",
3140+
)
3141+
snapshot.match(f"upload-part-{part}", response)
3142+
multipart_upload_parts.append(
3143+
{
3144+
"ETag": response["ETag"],
3145+
"PartNumber": part_number,
3146+
"ChecksumSHA256": response["ChecksumSHA256"],
3147+
}
3148+
)
3149+
3150+
response = s3_client.list_parts(Bucket=s3_bucket, Key=key_name, UploadId=upload_id)
3151+
snapshot.match("list-parts", response)
3152+
3153+
# testing completing the multipart without the checksum of parts
3154+
multipart_upload_parts_no_checksum = [
3155+
{"ETag": upload_part["ETag"], "PartNumber": upload_part["PartNumber"]}
3156+
for upload_part in multipart_upload_parts
3157+
]
3158+
3159+
with pytest.raises(ClientError) as e:
3160+
s3_client.complete_multipart_upload(
3161+
Bucket=s3_bucket,
3162+
Key=key_name,
3163+
MultipartUpload={"Parts": multipart_upload_parts_no_checksum},
3164+
UploadId=upload_id,
3165+
)
3166+
snapshot.match("complete-multipart-without-checksum", e.value.response)
3167+
3168+
response = s3_client.complete_multipart_upload(
3169+
Bucket=s3_bucket,
3170+
Key=key_name,
3171+
MultipartUpload={"Parts": multipart_upload_parts},
3172+
UploadId=upload_id,
3173+
)
3174+
snapshot.match("complete-multipart-checksum", response)
3175+
3176+
get_object_with_checksum = s3_client.get_object(
3177+
Bucket=s3_bucket, Key=key_name, ChecksumMode="ENABLED"
3178+
)
3179+
# empty the stream, it's a 15MB string, we don't need to snapshot that
3180+
get_object_with_checksum["Body"].read()
3181+
snapshot.match("get-object-with-checksum", get_object_with_checksum)
3182+
3183+
object_attrs = s3_client.get_object_attributes(
3184+
Bucket=s3_bucket,
3185+
Key=key_name,
3186+
ObjectAttributes=["Checksum"],
3187+
)
3188+
snapshot.match("get-object-attrs", object_attrs)
3189+
3190+
@pytest.mark.aws_validated
3191+
@pytest.mark.xfail(reason="Behaviour not implemented yet")
3192+
def test_multipart_parts_checksum_exceptions(self, s3_client, s3_bucket, snapshot):
3193+
snapshot.add_transformer(
3194+
[
3195+
snapshot.transform.key_value("Bucket", reference_replacement=False),
3196+
snapshot.transform.key_value("Location"),
3197+
snapshot.transform.key_value("UploadId"),
3198+
snapshot.transform.key_value("DisplayName", reference_replacement=False),
3199+
snapshot.transform.key_value("ID", reference_replacement=False),
3200+
]
3201+
)
3202+
3203+
key_name = "test-multipart-checksum-exc"
3204+
response = s3_client.create_multipart_upload(Bucket=s3_bucket, Key=key_name)
3205+
snapshot.match("create-mpu-no-checksum", response)
3206+
upload_id = response["UploadId"]
3207+
3208+
# data must be at least 5MiB
3209+
part_data = "abc"
3210+
part_data = to_bytes(part_data)
3211+
checksum_part = hash_sha256(part_data)
3212+
3213+
# Write contents to memory rather than a file.
3214+
upload_file_object = BytesIO(part_data)
3215+
with pytest.raises(ClientError) as e:
3216+
s3_client.upload_part(
3217+
Bucket=s3_bucket,
3218+
Key=key_name,
3219+
Body=upload_file_object,
3220+
PartNumber=1,
3221+
UploadId=upload_id,
3222+
ChecksumAlgorithm="SHA256",
3223+
)
3224+
snapshot.match("upload-part-with-checksum", e.value.response)
3225+
3226+
upload_resp = s3_client.upload_part(
3227+
Bucket=s3_bucket,
3228+
Key=key_name,
3229+
Body=upload_file_object,
3230+
PartNumber=1,
3231+
UploadId=upload_id,
3232+
)
3233+
snapshot.match("upload-part-no-checksum-ok", upload_resp)
3234+
3235+
with pytest.raises(ClientError) as e:
3236+
s3_client.complete_multipart_upload(
3237+
Bucket=s3_bucket,
3238+
Key=key_name,
3239+
MultipartUpload={
3240+
"Parts": [
3241+
{
3242+
"ETag": upload_resp["ETag"],
3243+
"PartNumber": 1,
3244+
"ChecksumSHA256": checksum_part,
3245+
}
3246+
],
3247+
},
3248+
UploadId=upload_id,
3249+
)
3250+
snapshot.match("complete-part-with-checksum", e.value.response)
3251+
3252+
response = s3_client.create_multipart_upload(
3253+
Bucket=s3_bucket, Key=key_name, ChecksumAlgorithm="SHA256"
3254+
)
3255+
snapshot.match("create-mpu-with-checksum", response)
3256+
upload_id = response["UploadId"]
3257+
3258+
upload_file_object = BytesIO(part_data)
3259+
with pytest.raises(ClientError) as e:
3260+
s3_client.upload_part(
3261+
Bucket=s3_bucket,
3262+
Key=key_name,
3263+
Body=upload_file_object,
3264+
PartNumber=1,
3265+
UploadId=upload_id,
3266+
)
3267+
snapshot.match("upload-part-no-checksum-exc", e.value.response)
3268+
31033269

31043270
class TestS3TerraformRawRequests:
31053271
@pytest.mark.only_localstack

‎tests/integration/s3/test_s3.snapshot.json‎

Lines changed: 194 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -4576,5 +4576,199 @@
45764576
}
45774577
}
45784578
}
4579+
},
4580+
"tests/integration/s3/test_s3.py::TestS3::test_complete_multipart_parts_checksum": {
4581+
"recorded-date": "17-02-2023, 21:42:39",
4582+
"recorded-content": {
4583+
"create-mpu-checksum": {
4584+
"Bucket": "bucket",
4585+
"ChecksumAlgorithm": "SHA256",
4586+
"Key": "test-multipart-checksum",
4587+
"UploadId": "<upload-id:1>",
4588+
"ResponseMetadata": {
4589+
"HTTPHeaders": {},
4590+
"HTTPStatusCode": 200
4591+
}
4592+
},
4593+
"upload-part-0": {
4594+
"ChecksumSHA256": "DjU70AB1bON8k0n0fVHv2PJQVWcA/jWsITp6ti20Tbs=",
4595+
"ETag": "\"c4c753e69bb853187f5854c46cf801c6\"",
4596+
"ResponseMetadata": {
4597+
"HTTPHeaders": {},
4598+
"HTTPStatusCode": 200
4599+
}
4600+
},
4601+
"upload-part-1": {
4602+
"ChecksumSHA256": "DjU70AB1bON8k0n0fVHv2PJQVWcA/jWsITp6ti20Tbs=",
4603+
"ETag": "\"c4c753e69bb853187f5854c46cf801c6\"",
4604+
"ResponseMetadata": {
4605+
"HTTPHeaders": {},
4606+
"HTTPStatusCode": 200
4607+
}
4608+
},
4609+
"upload-part-2": {
4610+
"ChecksumSHA256": "DjU70AB1bON8k0n0fVHv2PJQVWcA/jWsITp6ti20Tbs=",
4611+
"ETag": "\"c4c753e69bb853187f5854c46cf801c6\"",
4612+
"ResponseMetadata": {
4613+
"HTTPHeaders": {},
4614+
"HTTPStatusCode": 200
4615+
}
4616+
},
4617+
"list-parts": {
4618+
"Bucket": "bucket",
4619+
"ChecksumAlgorithm": "SHA256",
4620+
"Initiator": {
4621+
"DisplayName": "display-name",
4622+
"ID": "i-d"
4623+
},
4624+
"IsTruncated": false,
4625+
"Key": "test-multipart-checksum",
4626+
"MaxParts": 1000,
4627+
"NextPartNumberMarker": 3,
4628+
"Owner": {
4629+
"DisplayName": "display-name",
4630+
"ID": "i-d"
4631+
},
4632+
"PartNumberMarker": 0,
4633+
"Parts": [
4634+
{
4635+
"ChecksumSHA256": "DjU70AB1bON8k0n0fVHv2PJQVWcA/jWsITp6ti20Tbs=",
4636+
"ETag": "\"c4c753e69bb853187f5854c46cf801c6\"",
4637+
"LastModified": "datetime",
4638+
"PartNumber": 1,
4639+
"Size": 5242881
4640+
},
4641+
{
4642+
"ChecksumSHA256": "DjU70AB1bON8k0n0fVHv2PJQVWcA/jWsITp6ti20Tbs=",
4643+
"ETag": "\"c4c753e69bb853187f5854c46cf801c6\"",
4644+
"LastModified": "datetime",
4645+
"PartNumber": 2,
4646+
"Size": 5242881
4647+
},
4648+
{
4649+
"ChecksumSHA256": "DjU70AB1bON8k0n0fVHv2PJQVWcA/jWsITp6ti20Tbs=",
4650+
"ETag": "\"c4c753e69bb853187f5854c46cf801c6\"",
4651+
"LastModified": "datetime",
4652+
"PartNumber": 3,
4653+
"Size": 5242881
4654+
}
4655+
],
4656+
"StorageClass": "STANDARD",
4657+
"UploadId": "<upload-id:1>",
4658+
"ResponseMetadata": {
4659+
"HTTPHeaders": {},
4660+
"HTTPStatusCode": 200
4661+
}
4662+
},
4663+
"complete-multipart-without-checksum": {
4664+
"Error": {
4665+
"Code": "InvalidRequest",
4666+
"Message": "The upload was created using a sha256 checksum. The complete request must include the checksum for each part. It was missing for part 1 in the request."
4667+
},
4668+
"ResponseMetadata": {
4669+
"HTTPHeaders": {},
4670+
"HTTPStatusCode": 400
4671+
}
4672+
},
4673+
"complete-multipart-checksum": {
4674+
"Bucket": "bucket",
4675+
"ChecksumSHA256": "dVAleH1OmqkLvByTMLIWSjNCz3x2Ul1KJEZw3eQ2Fqg=-3",
4676+
"ETag": "\"c7cb0938a47e31f70cf07028d22e6913-3\"",
4677+
"Key": "test-multipart-checksum",
4678+
"Location": "<location:1>",
4679+
"ResponseMetadata": {
4680+
"HTTPHeaders": {},
4681+
"HTTPStatusCode": 200
4682+
}
4683+
},
4684+
"get-object-with-checksum": {
4685+
"AcceptRanges": "bytes",
4686+
"Body": "",
4687+
"ChecksumSHA256": "dVAleH1OmqkLvByTMLIWSjNCz3x2Ul1KJEZw3eQ2Fqg=-3",
4688+
"ContentLength": 15728643,
4689+
"ContentType": "binary/octet-stream",
4690+
"ETag": "\"c7cb0938a47e31f70cf07028d22e6913-3\"",
4691+
"LastModified": "datetime",
4692+
"Metadata": {},
4693+
"ResponseMetadata": {
4694+
"HTTPHeaders": {},
4695+
"HTTPStatusCode": 200
4696+
}
4697+
},
4698+
"get-object-attrs": {
4699+
"Checksum": {
4700+
"ChecksumSHA256": "dVAleH1OmqkLvByTMLIWSjNCz3x2Ul1KJEZw3eQ2Fqg="
4701+
},
4702+
"LastModified": "datetime",
4703+
"ResponseMetadata": {
4704+
"HTTPHeaders": {},
4705+
"HTTPStatusCode": 200
4706+
}
4707+
}
4708+
}
4709+
},
4710+
"tests/integration/s3/test_s3.py::TestS3::test_multipart_parts_checksum_exceptions": {
4711+
"recorded-date": "17-02-2023, 21:57:49",
4712+
"recorded-content": {
4713+
"create-mpu-no-checksum": {
4714+
"Bucket": "bucket",
4715+
"Key": "test-multipart-checksum-exc",
4716+
"UploadId": "<upload-id:1>",
4717+
"ResponseMetadata": {
4718+
"HTTPHeaders": {},
4719+
"HTTPStatusCode": 200
4720+
}
4721+
},
4722+
"upload-part-with-checksum": {
4723+
"Error": {
4724+
"Code": "InvalidRequest",
4725+
"Message": "Checksum Type mismatch occurred, expected checksum Type: null, actual checksum Type: sha256"
4726+
},
4727+
"ResponseMetadata": {
4728+
"HTTPHeaders": {},
4729+
"HTTPStatusCode": 400
4730+
}
4731+
},
4732+
"upload-part-no-checksum-ok": {
4733+
"ETag": "\"d41d8cd98f00b204e9800998ecf8427e\"",
4734+
"ResponseMetadata": {
4735+
"HTTPHeaders": {},
4736+
"HTTPStatusCode": 200
4737+
}
4738+
},
4739+
"complete-part-with-checksum": {
4740+
"Error": {
4741+
"Code": "InvalidPart",
4742+
"ETag": "d41d8cd98f00b204e9800998ecf8427e",
4743+
"Message": "One or more of the specified parts could not be found. The part may not have been uploaded, or the specified entity tag may not match the part's entity tag.",
4744+
"PartNumber": "1",
4745+
"UploadId": "<upload-id:1>"
4746+
},
4747+
"ResponseMetadata": {
4748+
"HTTPHeaders": {},
4749+
"HTTPStatusCode": 400
4750+
}
4751+
},
4752+
"create-mpu-with-checksum": {
4753+
"Bucket": "bucket",
4754+
"ChecksumAlgorithm": "SHA256",
4755+
"Key": "test-multipart-checksum-exc",
4756+
"UploadId": "<upload-id:2>",
4757+
"ResponseMetadata": {
4758+
"HTTPHeaders": {},
4759+
"HTTPStatusCode": 200
4760+
}
4761+
},
4762+
"upload-part-no-checksum-exc": {
4763+
"Error": {
4764+
"Code": "InvalidRequest",
4765+
"Message": "Checksum Type mismatch occurred, expected checksum Type: sha256, actual checksum Type: null"
4766+
},
4767+
"ResponseMetadata": {
4768+
"HTTPHeaders": {},
4769+
"HTTPStatusCode": 400
4770+
}
4771+
}
4772+
}
45794773
}
45804774
}

0 commit comments

Comments
 (0)