Skip to content

Commit cf8a254

Browse files
bentskubentsku
authored
Implement more ASF S3 operations in provider (#6868)
1 parent 6779925 commit cf8a254

File tree

9 files changed

+3116
-2584
lines changed

9 files changed

+3116
-2584
lines changed

‎localstack/aws/api/s3/__init__.py‎

Lines changed: 52 additions & 31 deletions
Original file line numberDiff line numberDiff line change
@@ -164,6 +164,8 @@
164164
Years = int
165165
BucketRegion = str
166166
BucketContentType = str
167+
IfCondition = str
168+
RestoreObjectOutputStatusCode = int
167169

168170

169171
class AnalyticsS3ExportFileFormat(str):
@@ -558,6 +560,7 @@ class BucketAlreadyOwnedByYou(ServiceException):
558560
code: str = "BucketAlreadyOwnedByYou"
559561
sender_fault: bool = False
560562
status_code: int = 400
563+
BucketName: Optional[BucketName]
561564

562565

563566
class InvalidObjectState(ServiceException):
@@ -578,7 +581,8 @@ class NoSuchBucket(ServiceException):
578581
class NoSuchKey(ServiceException):
579582
code: str = "NoSuchKey"
580583
sender_fault: bool = False
581-
status_code: int = 400
584+
status_code: int = 404
585+
Key: Optional[ObjectKey]
582586

583587

584588
class NoSuchUpload(ServiceException):
@@ -613,6 +617,24 @@ class InvalidBucketName(ServiceException):
613617
BucketName: Optional[BucketName]
614618

615619

620+
class PreconditionFailed(ServiceException):
621+
code: str = "PreconditionFailed"
622+
sender_fault: bool = False
623+
status_code: int = 412
624+
Condition: Optional[IfCondition]
625+
626+
627+
ObjectSize = int
628+
629+
630+
class InvalidRange(ServiceException):
631+
code: str = "InvalidRange"
632+
sender_fault: bool = False
633+
status_code: int = 416
634+
ActualObjectSize: Optional[ObjectSize]
635+
RangeRequested: Optional[ContentRange]
636+
637+
616638
AbortDate = datetime
617639

618640

@@ -1223,32 +1245,6 @@ class DeleteObjectTaggingRequest(ServiceRequest):
12231245
ExpectedBucketOwner: Optional[AccountId]
12241246

12251247

1226-
class Error(TypedDict, total=False):
1227-
Key: Optional[ObjectKey]
1228-
VersionId: Optional[ObjectVersionId]
1229-
Code: Optional[Code]
1230-
Message: Optional[Message]
1231-
1232-
1233-
Errors = List[Error]
1234-
1235-
1236-
class DeletedObject(TypedDict, total=False):
1237-
Key: Optional[ObjectKey]
1238-
VersionId: Optional[ObjectVersionId]
1239-
DeleteMarker: Optional[DeleteMarker]
1240-
DeleteMarkerVersionId: Optional[DeleteMarkerVersionId]
1241-
1242-
1243-
DeletedObjects = List[DeletedObject]
1244-
1245-
1246-
class DeleteObjectsOutput(TypedDict, total=False):
1247-
Deleted: Optional[DeletedObjects]
1248-
RequestCharged: Optional[RequestCharged]
1249-
Errors: Optional[Errors]
1250-
1251-
12521248
class DeleteObjectsRequest(ServiceRequest):
12531249
Bucket: BucketName
12541250
Delete: Delete
@@ -1264,6 +1260,16 @@ class DeletePublicAccessBlockRequest(ServiceRequest):
12641260
ExpectedBucketOwner: Optional[AccountId]
12651261

12661262

1263+
class DeletedObject(TypedDict, total=False):
1264+
Key: Optional[ObjectKey]
1265+
VersionId: Optional[ObjectVersionId]
1266+
DeleteMarker: Optional[DeleteMarker]
1267+
DeleteMarkerVersionId: Optional[DeleteMarkerVersionId]
1268+
1269+
1270+
DeletedObjects = List[DeletedObject]
1271+
1272+
12671273
class ReplicationTimeValue(TypedDict, total=False):
12681274
Minutes: Optional[Minutes]
12691275

@@ -1305,10 +1311,20 @@ class EndEvent(TypedDict, total=False):
13051311
pass
13061312

13071313

1314+
class Error(TypedDict, total=False):
1315+
Key: Optional[ObjectKey]
1316+
VersionId: Optional[ObjectVersionId]
1317+
Code: Optional[Code]
1318+
Message: Optional[Message]
1319+
1320+
13081321
class ErrorDocument(TypedDict, total=False):
13091322
Key: ObjectKey
13101323

13111324

1325+
Errors = List[Error]
1326+
1327+
13121328
class EventBridgeConfiguration(TypedDict, total=False):
13131329
pass
13141330

@@ -1737,9 +1753,6 @@ class GetObjectAclRequest(ServiceRequest):
17371753
ExpectedBucketOwner: Optional[AccountId]
17381754

17391755

1740-
ObjectSize = int
1741-
1742-
17431756
class ObjectPart(TypedDict, total=False):
17441757
PartNumber: Optional[PartNumber]
17451758
Size: Optional[Size]
@@ -1861,6 +1874,7 @@ class GetObjectOutput(TypedDict, total=False):
18611874
ObjectLockMode: Optional[ObjectLockMode]
18621875
ObjectLockRetainUntilDate: Optional[ObjectLockRetainUntilDate]
18631876
ObjectLockLegalHoldStatus: Optional[ObjectLockLegalHoldStatus]
1877+
StatusCode: Optional[GetObjectResponseStatusCode]
18641878

18651879

18661880
ResponseExpires = datetime
@@ -2741,6 +2755,7 @@ class RequestProgress(TypedDict, total=False):
27412755
class RestoreObjectOutput(TypedDict, total=False):
27422756
RequestCharged: Optional[RequestCharged]
27432757
RestoreOutputPath: Optional[RestoreOutputPath]
2758+
StatusCode: Optional[RestoreObjectOutputStatusCode]
27442759

27452760

27462761
class SelectParameters(TypedDict, total=False):
@@ -2930,6 +2945,12 @@ class HeadBucketOutput(TypedDict, total=False):
29302945
BucketContentType: Optional[BucketContentType]
29312946

29322947

2948+
class DeleteResult(TypedDict, total=False):
2949+
Deleted: Optional[DeletedObjects]
2950+
RequestCharged: Optional[RequestCharged]
2951+
Errors: Optional[Errors]
2952+
2953+
29332954
class S3Api:
29342955

29352956
service = "s3"
@@ -3195,7 +3216,7 @@ def delete_objects(
31953216
bypass_governance_retention: BypassGovernanceRetention = None,
31963217
expected_bucket_owner: AccountId = None,
31973218
checksum_algorithm: ChecksumAlgorithm = None,
3198-
) -> DeleteObjectsOutput:
3219+
) -> DeleteResult:
31993220
raise NotImplementedError
32003221

32013222
@handler("DeletePublicAccessBlock")

‎localstack/aws/protocol/serializer.py‎

Lines changed: 14 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1350,6 +1350,8 @@ class S3ResponseSerializer(RestXMLResponseSerializer):
13501350
serialization.
13511351
"""
13521352

1353+
SUPPORTED_MIME_TYPES = [TEXT_XML, APPLICATION_XML]
1354+
13531355
def _serialize_error(
13541356
self,
13551357
error: ServiceException,
@@ -1371,6 +1373,18 @@ def _serialize_error(
13711373

13721374
response.set_response(self._encode_payload(self._node_to_string(root, mime_type)))
13731375

1376+
def _prepare_additional_traits_in_response(
1377+
self, response: HttpResponse, operation_model: OperationModel
1378+
):
1379+
"""Adds the request ID to the headers (in contrast to the body - as in the Query protocol)."""
1380+
response = super()._prepare_additional_traits_in_response(response, operation_model)
1381+
request_id = gen_amzn_requestid_long()
1382+
response.headers["x-amz-request-id"] = request_id
1383+
response.headers[
1384+
"x-amz-id-2"
1385+
] = f"MzRISOwyjmnup{request_id}7/JypPGXLh0OVFGcJaaO3KW/hRAqKOpIEEp"
1386+
return response
1387+
13741388

13751389
class SqsResponseSerializer(QueryResponseSerializer):
13761390
"""

‎localstack/aws/spec-patches.json‎

Lines changed: 140 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -115,6 +115,146 @@
115115
"op": "add",
116116
"path": "/shapes/GetBucketLocationOutput/payload",
117117
"value": "LocationConstraint"
118+
},
119+
{
120+
"op": "add",
121+
"path": "/shapes/BucketAlreadyOwnedByYou/members/BucketName",
122+
"value": {
123+
"shape": "BucketName"
124+
}
125+
},
126+
{
127+
"op": "add",
128+
"path": "/shapes/GetObjectOutput/members/StatusCode",
129+
"value": {
130+
"shape": "GetObjectResponseStatusCode",
131+
"location": "statusCode"
132+
}
133+
},
134+
{
135+
"op": "add",
136+
"path": "/shapes/NoSuchKey/members/Key",
137+
"value": {
138+
"shape": "ObjectKey"
139+
}
140+
},
141+
{
142+
"op": "add",
143+
"path": "/shapes/NoSuchKey/error",
144+
"value": {
145+
"httpStatusCode": 404
146+
}
147+
},
148+
{
149+
"op": "add",
150+
"path": "/shapes/PreconditionFailed",
151+
"value": {
152+
"type": "structure",
153+
"members": {
154+
"Condition": {
155+
"shape": "IfCondition"
156+
}
157+
},
158+
"error": {
159+
"httpStatusCode": 412
160+
},
161+
"documentation": "<p>At least one of the pre-conditions you specified did not hold</p>",
162+
"exception": true
163+
}
164+
},
165+
{
166+
"op": "add",
167+
"path": "/shapes/IfCondition",
168+
"value": {
169+
"type": "string"
170+
}
171+
},
172+
{
173+
"op": "add",
174+
"path": "/shapes/InvalidRange",
175+
"value": {
176+
"type": "structure",
177+
"members": {
178+
"ActualObjectSize": {
179+
"shape": "ObjectSize"
180+
},
181+
"RangeRequested": {
182+
"shape": "ContentRange"
183+
}
184+
},
185+
"error": {
186+
"httpStatusCode": 416
187+
},
188+
"documentation": "<p>The requested range is not satisfiable</p>",
189+
"exception": true
190+
}
191+
},
192+
{
193+
"op": "add",
194+
"path": "/shapes/HeadObjectOutput/members/Expires",
195+
"value": {
196+
"shape": "Expires",
197+
"documentation": "<p>The date and time at which the object is no longer cacheable.</p>",
198+
"location": "header",
199+
"locationName": "expires"
200+
}
201+
},
202+
{
203+
"op": "add",
204+
"path": "/shapes/GetObjectOutput/members/Expires",
205+
"value": {
206+
"shape": "Expires",
207+
"documentation": "<p>The date and time at which the object is no longer cacheable.</p>",
208+
"location": "header",
209+
"locationName": "expires"
210+
}
211+
},
212+
{
213+
"op": "remove",
214+
"path": "/shapes/DeleteObjectsOutput"
215+
},
216+
{
217+
"op": "add",
218+
"path": "/shapes/DeleteResult",
219+
"value": {
220+
"type": "structure",
221+
"members": {
222+
"Deleted": {
223+
"shape": "DeletedObjects",
224+
"documentation": "<p>Container element for a successful delete. It identifies the object that was successfully deleted.</p>"
225+
},
226+
"RequestCharged": {
227+
"shape": "RequestCharged",
228+
"location": "header",
229+
"locationName": "x-amz-request-charged"
230+
},
231+
"Errors": {
232+
"shape": "Errors",
233+
"documentation": "<p>Container for a failed delete action that describes the object that Amazon S3 attempted to delete and the error it encountered.</p>",
234+
"locationName": "Error"
235+
}
236+
}
237+
}
238+
},
239+
{
240+
"op": "replace",
241+
"path": "/operations/DeleteObjects/output/shape",
242+
"value": "DeleteResult"
243+
},
244+
{
245+
"op": "add",
246+
"path": "/shapes/RestoreObjectOutputStatusCode",
247+
"value": {
248+
"type": "integer"
249+
}
250+
},
251+
{
252+
"op": "add",
253+
"path": "/shapes/RestoreObjectOutput/members/StatusCode",
254+
"value": {
255+
"shape": "RestoreObjectOutputStatusCode",
256+
"location": "statusCode"
257+
}
118258
}
119259
]
120260
}

‎localstack/services/s3/models.py‎

Lines changed: 7 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,7 @@
11
from typing import Dict
22

33
from localstack.aws.api.s3 import (
4+
BucketLifecycleConfiguration,
45
BucketName,
56
CORSConfiguration,
67
NotificationConfiguration,
@@ -21,5 +22,11 @@ class S3Store(BaseStore):
2122
# maps bucket name to bucket's replication settings
2223
bucket_replication: Dict[BucketName, ReplicationConfiguration] = LocalAttribute(default=dict)
2324

25+
# maps bucket name to bucket's lifecycle configuration
26+
# TODO: need to check "globality" of parameters / redirect
27+
bucket_lifecycle_configuration: Dict[BucketName, BucketLifecycleConfiguration] = LocalAttribute(
28+
default=dict
29+
)
30+
2431

2532
s3_stores = AccountRegionBundle("s3", S3Store)

0 commit comments

Comments
 (0)