Skip to content

Commit ab94efd

Browse files
andrewsgandrewsg
authored
Feat: Add restore_bucket and handling for soft-deleted buckets (#1365)
1 parent 42392ef commit ab94efd

File tree

6 files changed

+412
-18
lines changed

6 files changed

+412
-18
lines changed

‎google/cloud/storage/_helpers.py‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -293,6 +293,9 @@ def reload(
293293
)
294294
if soft_deleted is not None:
295295
query_params["softDeleted"] = soft_deleted
296+
# Soft delete reload requires a generation, even for targets
297+
# that don't include them in default query params (buckets).
298+
query_params["generation"] = self.generation
296299
headers = self._encryption_headers()
297300
_add_etag_match_headers(
298301
headers, if_etag_match=if_etag_match, if_etag_not_match=if_etag_not_match

‎google/cloud/storage/bucket.py‎

Lines changed: 63 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -626,6 +626,10 @@ class Bucket(_PropertyMixin):
626626
:type user_project: str
627627
:param user_project: (Optional) the project ID to be billed for API
628628
requests made via this instance.
629+
630+
:type generation: int
631+
:param generation: (Optional) If present, selects a specific revision of
632+
this bucket.
629633
"""
630634

631635
_MAX_OBJECTS_FOR_ITERATION = 256
@@ -659,7 +663,7 @@ class Bucket(_PropertyMixin):
659663
)
660664
"""Allowed values for :attr:`location_type`."""
661665

662-
def __init__(self, client, name=None, user_project=None):
666+
def __init__(self, client, name=None, user_project=None, generation=None):
663667
"""
664668
property :attr:`name`
665669
Get the bucket's name.
@@ -672,6 +676,9 @@ def __init__(self, client, name=None, user_project=None):
672676
self._label_removals = set()
673677
self._user_project = user_project
674678

679+
if generation is not None:
680+
self._properties["generation"] = generation
681+
675682
def __repr__(self):
676683
return f"<Bucket: {self.name}>"
677684

@@ -726,6 +733,50 @@ def user_project(self):
726733
"""
727734
return self._user_project
728735

736+
@property
737+
def generation(self):
738+
"""Retrieve the generation for the bucket.
739+
740+
:rtype: int or ``NoneType``
741+
:returns: The generation of the bucket or ``None`` if the bucket's
742+
resource has not been loaded from the server.
743+
"""
744+
generation = self._properties.get("generation")
745+
if generation is not None:
746+
return int(generation)
747+
748+
@property
749+
def soft_delete_time(self):
750+
"""If this bucket has been soft-deleted, returns the time at which it became soft-deleted.
751+
752+
:rtype: :class:`datetime.datetime` or ``NoneType``
753+
:returns:
754+
(readonly) The time that the bucket became soft-deleted.
755+
Note this property is only set for soft-deleted buckets.
756+
"""
757+
soft_delete_time = self._properties.get("softDeleteTime")
758+
if soft_delete_time is not None:
759+
return _rfc3339_nanos_to_datetime(soft_delete_time)
760+
761+
@property
762+
def hard_delete_time(self):
763+
"""If this bucket has been soft-deleted, returns the time at which it will be permanently deleted.
764+
765+
:rtype: :class:`datetime.datetime` or ``NoneType``
766+
:returns:
767+
(readonly) The time that the bucket will be permanently deleted.
768+
Note this property is only set for soft-deleted buckets.
769+
"""
770+
hard_delete_time = self._properties.get("hardDeleteTime")
771+
if hard_delete_time is not None:
772+
return _rfc3339_nanos_to_datetime(hard_delete_time)
773+
774+
@property
775+
def _query_params(self):
776+
"""Default query parameters."""
777+
params = super()._query_params
778+
return params
779+
729780
@classmethod
730781
def from_string(cls, uri, client=None):
731782
"""Get a constructor for bucket object by URI.
@@ -1045,6 +1096,7 @@ def reload(
10451096
if_metageneration_match=None,
10461097
if_metageneration_not_match=None,
10471098
retry=DEFAULT_RETRY,
1099+
soft_deleted=None,
10481100
):
10491101
"""Reload properties from Cloud Storage.
10501102
@@ -1084,6 +1136,13 @@ def reload(
10841136
:type retry: google.api_core.retry.Retry or google.cloud.storage.retry.ConditionalRetryPolicy
10851137
:param retry:
10861138
(Optional) How to retry the RPC. See: :ref:`configuring_retries`
1139+
1140+
:type soft_deleted: bool
1141+
:param soft_deleted: (Optional) If True, looks for a soft-deleted
1142+
bucket. Will only return the bucket metadata if the bucket exists
1143+
and is in a soft-deleted state. The bucket ``generation`` must be
1144+
set if ``soft_deleted`` is set to True.
1145+
See: https://cloud.google.com/storage/docs/soft-delete
10871146
"""
10881147
super(Bucket, self).reload(
10891148
client=client,
@@ -1094,6 +1153,7 @@ def reload(
10941153
if_metageneration_match=if_metageneration_match,
10951154
if_metageneration_not_match=if_metageneration_not_match,
10961155
retry=retry,
1156+
soft_deleted=soft_deleted,
10971157
)
10981158

10991159
@create_trace_span(name="Storage.Bucket.patch")
@@ -2159,8 +2219,8 @@ def restore_blob(
21592219
:param client: (Optional) The client to use. If not passed, falls back
21602220
to the ``client`` stored on the current bucket.
21612221
2162-
:type generation: long
2163-
:param generation: (Optional) If present, selects a specific revision of this object.
2222+
:type generation: int
2223+
:param generation: Selects the specific revision of the object.
21642224
21652225
:type copy_source_acl: bool
21662226
:param copy_source_acl: (Optional) If true, copy the soft-deleted object's access controls.

‎google/cloud/storage/client.py‎

Lines changed: 122 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -30,6 +30,7 @@
3030
from google.cloud.client import ClientWithProject
3131
from google.cloud.exceptions import NotFound
3232

33+
from google.cloud.storage._helpers import _add_generation_match_parameters
3334
from google.cloud.storage._helpers import _bucket_bound_hostname_url
3435
from google.cloud.storage._helpers import _get_api_endpoint_override
3536
from google.cloud.storage._helpers import _get_environ_project
@@ -367,7 +368,7 @@ def get_service_account_email(
367368
api_response = self._get_resource(path, timeout=timeout, retry=retry)
368369
return api_response["email_address"]
369370

370-
def bucket(self, bucket_name, user_project=None):
371+
def bucket(self, bucket_name, user_project=None, generation=None):
371372
"""Factory constructor for bucket object.
372373
373374
.. note::
@@ -381,10 +382,19 @@ def bucket(self, bucket_name, user_project=None):
381382
:param user_project: (Optional) The project ID to be billed for API
382383
requests made via the bucket.
383384
385+
:type generation: int
386+
:param generation: (Optional) If present, selects a specific revision of
387+
this bucket.
388+
384389
:rtype: :class:`google.cloud.storage.bucket.Bucket`
385390
:returns: The bucket object created.
386391
"""
387-
return Bucket(client=self, name=bucket_name, user_project=user_project)
392+
return Bucket(
393+
client=self,
394+
name=bucket_name,
395+
user_project=user_project,
396+
generation=generation,
397+
)
388398

389399
def batch(self, raise_exception=True):
390400
"""Factory constructor for batch object.
@@ -789,7 +799,7 @@ def _delete_resource(
789799
_target_object=_target_object,
790800
)
791801

792-
def _bucket_arg_to_bucket(self, bucket_or_name):
802+
def _bucket_arg_to_bucket(self, bucket_or_name, generation=None):
793803
"""Helper to return given bucket or create new by name.
794804
795805
Args:
@@ -798,17 +808,27 @@ def _bucket_arg_to_bucket(self, bucket_or_name):
798808
str, \
799809
]):
800810
The bucket resource to pass or name to create.
811+
generation (Optional[int]):
812+
The bucket generation. If generation is specified,
813+
bucket_or_name must be a name (str).
801814
802815
Returns:
803816
google.cloud.storage.bucket.Bucket
804817
The newly created bucket or the given one.
805818
"""
806819
if isinstance(bucket_or_name, Bucket):
820+
if generation:
821+
raise ValueError(
822+
"The generation can only be specified if a "
823+
"name is used to specify a bucket, not a Bucket object. "
824+
"Create a new Bucket object with the correct generation "
825+
"instead."
826+
)
807827
bucket = bucket_or_name
808828
if bucket.client is None:
809829
bucket._client = self
810830
else:
811-
bucket = Bucket(self, name=bucket_or_name)
831+
bucket = Bucket(self, name=bucket_or_name, generation=generation)
812832
return bucket
813833

814834
@create_trace_span(name="Storage.Client.getBucket")
@@ -819,6 +839,9 @@ def get_bucket(
819839
if_metageneration_match=None,
820840
if_metageneration_not_match=None,
821841
retry=DEFAULT_RETRY,
842+
*,
843+
generation=None,
844+
soft_deleted=None,
822845
):
823846
"""Retrieve a bucket via a GET request.
824847
@@ -837,12 +860,12 @@ def get_bucket(
837860
Can also be passed as a tuple (connect_timeout, read_timeout).
838861
See :meth:`requests.Session.request` documentation for details.
839862
840-
if_metageneration_match (Optional[long]):
863+
if_metageneration_match (Optional[int]):
841864
Make the operation conditional on whether the
842-
blob's current metageneration matches the given value.
865+
bucket's current metageneration matches the given value.
843866
844-
if_metageneration_not_match (Optional[long]):
845-
Make the operation conditional on whether the blob's
867+
if_metageneration_not_match (Optional[int]):
868+
Make the operation conditional on whether the bucket's
846869
current metageneration does not match the given value.
847870
848871
retry (Optional[Union[google.api_core.retry.Retry, google.cloud.storage.retry.ConditionalRetryPolicy]]):
@@ -859,6 +882,19 @@ def get_bucket(
859882
See the retry.py source code and docstrings in this package (google.cloud.storage.retry) for
860883
information on retry types and how to configure them.
861884
885+
generation (Optional[int]):
886+
The generation of the bucket. The generation can be used to
887+
specify a specific soft-deleted version of the bucket, in
888+
conjunction with the ``soft_deleted`` argument below. If
889+
``soft_deleted`` is not True, the generation is unused.
890+
891+
soft_deleted (Optional[bool]):
892+
If True, looks for a soft-deleted bucket. Will only return
893+
the bucket metadata if the bucket exists and is in a
894+
soft-deleted state. The bucket ``generation`` is required if
895+
``soft_deleted`` is set to True.
896+
See: https://cloud.google.com/storage/docs/soft-delete
897+
862898
Returns:
863899
google.cloud.storage.bucket.Bucket
864900
The bucket matching the name provided.
@@ -867,13 +903,14 @@ def get_bucket(
867903
google.cloud.exceptions.NotFound
868904
If the bucket is not found.
869905
"""
870-
bucket = self._bucket_arg_to_bucket(bucket_or_name)
906+
bucket = self._bucket_arg_to_bucket(bucket_or_name, generation=generation)
871907
bucket.reload(
872908
client=self,
873909
timeout=timeout,
874910
if_metageneration_match=if_metageneration_match,
875911
if_metageneration_not_match=if_metageneration_not_match,
876912
retry=retry,
913+
soft_deleted=soft_deleted,
877914
)
878915
return bucket
879916

@@ -1386,6 +1423,8 @@ def list_buckets(
13861423
page_size=None,
13871424
timeout=_DEFAULT_TIMEOUT,
13881425
retry=DEFAULT_RETRY,
1426+
*,
1427+
soft_deleted=None,
13891428
):
13901429
"""Get all buckets in the project associated to the client.
13911430
@@ -1438,6 +1477,12 @@ def list_buckets(
14381477
:param retry:
14391478
(Optional) How to retry the RPC. See: :ref:`configuring_retries`
14401479
1480+
:type soft_deleted: bool
1481+
:param soft_deleted:
1482+
(Optional) If true, only soft-deleted buckets will be listed as distinct results in order of increasing
1483+
generation number. This parameter can only be used successfully if the bucket has a soft delete policy.
1484+
See: https://cloud.google.com/storage/docs/soft-delete
1485+
14411486
:rtype: :class:`~google.api_core.page_iterator.Iterator`
14421487
:raises ValueError: if both ``project`` is ``None`` and the client's
14431488
project is also ``None``.
@@ -1469,6 +1514,9 @@ def list_buckets(
14691514
if fields is not None:
14701515
extra_params["fields"] = fields
14711516

1517+
if soft_deleted is not None:
1518+
extra_params["softDeleted"] = soft_deleted
1519+
14721520
return self._list_resource(
14731521
"/b",
14741522
_item_to_bucket,
@@ -1480,6 +1528,71 @@ def list_buckets(
14801528
retry=retry,
14811529
)
14821530

1531+
def restore_bucket(
1532+
self,
1533+
bucket_name,
1534+
generation,
1535+
projection="noAcl",
1536+
if_metageneration_match=None,
1537+
if_metageneration_not_match=None,
1538+
timeout=_DEFAULT_TIMEOUT,
1539+
retry=DEFAULT_RETRY,
1540+
):
1541+
"""Restores a soft-deleted bucket.
1542+
1543+
:type bucket_name: str
1544+
:param bucket_name: The name of the bucket to be restored.
1545+
1546+
:type generation: int
1547+
:param generation: Selects the specific revision of the bucket.
1548+
1549+
:type projection: str
1550+
:param projection:
1551+
(Optional) Specifies the set of properties to return. If used, must
1552+
be 'full' or 'noAcl'. Defaults to 'noAcl'.
1553+
1554+
if_metageneration_match (Optional[int]):
1555+
Make the operation conditional on whether the
1556+
blob's current metageneration matches the given value.
1557+
1558+
if_metageneration_not_match (Optional[int]):
1559+
Make the operation conditional on whether the blob's
1560+
current metageneration does not match the given value.
1561+
1562+
:type timeout: float or tuple
1563+
:param timeout:
1564+
(Optional) The amount of time, in seconds, to wait
1565+
for the server response. See: :ref:`configuring_timeouts`
1566+
1567+
:type retry: google.api_core.retry.Retry or google.cloud.storage.retry.ConditionalRetryPolicy
1568+
:param retry:
1569+
(Optional) How to retry the RPC.
1570+
1571+
Users can configure non-default retry behavior. A ``None`` value will
1572+
disable retries. See [Configuring Retries](https://cloud.google.com/python/docs/reference/storage/latest/retry_timeout).
1573+
1574+
:rtype: :class:`google.cloud.storage.bucket.Bucket`
1575+
:returns: The restored Bucket.
1576+
"""
1577+
query_params = {"generation": generation, "projection": projection}
1578+
1579+
_add_generation_match_parameters(
1580+
query_params,
1581+
if_metageneration_match=if_metageneration_match,
1582+
if_metageneration_not_match=if_metageneration_not_match,
1583+
)
1584+
1585+
bucket = self.bucket(bucket_name)
1586+
api_response = self._post_resource(
1587+
f"{bucket.path}/restore",
1588+
None,
1589+
query_params=query_params,
1590+
timeout=timeout,
1591+
retry=retry,
1592+
)
1593+
bucket._set_properties(api_response)
1594+
return bucket
1595+
14831596
@create_trace_span(name="Storage.Client.createHmacKey")
14841597
def create_hmac_key(
14851598
self,

0 commit comments

Comments
 (0)