Skip to content

Commit fd833ce

Browse files
dhermesdhermes
committed
Moving behavior in Connection.lookup to datastore.api.
This was so that Connection.lookup was only performing the API call by the same name. The "lookup until all deferred return" behavior has been moving into `datastore.get` (which is where end users expect the behavior).
1 parent 57e11b1 commit fd833ce

File tree

4 files changed

+280
-175
lines changed

4 files changed

+280
-175
lines changed

‎gcloud/datastore/api.py‎

Lines changed: 85 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -24,6 +24,10 @@
2424
from gcloud.datastore import helpers
2525

2626

27+
_MAX_LOOPS = 128
28+
"""Maximum number of iterations to wait for deferred keys."""
29+
30+
2731
def _require_dataset_id(dataset_id=None, first_key=None):
2832
"""Infer a dataset ID from the environment, if not passed explicitly.
2933
@@ -80,6 +84,85 @@ def _require_connection(connection=None):
8084
return connection
8185

8286

87+
def _extended_lookup(connection, dataset_id, key_pbs,
88+
missing=None, deferred=None,
89+
eventual=False, transaction_id=None):
90+
"""Repeat lookup until all keys found (unless stop requested).
91+
92+
Helper method for :func:`get`.
93+
94+
:type connection: :class:`gcloud.datastore.connection.Connection`
95+
:param connection: The connection used to connect to datastore.
96+
97+
:type dataset_id: string
98+
:param dataset_id: The ID of the dataset of which to make the request.
99+
100+
:type key_pbs: list of :class:`gcloud.datastore._datastore_v1_pb2.Key`
101+
:param key_pbs: The keys to retrieve from the datastore.
102+
103+
:type missing: an empty list or None.
104+
:param missing: If a list is passed, the key-only entity protobufs
105+
returned by the backend as "missing" will be copied
106+
into it. Use only as a keyword param.
107+
108+
:type deferred: an empty list or None.
109+
:param deferred: If a list is passed, the key protobufs returned
110+
by the backend as "deferred" will be copied into it.
111+
Use only as a keyword param.
112+
113+
:type eventual: boolean
114+
:param eventual: If False (the default), request ``STRONG`` read
115+
consistency. If True, request ``EVENTUAL`` read
116+
consistency.
117+
118+
:type transaction_id: string
119+
:param transaction_id: If passed, make the request in the scope of
120+
the given transaction. Incompatible with
121+
``eventual==True``.
122+
123+
:rtype: list of :class:`gcloud.datastore._datastore_v1_pb2.Entity`
124+
:returns: The requested entities.
125+
:raises: :class:`ValueError` if missing / deferred are not null or
126+
empty list.
127+
"""
128+
if missing is not None and missing != []:
129+
raise ValueError('missing must be None or an empty list')
130+
131+
if deferred is not None and deferred != []:
132+
raise ValueError('deferred must be None or an empty list')
133+
134+
results = []
135+
136+
loop_num = 0
137+
while loop_num < _MAX_LOOPS: # loop against possible deferred.
138+
loop_num += 1
139+
140+
results_found, missing_found, deferred_found = connection.lookup(
141+
dataset_id=dataset_id,
142+
key_pbs=key_pbs,
143+
eventual=eventual,
144+
transaction_id=transaction_id,
145+
)
146+
147+
results.extend(results_found)
148+
149+
if missing is not None:
150+
missing.extend(missing_found)
151+
152+
if deferred is not None:
153+
deferred.extend(deferred_found)
154+
break
155+
156+
if len(deferred_found) == 0:
157+
break
158+
159+
# We have deferred keys, and the user didn't ask to know about
160+
# them, so retry (but only with the deferred ones).
161+
key_pbs = deferred_found
162+
163+
return results
164+
165+
83166
def get(keys, missing=None, deferred=None, connection=None, dataset_id=None):
84167
"""Retrieves entities, along with their attributes.
85168
@@ -122,7 +205,8 @@ def get(keys, missing=None, deferred=None, connection=None, dataset_id=None):
122205

123206
transaction = Transaction.current()
124207

125-
entity_pbs = connection.lookup(
208+
entity_pbs = _extended_lookup(
209+
connection,
126210
dataset_id=dataset_id,
127211
key_pbs=[k.to_protobuf() for k in keys],
128212
missing=missing,

‎gcloud/datastore/connection.py‎

Lines changed: 14 additions & 74 deletions
Original file line numberDiff line numberDiff line change
@@ -122,7 +122,6 @@ def build_api_url(cls, dataset_id, method, base_url=None,
122122
dataset_id=dataset_id, method=method)
123123

124124
def lookup(self, dataset_id, key_pbs,
125-
missing=None, deferred=None,
126125
eventual=False, transaction_id=None):
127126
"""Lookup keys from a dataset in the Cloud Datastore.
128127
@@ -150,16 +149,6 @@ def lookup(self, dataset_id, key_pbs,
150149
:type key_pbs: list of :class:`gcloud.datastore._datastore_v1_pb2.Key`
151150
:param key_pbs: The keys to retrieve from the datastore.
152151
153-
:type missing: an empty list or None.
154-
:param missing: If a list is passed, the key-only entity protobufs
155-
returned by the backend as "missing" will be copied
156-
into it. Use only as a keyword param.
157-
158-
:type deferred: an empty list or None.
159-
:param deferred: If a list is passed, the key protobufs returned
160-
by the backend as "deferred" will be copied into it.
161-
Use only as a keyword param.
162-
163152
:type eventual: boolean
164153
:param eventual: If False (the default), request ``STRONG`` read
165154
consistency. If True, request ``EVENTUAL`` read
@@ -170,35 +159,24 @@ def lookup(self, dataset_id, key_pbs,
170159
the given transaction. Incompatible with
171160
``eventual==True``.
172161
173-
:rtype: list of :class:`gcloud.datastore._datastore_v1_pb2.Entity`
174-
(or a single Entity)
175-
:returns: The entities corresponding to the keys provided.
176-
If a single key was provided and no results matched,
177-
this will return None.
178-
If multiple keys were provided and no results matched,
179-
this will return an empty list.
180-
:raises: ValueError if ``eventual`` is True
162+
:rtype: tuple
163+
:returns: A triple of (``results``, ``missing``, ``deferred``) where
164+
both ``results`` and ``missing`` are lists of
165+
:class:`gcloud.datastore._datastore_v1_pb2.Entity` and
166+
``deferred`` is a list of
167+
:class:`gcloud.datastore._datastore_v1_pb2.Key`.
181168
"""
182-
if missing is not None and missing != []:
183-
raise ValueError('missing must be None or an empty list')
184-
185-
if deferred is not None and deferred != []:
186-
raise ValueError('deferred must be None or an empty list')
187-
188169
lookup_request = datastore_pb.LookupRequest()
189170
_set_read_options(lookup_request, eventual, transaction_id)
190171
helpers._add_keys_to_request(lookup_request.key, key_pbs)
191172

192-
results, missing_found, deferred_found = self._lookup(
193-
lookup_request, dataset_id, deferred is not None)
194-
195-
if missing is not None:
196-
missing.extend(missing_found)
173+
lookup_response = self._rpc(dataset_id, 'lookup', lookup_request,
174+
datastore_pb.LookupResponse)
197175

198-
if deferred is not None:
199-
deferred.extend(deferred_found)
176+
results = [result.entity for result in lookup_response.found]
177+
missing = [result.entity for result in lookup_response.missing]
200178

201-
return results
179+
return results, missing, lookup_response.deferred
202180

203181
def run_query(self, dataset_id, query_pb, namespace=None,
204182
eventual=False, transaction_id=None):
@@ -376,41 +354,14 @@ def allocate_ids(self, dataset_id, key_pbs):
376354
datastore_pb.AllocateIdsResponse)
377355
return list(response.key)
378356

379-
def _lookup(self, lookup_request, dataset_id, stop_on_deferred):
380-
"""Repeat lookup until all keys found (unless stop requested).
381-
382-
Helper method for ``lookup()``.
383-
"""
384-
results = []
385-
missing = []
386-
deferred = []
387-
while True: # loop against possible deferred.
388-
lookup_response = self._rpc(dataset_id, 'lookup', lookup_request,
389-
datastore_pb.LookupResponse)
390-
391-
results.extend(
392-
[result.entity for result in lookup_response.found])
393-
394-
missing.extend(
395-
[result.entity for result in lookup_response.missing])
396-
397-
if stop_on_deferred:
398-
deferred.extend([key for key in lookup_response.deferred])
399-
break
400-
401-
if not lookup_response.deferred:
402-
break
403-
404-
# We have deferred keys, and the user didn't ask to know about
405-
# them, so retry (but only with the deferred ones).
406-
_copy_deferred_keys(lookup_request, lookup_response)
407-
return results, missing, deferred
408-
409357

410358
def _set_read_options(request, eventual, transaction_id):
411359
"""Validate rules for read options, and assign to the request.
412360
413361
Helper method for ``lookup()`` and ``run_query``.
362+
363+
:raises: :class:`ValueError` if ``eventual`` is ``True`` and the
364+
``transaction_id`` is not ``None``.
414365
"""
415366
if eventual and (transaction_id is not None):
416367
raise ValueError('eventual must be False when in a transaction')
@@ -420,14 +371,3 @@ def _set_read_options(request, eventual, transaction_id):
420371
opts.read_consistency = datastore_pb.ReadOptions.EVENTUAL
421372
elif transaction_id:
422373
opts.transaction = transaction_id
423-
424-
425-
def _copy_deferred_keys(lookup_request, lookup_response):
426-
"""Clear requested keys and copy deferred keys back in.
427-
428-
Helper for ``Connection.lookup()``.
429-
"""
430-
for old_key in list(lookup_request.key):
431-
lookup_request.key.remove(old_key)
432-
for def_key in lookup_response.deferred:
433-
lookup_request.key.add().CopyFrom(def_key)

0 commit comments

Comments
 (0)